From 9de5a3d087be74e5ee3001a04fe9d92e5913f091 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 11 Sep 2025 13:21:35 +0000 Subject: [PATCH 01/45] Initial commit --- dev/parm/config/gfs/config.atmanl.j2 | 24 +--- dev/parm/config/gfs/config.atmensanl.j2 | 13 +- dev/parm/config/gfs/config.ecen_fv3jedi | 6 - dev/parm/config/gfs/yaml/defaults.yaml | 2 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/atm_analysis.py | 134 ++++++--------------- ush/python/pygfs/task/atmens_analysis.py | 21 +--- ush/python/pygfs/task/ensemble_recenter.py | 22 +--- 8 files changed, 55 insertions(+), 169 deletions(-) diff --git a/dev/parm/config/gfs/config.atmanl.j2 b/dev/parm/config/gfs/config.atmanl.j2 index 14d2acfa959..7ebe69be538 100644 --- a/dev/parm/config/gfs/config.atmanl.j2 +++ b/dev/parm/config/gfs/config.atmanl.j2 @@ -5,33 +5,15 @@ echo "BEGIN: config.atmanl" +export OBS_LIST_YAML="{{ OBS_LIST_YAML }}" export JCB_ALGO_YAML_VAR="{{ JCB_ALGO_YAML_VAR }}" export JCB_ALGO_YAML_FV3INC="{{ JCB_ALGO_YAML_FV3INC }}" - export STATICB_TYPE="{{ STATICB_TYPE }}" -export LOCALIZATION_TYPE="bump" -export INTERP_METHOD='barycentric' - -if [[ ${DOHYBVAR} = "YES" ]]; then - # shellcheck disable=SC2153 - export CASE_ANL=${CASE_ENS} - export BERROR_YAML="atmosphere_background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}" -else - export CASE_ANL=${CASE} - export BERROR_YAML="atmosphere_background_error_static_${STATICB_TYPE}" -fi - -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_det_jedi_config.yaml.j2" -export STAGE_CRTM_COEFF_YAML="${PARMgfs}/gdas/atm/atm_stage_crtm_coeff.yaml.j2" -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/atm/atm_stage_jedi_fix.yaml.j2" -export STAGE_BKG_YAML="${PARMgfs}/gdas/atm/atm_det_stage_bkg.yaml.j2" -export STAGE_BERROR_YAML="${PARMgfs}/gdas/atm/atm_det_stage_berror_${STATICB_TYPE}.yaml.j2" -export STAGE_FV3ENS_YAML="${PARMgfs}/gdas/atm/atm_det_stage_fv3ens.yaml.j2" - export layout_x_atmanl="{{ LAYOUT_X_ATMANL }}" export layout_y_atmanl="{{ LAYOUT_Y_ATMANL }}" - export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_det_config.yaml.j2" + echo "END: config.atmanl" diff --git a/dev/parm/config/gfs/config.atmensanl.j2 b/dev/parm/config/gfs/config.atmensanl.j2 index a8d72c7889d..c8f333ca8f1 100644 --- a/dev/parm/config/gfs/config.atmensanl.j2 +++ b/dev/parm/config/gfs/config.atmensanl.j2 @@ -5,22 +5,19 @@ echo "BEGIN: config.atmensanl" +export OBS_LIST_YAML="{{ OBS_LIST_YAML }}" export JCB_ALGO_YAML_LETKF="{{ JCB_ALGO_YAML_LETKF }}" export JCB_ALGO_YAML_OBS="{{ JCB_ALGO_YAML_OBS }}" export JCB_ALGO_YAML_SOL="{{ JCB_ALGO_YAML_SOL }}" export JCB_ALGO_YAML_FV3INC="{{ JCB_ALGO_YAML_FV3INC }}" - -export INTERP_METHOD='barycentric' +export layout_x_atmensanl="{{ LAYOUT_X_ATMENSANL }}" +export layout_y_atmensanl="{{ LAYOUT_Y_ATMENSANL }}" +export io_layout_x="{{ IO_LAYOUT_X }}" +export io_layout_y="{{ IO_LAYOUT_Y }}" export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ens_jedi_config.yaml.j2" export STAGE_CRTM_COEFF_YAML="${PARMgfs}/gdas/atm/atm_stage_crtm_coeff.yaml.j2" export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/atm/atm_stage_jedi_fix.yaml.j2" export STAGE_BKG_YAML="${PARMgfs}/gdas/atm/atm_ens_stage_bkg.yaml.j2" -export layout_x_atmensanl="{{ LAYOUT_X_ATMENSANL }}" -export layout_y_atmensanl="{{ LAYOUT_Y_ATMENSANL }}" - -export io_layout_x="{{ IO_LAYOUT_X }}" -export io_layout_y="{{ IO_LAYOUT_Y }}" - echo "END: config.atmensanl" diff --git a/dev/parm/config/gfs/config.ecen_fv3jedi b/dev/parm/config/gfs/config.ecen_fv3jedi index bf33719bc7d..118c1767348 100644 --- a/dev/parm/config/gfs/config.ecen_fv3jedi +++ b/dev/parm/config/gfs/config.ecen_fv3jedi @@ -18,10 +18,4 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ecen_jedi_config.yaml.j2" export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/atm/atm_stage_jedi_fix.yaml.j2" export STAGE_YAML="${PARMgfs}/gdas/atm/atm_ecen_stage.yaml.j2" -if [[ ${DOHYBVAR} = "YES" ]]; then - export CASE_ANL=${CASE_ENS} -else - export CASE_ANL=${CASE} -fi - echo "END: config.ecen_fv3jedi" diff --git a/dev/parm/config/gfs/yaml/defaults.yaml b/dev/parm/config/gfs/yaml/defaults.yaml index 5802aa9aa9d..b26ddde38d7 100644 --- a/dev/parm/config/gfs/yaml/defaults.yaml +++ b/dev/parm/config/gfs/yaml/defaults.yaml @@ -29,6 +29,7 @@ base: NMEM_ENS_GFS_OFFSET: 20 atmanl: + OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" JCB_ALGO_YAML_FV3INC: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar-fv3inc.yaml.j2" STATICB_TYPE: "gsibec" @@ -38,6 +39,7 @@ atmanl: IO_LAYOUT_Y: 1 atmensanl: + OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 560fcbf6464..24aae688b05 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 560fcbf6464d0285e8b0603a05d3fd0380327b4a +Subproject commit 24aae688b053f0930b154097b9f6584816722c9a diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index dae5469a1c3..ede93db233d 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -7,20 +7,21 @@ from logging import getLogger from pprint import pformat from typing import Any, Dict -from wxflow import (AttrDict, FileHandler, Task, +from wxflow import (AttrDict, FileHandler, add_to_datetime, to_timedelta, parse_j2yaml, logit) +from pygfs.task.atm_analysis import AtmAnalysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmAnalysis(Task): +class AtmDetAnalysis(AtmAnalysis): """ - Class for JEDI-based global atm analysis tasks + Class for JEDI-based global atm deterministic analysis tasks """ - @logit(logger, name="AtmAnalysis") + @logit(logger, name="AtmDetAnalysis") def __init__(self, config: Dict[str, Any]): """Constructor global atm analysis task @@ -40,30 +41,16 @@ def __init__(self, config: Dict[str, Any]): """ super().__init__(config) - _res = int(self.task_config.CASE[1:]) - _res_anl = int(self.task_config.CASE_ANL[1:]) - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _localization_type = 'bump' + if self.task_config.DOHYBVAR: + _BERROR_YAML="atmosphere_background_error_hybrid_${self.task_config.STATICB_TYPE}_${_localization_type}" + else: + _BERROR_YAML="atmosphere_background_error_static_${self.task_config.STATICB_TYPE}" # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { - 'npx_ges': _res + 1, - 'npy_ges': _res + 1, - 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, - 'npx_anl': _res_anl + 1, - 'npy_anl': _res_anl + 1, - 'npz_anl': self.task_config.LEVS - 1, - 'ATM_WINDOW_BEGIN': _window_begin, - 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", - 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX_ENS': f"enkf{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", - 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + 'BERROR_YAML': _BERROR_YAML, } ) @@ -98,15 +85,12 @@ def initialize(self) -> None: None """ - # stage observations - logger.info(f"Staging list of observation files") - obs_dict = self.jedi_dict['atmanlvar'].render_jcb(self.task_config, 'atm_obs_staging') - FileHandler(obs_dict).sync() - logger.debug(f"Observation files:\n{pformat(obs_dict)}") + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.stage).sync() - # stage bias corrections - logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi_dict['atmanlvar'].render_jcb(self.task_config, 'atm_bias_staging') + # Extract bias corrections from tar files + logger.info(f"Extracting bias corrections from tar files") if bias_dict['copy'] is None: logger.info(f"No bias correction files to stage") else: @@ -117,48 +101,6 @@ def initialize(self) -> None: # extract bias corrections Jedi.extract_tar_from_filehandler_dict(bias_dict) - # stage CRTM fix files - logger.info(f"Staging CRTM fix files from {self.task_config.STAGE_CRTM_COEFF_YAML}") - crtm_fix_dict = parse_j2yaml(self.task_config.STAGE_CRTM_COEFF_YAML, self.task_config) - FileHandler(crtm_fix_dict).sync() - logger.debug(f"CRTM fix files:\n{pformat(crtm_fix_dict)}") - - # stage fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_dict = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_dict).sync() - logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") - - # stage static background error files, otherwise it will assume ID matrix - logger.info(f"Stage files for STATICB_TYPE {self.task_config.STATICB_TYPE}") - if self.task_config.STATICB_TYPE != 'identity': - berror_staging_dict = parse_j2yaml(self.task_config.STAGE_BERROR_YAML, self.task_config) - else: - berror_staging_dict = {} - FileHandler(berror_staging_dict).sync() - logger.debug(f"Background error files:\n{pformat(berror_staging_dict)}") - - # stage ensemble files for use in hybrid background error - if self.task_config.DOHYBVAR: - logger.debug(f"Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") - fv3ens_staging_dict = parse_j2yaml(self.task_config.STAGE_FV3ENS_YAML, self.task_config) - FileHandler(fv3ens_staging_dict).sync() - logger.debug(f"Ensemble files:\n{pformat(fv3ens_staging_dict)}") - - # stage backgrounds - logger.info(f"Staging background files from {self.task_config.STAGE_BKG_YAML}") - bkg_staging_dict = parse_j2yaml(self.task_config.STAGE_BKG_YAML, self.task_config) - FileHandler(bkg_staging_dict).sync() - logger.debug(f"Background files:\n{pformat(bkg_staging_dict)}") - - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config.DATA, 'anl'), - os.path.join(self.task_config.DATA, 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - # initialize JEDI variational application logger.info(f"Initializing JEDI variational DA application") self.jedi_dict['atmanlvar'].initialize(self.task_config, clean_empty_obsspaces=True) @@ -225,18 +167,18 @@ def finalize(self) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # get list of yamls to copy to ROTDIR - yamls = glob.glob(os.path.join(self.task_config.DATA, '*atm*yaml')) +# yamls = glob.glob(os.path.join(self.task_config.DATA, '*atm*yaml')) # copy full YAML from executable to ROTDIR - for src in yamls: - yaml_base = os.path.splitext(os.path.basename(src))[0] - dest_yaml_name = f"{self.task_config.APREFIX}{yaml_base}.yaml" - dest = os.path.join(self.task_config.COMOUT_CONF, dest_yaml_name) - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() +# for src in yamls: +# yaml_base = os.path.splitext(os.path.basename(src))[0] +# dest_yaml_name = f"{self.task_config.APREFIX}{yaml_base}.yaml" +# dest = os.path.join(self.task_config.COMOUT_CONF, dest_yaml_name) +# logger.debug(f"Copying {src} to {dest}") +# yaml_copy = { +# 'copy': [[src, dest]] +# } +# FileHandler(yaml_copy).sync() # path of output radiance bias correction tarfile bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" @@ -268,18 +210,18 @@ def finalize(self) -> None: logger.info(f"Add {radbcor.getnames()}") # Copy FV3 atm increment to comrot directory - logger.info("Copy UFS model readable atm increment file") - inc_copy = {'copy': []} - for itile in range(6): - src = os.path.join(self.task_config.DATA, "anl", - f"{self.task_config.APREFIX}cubed_sphere_grid_atminc.tile{itile+1}.nc") - dest = self.task_config.COMOUT_ATMOS_ANALYSIS - inc_copy['copy'].append([src, dest]) - - # copy increments - src_list, dest_list = zip(*inc_copy['copy']) - logger.debug(f"Copying {src_list}\nto {dest_list}") - FileHandler(inc_copy).sync() +# logger.info("Copy UFS model readable atm increment file") +# inc_copy = {'copy': []} +# for itile in range(6): +# src = os.path.join(self.task_config.DATA, "anl", +# f"{self.task_config.APREFIX}cubed_sphere_grid_atminc.tile{itile+1}.nc") +# dest = self.task_config.COMOUT_ATMOS_ANALYSIS +# inc_copy['copy'].append([src, dest]) + +# # copy increments +# src_list, dest_list = zip(*inc_copy['copy']) +# logger.debug(f"Copying {src_list}\nto {dest_list}") +# FileHandler(inc_copy).sync() def clean(self): super().clean() diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 6db19c11166..5fbc184310e 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -13,12 +13,13 @@ parse_j2yaml, logit, Template, TemplateConstants) +from pygfs.task.atm_analysis import AtmAnalysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmEnsAnalysis(Task): +class AtmEnsAnalysis(AtmAnalysis): """ Class for JEDI-based global atmens analysis tasks """ @@ -42,26 +43,10 @@ def __init__(self, config: Dict[str, Any]): """ super().__init__(config) - _res = int(self.task_config.CASE_ENS[1:]) - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { - 'npx_ges': _res + 1, - 'npy_ges': _res + 1, - 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, - 'ATM_WINDOW_BEGIN': _window_begin, - 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", - 'OPREFIX': f"{self.task_config.EUPD_CYC}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN.replace('enkf', '')}.t{self.task_config.cyc:02d}z.", - 'APREFIX_ENS': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'atm_obsdatain_path': f"./obs/", - 'atm_obsdataout_path': f"./diags/", - 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications + # Currently empty, but could be used in the future } ) diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index 2f50f45501e..011ed04cacc 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -4,16 +4,17 @@ from logging import getLogger import os from pprint import pformat -from pygfs.jedi import Jedi from wxflow import (AttrDict, FileHandler, Task, Executable, Template, TemplateConstants, add_to_datetime, to_timedelta, to_isotime, to_YMD, parse_j2yaml, logit) +from pygfs.task.atm_analysis import AtmAnalysis +from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class EnsembleRecenter(Task): +class EnsembleRecenter(AtmAnalysis): """ Class for JEDI-based ensemble increment recentering """ @@ -37,10 +38,6 @@ def __init__(self, config): """ super().__init__(config) - _res = int(self.task_config.CASE[1:]) - _res_anl = int(self.task_config.CASE_ANL[1:]) - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _iau_times_iso = [] for hour in self.task_config.IAUFHRS: _iau_times_iso.append(to_isotime(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2)) @@ -48,19 +45,6 @@ def __init__(self, config): # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { - 'npx_ges': _res + 1, - 'npy_ges': _res + 1, - 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, - 'npx_anl': _res_anl + 1, - 'npy_anl': _res_anl + 1, - 'npz_anl': self.task_config.LEVS - 1, - 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", - 'ATM_WINDOW_BEGIN': _window_begin, - 'APREFIX': f"{self.task_config.RUN.replace('enkf', '')}.t{self.task_config.cyc:02d}z.", - 'APREFIX_ENS': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", 'iau_times_iso': _iau_times_iso } ) From 6819212523afc135a9bed7c665577b7d78df24ed Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 11 Sep 2025 20:01:59 +0000 Subject: [PATCH 02/45] Save changes --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/atm_analysis.py | 14 ++--- ush/python/pygfs/task/atmens_analysis.py | 56 ++---------------- ush/python/pygfs/task/ensemble_recenter.py | 66 ++++------------------ 4 files changed, 21 insertions(+), 117 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 24aae688b05..e572ab4cb4b 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 24aae688b053f0930b154097b9f6584816722c9a +Subproject commit e572ab4cb4bc2ccee2517426f3d20b919865de65 diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index ede93db233d..fc58a98ef93 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -48,18 +48,15 @@ def __init__(self, config: Dict[str, Any]): _BERROR_YAML="atmosphere_background_error_static_${self.task_config.STATICB_TYPE}" # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + self.task_config.update(AttrDict( { 'BERROR_YAML': _BERROR_YAML, } - ) - - # Extend task_config with local_dict - self.task_config = AttrDict(**self.task_config, **local_dict) + )) # Create dictionary of Jedi objects expected_keys = ['atmanlvar', 'atmanlfv3inc'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -102,11 +99,8 @@ def initialize(self) -> None: Jedi.extract_tar_from_filehandler_dict(bias_dict) # initialize JEDI variational application - logger.info(f"Initializing JEDI variational DA application") + logger.info(f"Initializing JEDI applications") self.jedi_dict['atmanlvar'].initialize(self.task_config, clean_empty_obsspaces=True) - - # initialize JEDI FV3 increment conversion application - logger.info(f"Initializing JEDI FV3 increment conversion application") self.jedi_dict['atmanlfv3inc'].initialize(self.task_config) @logit(logger) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 5fbc184310e..4c858d8bebe 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -44,18 +44,15 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + self.task_config.update(AttrDict( { # Currently empty, but could be used in the future } ) - # Extend task_config with local_dict - self.task_config = AttrDict(**self.task_config, **local_dict) - # Create dictionary of JEDI objects expected_keys = ['atmensanlobs', 'atmensanlsol', 'atmensanlfv3inc', 'atmensanlletkf'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -80,58 +77,17 @@ def initialize(self) -> None: None """ - # stage observations - logger.info(f"Staging list of observation files") - obs_dict = self.jedi_dict['atmensanlobs'].render_jcb(self.task_config, 'atm_obs_staging') - FileHandler(obs_dict).sync() - logger.debug(f"Observation files:\n{pformat(obs_dict)}") - - # stage bias corrections - logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi_dict['atmensanlobs'].render_jcb(self.task_config, 'atm_bias_staging') - bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.stage).sync() # extract bias corrections Jedi.extract_tar_from_filehandler_dict(bias_dict) - # stage CRTM fix files - logger.info(f"Staging CRTM fix files from {self.task_config.STAGE_CRTM_COEFF_YAML}") - crtm_fix_dict = parse_j2yaml(self.task_config.STAGE_CRTM_COEFF_YAML, self.task_config) - FileHandler(crtm_fix_dict).sync() - logger.debug(f"CRTM fix files:\n{pformat(crtm_fix_dict)}") - - # stage fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_dict = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_dict).sync() - logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") - - # stage backgrounds - logger.info(f"Stage ensemble member background files") - bkg_staging_dict = parse_j2yaml(self.task_config.STAGE_BKG_YAML, self.task_config) - FileHandler(bkg_staging_dict).sync() - logger.debug(f"Ensemble member background files:\n{pformat(bkg_staging_dict)}") - - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config.DATA, 'anl'), - os.path.join(self.task_config.DATA, 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - - # initialize JEDI LETKF observer application + # initialize JEDI applications logger.info(f"Initializing JEDI LETKF observer application") self.jedi_dict['atmensanlobs'].initialize(self.task_config, clean_empty_obsspaces=True) - - # initialize JEDI LETKF solver application - logger.info(f"Initializing JEDI LETKF solver application") self.jedi_dict['atmensanlsol'].initialize(self.task_config) - - # initialize JEDI FV3 increment conversion application - logger.info(f"Initializing JEDI FV3 increment conversion application") self.jedi_dict['atmensanlfv3inc'].initialize(self.task_config) @logit(logger) diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index 011ed04cacc..ecdb43d2821 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -43,18 +43,15 @@ def __init__(self, config): _iau_times_iso.append(to_isotime(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2)) # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + self.task_config.update(AttrDict( { 'iau_times_iso': _iau_times_iso } ) - # Extend task_config with local_dict - self.task_config = AttrDict(**self.task_config, **local_dict) - # Create dictionary of Jedi objects expected_keys = ['correction_increment', 'ensemble_recenter'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -75,23 +72,15 @@ def initialize(self) -> None: None """ + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.stage).sync() + # Initialize JEDI ensemble increment recentering application - logger.info(f"Initializing JEDI ensemble recentering applications") + logger.info(f"Initializing JEDI applications") self.jedi_dict['correction_increment'].initialize(self.task_config) self.jedi_dict['ensemble_recenter'].initialize(self.task_config) - # Stage fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_dict = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_dict).sync() - logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") - - # Stage background and increment files - logger.info(f"Staging background and increment files from {self.task_config.STAGE_YAML}") - fh_dict = parse_j2yaml(self.task_config.STAGE_YAML, self.task_config) - FileHandler(fh_dict).sync() - logger.debug(f"JEDI background and increment files:\n{pformat(fh_dict)}") - @logit(logger) def execute(self) -> None: """Run JEDI executable @@ -130,41 +119,6 @@ def finalize(self) -> None: None """ - fh_dict = {'copy': []} - - # create template dictionaries - template_inc = self.task_config.COM_ATMOS_ANALYSIS_TMPL - tmpl_inc_dict = { - 'ROTDIR': self.task_config.ROTDIR, - 'RUN': self.task_config.RUN, - 'YMD': to_YMD(self.task_config.current_cycle), - 'HH': self.task_config.current_cycle.strftime('%H') - } - - # Copy increments to COM - for imem in range(1, self.task_config.NMEM_ENS + 1): - memchar = f"mem{imem:03d}" - tmpl_inc_dict['MEMDIR'] = memchar - incdir = Template.substitute_structure(template_inc, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_inc_dict.get) - for fh in self.task_config.IAUFHRS: - hr = format(fh, '03') - for itile in range(6): - src = os.path.join(self.task_config.DATA, memchar, - f"{self.task_config.APREFIX_ENS}cubed_sphere_grid_ratmi{hr}.tile{itile+1}.nc") - if fh == 6: - dest = os.path.join(incdir, - f"{self.task_config.APREFIX_ENS}cubed_sphere_grid_ratminc.tile{itile+1}.nc") - else: - dest = incdir - fh_dict['copy'].append([src, dest]) - - # Copy YAMLs to COM - for app_name in self.jedi_dict.keys(): - src = os.path.join(self.task_config.DATA, - f"{app_name}.yaml") - dest = os.path.join(self.task_config.COMOUT_CONF, - f"{self.task_config.APREFIX_ENS}{app_name}.yaml") - fh_dict['copy'].append([src, dest]) - - # Sync file handler - FileHandler(fh_dict).sync() + # Save output files to COM + logger.info(f"Saving output files to COM") + FileHandler(self.task_config.stage).sync() From feefae185bab55a41a5048ee45fe80efa9646782 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 15:32:19 +0000 Subject: [PATCH 03/45] Save changes --- dev/parm/config/gfs/config.ecen_fv3jedi | 4 +- ush/python/pygfs/__init__.py | 1 + ush/python/pygfs/jedi/jedi.py | 53 ++++---- ush/python/pygfs/task/atm_analysis.py | 133 ++++++++------------- ush/python/pygfs/task/atmens_analysis.py | 98 +++++---------- ush/python/pygfs/task/ensemble_recenter.py | 16 +-- ush/python/pygfs/task/fv3_analysis.py | 73 +++++++++++ 7 files changed, 187 insertions(+), 191 deletions(-) create mode 100644 ush/python/pygfs/task/fv3_analysis.py diff --git a/dev/parm/config/gfs/config.ecen_fv3jedi b/dev/parm/config/gfs/config.ecen_fv3jedi index 118c1767348..4c121e4f317 100644 --- a/dev/parm/config/gfs/config.ecen_fv3jedi +++ b/dev/parm/config/gfs/config.ecen_fv3jedi @@ -14,8 +14,6 @@ export layout_y_ecen_fv3jedi=1 # Get task specific resources source "${EXPDIR}/config.resources" ecen_fv3jedi -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ecen_jedi_config.yaml.j2" -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/atm/atm_stage_jedi_fix.yaml.j2" -export STAGE_YAML="${PARMgfs}/gdas/atm/atm_ecen_stage.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ecen_config.yaml.j2" echo "END: config.ecen_fv3jedi" diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index 394a93e8a47..d0ab9838ce7 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,6 +2,7 @@ import os from .task.analysis import Analysis +from .task.fv3_analysis import FV3Analysis from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis from .task.aero_bmatrix import AerosolBMatrix diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 0420eb37284..93cece28794 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -181,13 +181,13 @@ def render_jcb(self, task_config: AttrDict, algorithm_in: Optional[str] = None) @staticmethod @logit(logger) - def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_block_names: Optional[list] = None): + def get_jedi_dict(jedi_config_dict: dict, task_config: AttrDict, expected_block_names: Optional[list] = None): """Get dictionary of Jedi objects from YAML specifying their configuration dictionaries Parameters ---------- - jedi_config_yaml : str - path to YAML specifying configuration dictionaries for Jedi objects + jedi_config_dict : dict + dictionary parsed from a J2-YAML file specifying configuration dictionaries for JEDI objects task_config : str attribute-dictionary of all configuration variables associated with a GDAS task expected_block_names (optional) : str @@ -201,9 +201,6 @@ def get_jedi_dict(jedi_config_yaml: str, task_config: AttrDict, expected_block_n # Initialize dictionary of Jedi objects jedi_dict = AttrDict() - # Parse J2-YAML file for dictionary of JEDI configuration dictionaries - jedi_config_dict = parse_j2yaml(jedi_config_yaml, task_config) - # Loop through dictionary of Jedi configuration dictionaries for block_name in jedi_config_dict: # yaml_name key is set to name for this block @@ -325,33 +322,33 @@ def extract_tar_from_filehandler_dict(filehandler_dict) -> None: # Extract tarball logger.info(f"Extract files from {tar_file}") - extract_tar(tar_file) - + Jedi.extract_tar(tar_file) -@logit(logger) -def extract_tar(tar_file: str) -> None: - """Extract files from a tarball + @staticmethod + @logit(logger) + def extract_tar(tar_file: str) -> None: + """Extract files from a tarball - This method extract files from a tarball + This method extract files from a tarball - Parameters - ---------- - tar_file - path/name of tarball + Parameters + ---------- + tar_file + path/name of tarball - Returns - ---------- - None - """ + Returns + ---------- + None + """ - # extract files from tar file - tar_path = os.path.dirname(tar_file) - try: - with tarfile.open(tar_file, "r") as tarball: - tarball.extractall(path=tar_path) - logger.info(f"Extract {tarball.getnames()}") - except Exception as e: - raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except Exception as e: + raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index fc58a98ef93..b91e12a40da 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -10,18 +10,18 @@ from wxflow import (AttrDict, FileHandler, add_to_datetime, to_timedelta, parse_j2yaml, - logit) -from pygfs.task.atm_analysis import AtmAnalysis + logit, save_as_yaml) +from pygfs.task.fv3_analysis import FV3Analysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmDetAnalysis(AtmAnalysis): +class AtmAnalysis(FV3Analysis): """ Class for JEDI-based global atm deterministic analysis tasks """ - @logit(logger, name="AtmDetAnalysis") + @logit(logger, name="AtmAnalysis") def __init__(self, config: Dict[str, Any]): """Constructor global atm analysis task @@ -41,15 +41,28 @@ def __init__(self, config: Dict[str, Any]): """ super().__init__(config) + _res = int(self.task_config.CASE[1:]) + if self.task_config.DOHYBVAR: + _res_anl = int(self.task_config.CASE_ENS[1:]) + else: + _res_anl = int(self.task_config.CASE[1:]) + _localization_type = 'bump' + if self.task_config.DOHYBVAR: - _BERROR_YAML="atmosphere_background_error_hybrid_${self.task_config.STATICB_TYPE}_${_localization_type}" + _BERROR_YAML=f"atmosphere_background_error_hybrid_{self.task_config.STATICB_TYPE}_{_localization_type}" else: - _BERROR_YAML="atmosphere_background_error_static_${self.task_config.STATICB_TYPE}" + _BERROR_YAML=f"atmosphere_background_error_static_{self.task_config.STATICB_TYPE}" # Create a local dictionary that is repeatedly used across this class self.task_config.update(AttrDict( { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, + 'observations': parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'], + 'bias_files': parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'], 'BERROR_YAML': _BERROR_YAML, } )) @@ -88,17 +101,13 @@ def initialize(self) -> None: # Extract bias corrections from tar files logger.info(f"Extracting bias corrections from tar files") - if bias_dict['copy'] is None: - logger.info(f"No bias correction files to stage") - else: - bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # extract bias corrections - Jedi.extract_tar_from_filehandler_dict(bias_dict) + bias_file_list = [] + for ob in self.task_config.observations: + if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: + bias_file_list.append(self.task_config.bias_files[ob]) + Jedi.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') - # initialize JEDI variational application + # Initialize JEDI variational application logger.info(f"Initializing JEDI applications") self.jedi_dict['atmanlvar'].initialize(self.task_config, clean_empty_obsspaces=True) self.jedi_dict['atmanlfv3inc'].initialize(self.task_config) @@ -138,84 +147,38 @@ def finalize(self) -> None: None """ - # ---- tar up diags - # path of output tar statfile - atmstat = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat") + # Set paths of output tar files + diagtar = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat") + radtar = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}rad_varbc_params.tar") - # get list of diag files to put in tarball - diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) - - logger.info(f"Compressing {len(diags)} diag files to {atmstat}.gz") + # Get lists of files to put in tarballs + diaglist = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) + satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) - # gzip the files first - logger.debug(f"Gzipping {len(diags)} diag files") - for diagfile in diags: + # Compress diag files + logger.info(f"Compressing {len(diaglist)} diag files") + for diagfile in diaglist: with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) - # open tar file for writing - logger.debug(f"Creating tar file {atmstat} with {len(diags)} gzipped diag files") - with tarfile.open(atmstat, "w") as archive: - for diagfile in diags: + # Create tarball of compressed diag files in COM + logger.debug(f"Creating tarball {diagtar} with {len(diaglist)} compressed diag files") + with tarfile.open(diagtar, "w") as archive: + for diagfile in diaglist: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - # get list of yamls to copy to ROTDIR -# yamls = glob.glob(os.path.join(self.task_config.DATA, '*atm*yaml')) - - # copy full YAML from executable to ROTDIR -# for src in yamls: -# yaml_base = os.path.splitext(os.path.basename(src))[0] -# dest_yaml_name = f"{self.task_config.APREFIX}{yaml_base}.yaml" -# dest = os.path.join(self.task_config.COMOUT_CONF, dest_yaml_name) -# logger.debug(f"Copying {src} to {dest}") -# yaml_copy = { -# 'copy': [[src, dest]] -# } -# FileHandler(yaml_copy).sync() - - # path of output radiance bias correction tarfile - bfile = f"{self.task_config.APREFIX}rad_varbc_params.tar" - radtar = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, bfile) - - # rename and copy tlapse radiance bias correction files from obs to bc - tlapobs = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) - copylist = [] - for tlapfile in tlapobs: - obsfile = os.path.basename(tlapfile).split('.', 2) - newfile = f"{self.task_config.APREFIX}{obsfile[2]}" - copylist.append([tlapfile, os.path.join(self.task_config.DATA, 'bc', newfile)]) - tlapse_dict = { - 'copy': copylist - } - FileHandler(tlapse_dict).sync() - - # get lists of radiance bias correction files to add to tarball - satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) - tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*tlapse.txt')) - - # tar radiance bias correction files to ROTDIR - logger.info(f"Creating radiance bias correction tar file {radtar}") + # Create tarball of radiance bias correction files + logger.info(f"Creating radiance bias correction tarball {radtar}") with tarfile.open(radtar, 'w') as radbcor: + logger.info(f"Adding {radbcor.getnames()}") for satfile in satlist: radbcor.add(satfile, arcname=os.path.basename(satfile)) for tlapfile in tlaplist: - radbcor.add(tlapfile, arcname=os.path.basename(tlapfile)) - logger.info(f"Add {radbcor.getnames()}") - - # Copy FV3 atm increment to comrot directory -# logger.info("Copy UFS model readable atm increment file") -# inc_copy = {'copy': []} -# for itile in range(6): -# src = os.path.join(self.task_config.DATA, "anl", -# f"{self.task_config.APREFIX}cubed_sphere_grid_atminc.tile{itile+1}.nc") -# dest = self.task_config.COMOUT_ATMOS_ANALYSIS -# inc_copy['copy'].append([src, dest]) - -# # copy increments -# src_list, dest_list = zip(*inc_copy['copy']) -# logger.debug(f"Copying {src_list}\nto {dest_list}") -# FileHandler(inc_copy).sync() - - def clean(self): - super().clean() + # Change OPREFIX to APREFIX in tlapse file name when adding to tarball + radbcor.add(tlapfile, arcname=os.path.basename(tlapfile.replace(self.task_config.OPREFIX, self.task_config.APREFIX))) + + # Save files from COM + logger.info(f"Saving files to COM") + FileHandler(self.task_config.save).sync() \ No newline at end of file diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 4c858d8bebe..d4392d3dd76 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -13,13 +13,13 @@ parse_j2yaml, logit, Template, TemplateConstants) -from pygfs.task.atm_analysis import AtmAnalysis +from pygfs.task.fv3_analysis import FV3Analysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmEnsAnalysis(AtmAnalysis): +class AtmEnsAnalysis(FV3Analysis): """ Class for JEDI-based global atmens analysis tasks """ @@ -43,11 +43,16 @@ def __init__(self, config: Dict[str, Any]): """ super().__init__(config) + _res = int(self.task_config.CASE_ENS[1:]) + # Create a local dictionary that is repeatedly used across this class self.task_config.update(AttrDict( { - # Currently empty, but could be used in the future - } + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'observations': parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'], + 'bias_files': parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'], + }) ) # Create dictionary of JEDI objects @@ -81,8 +86,13 @@ def initialize(self) -> None: logger.info(f"Staging files from COM") FileHandler(self.task_config.stage).sync() - # extract bias corrections - Jedi.extract_tar_from_filehandler_dict(bias_dict) + # Extract bias corrections from tar files + logger.info(f"Extracting bias corrections from tar files") + bias_file_list = [] + for ob in self.task_config.observations: + if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: + bias_file_list.append(self.task_config.bias_files[ob]) + Jedi.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') # initialize JEDI applications logger.info(f"Initializing JEDI LETKF observer application") @@ -142,76 +152,28 @@ def finalize(self) -> None: None """ - # ---- tar up diags - # path of output tar statfile - atmensstat = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS_ENS, f"{self.task_config.APREFIX_ENS}atmensstat") - - # get list of diag files to put in tarball - diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) + # Set paths of output tar files + diagtar = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS_ENS, f"{self.task_config.APREFIX_ENS}atmensstat") - logger.info(f"Compressing {len(diags)} diag files to {atmensstat}.gz") + # Get lists of files to put in tarballs + diaglist = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) - # gzip the files first - logger.debug(f"Gzipping {len(diags)} diag files") - for diagfile in diags: + # Compress diag files + logger.info(f"Compressing {len(diaglist)} diag files") + for diagfile in diaglist: with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) - # open tar file for writing - logger.debug(f"Creating tar file {atmensstat} with {len(diags)} gzipped diag files") - with tarfile.open(atmensstat, "w") as archive: - for diagfile in diags: + # Create tarball of compressed diag files in COM + logger.debug(f"Creating tarball {diagtar} with {len(diaglist)} compressed diag files") + with tarfile.open(diagtar, "w") as archive: + for diagfile in diaglist: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - # get list of yamls to cop to ROTDIR - yamls = glob.glob(os.path.join(self.task_config.DATA, '*atmens*yaml')) - - # copy full YAML from executable to ROTDIR - for src in yamls: - logger.info(f"Copying {src} to {self.task_config.COMOUT_CONF}") - yaml_base = os.path.splitext(os.path.basename(src))[0] - dest_yaml_name = f"{self.task_config.APREFIX_ENS}{yaml_base}.yaml" - dest = os.path.join(self.task_config.COMOUT_CONF, dest_yaml_name) - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() - - # create template dictionaries - template_inc = self.task_config.COM_ATMOS_ANALYSIS_TMPL - tmpl_inc_dict = { - 'ROTDIR': self.task_config.ROTDIR, - 'RUN': self.task_config.RUN, - 'YMD': to_YMD(self.task_config.current_cycle), - 'HH': self.task_config.current_cycle.strftime('%H') - } - - # copy ensemble mean analysis to comrot - logger.info("Copy ensemble mean analysis") - fh_dict = {'copy': [[f"{self.task_config.DATA}/anl/{self.task_config.APREFIX_ENS}cubed_sphere_grid_atmanl.ensmean.nc", - f"{self.task_config.COMOUT_ATMOS_ANALYSIS_ENS}"]]} - FileHandler(fh_dict).sync() - - # copy FV3 atm increment to comrot directory - logger.info("Copy UFS model readable atm increment file") - - # loop over ensemble members - inc_copy = {'copy': []} - for imem in range(1, self.task_config.NMEM_ENS + 1): - memchar = f"mem{imem:03d}" - - # create output path for member analysis increment - tmpl_inc_dict['MEMDIR'] = memchar - incdir = Template.substitute_structure(template_inc, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_inc_dict.get) - src = os.path.join(self.task_config.DATA, 'anl', memchar, - f"{self.task_config.APREFIX_ENS}cubed_sphere_grid_atminc.nc") - dest = incdir - inc_copy['copy'].append([src, dest]) - - logger.debug(f"Copying increments") - FileHandler(inc_copy).sync() + # Save files from COM + logger.info(f"Saving files to COM") + FileHandler(self.task_config.save).sync() def clean(self): super().clean() diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index ecdb43d2821..d814fda76ac 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -8,13 +8,13 @@ add_to_datetime, to_timedelta, to_isotime, to_YMD, parse_j2yaml, logit) -from pygfs.task.atm_analysis import AtmAnalysis +from pygfs.task.fv3_analysis import FV3Analysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class EnsembleRecenter(AtmAnalysis): +class EnsembleRecenter(FV3Analysis): """ Class for JEDI-based ensemble increment recentering """ @@ -38,16 +38,18 @@ def __init__(self, config): """ super().__init__(config) - _iau_times_iso = [] - for hour in self.task_config.IAUFHRS: - _iau_times_iso.append(to_isotime(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2)) + _res = int(self.task_config.CASE[1:]) + _res_anl = int(self.task_config.CASE_ENS[1:]) # Create a local dictionary that is repeatedly used across this class self.task_config.update(AttrDict( { - 'iau_times_iso': _iau_times_iso + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, } - ) + )) # Create dictionary of Jedi objects expected_keys = ['correction_increment', 'ensemble_recenter'] diff --git a/ush/python/pygfs/task/fv3_analysis.py b/ush/python/pygfs/task/fv3_analysis.py new file mode 100644 index 00000000000..08d7e7d58c1 --- /dev/null +++ b/ush/python/pygfs/task/fv3_analysis.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 + +from logging import getLogger +from typing import Any, Dict +from wxflow import (AttrDict, Task, + add_to_datetime, to_timedelta, to_isotime, + parse_j2yaml, + logit) + +logger = getLogger(__name__.split('.')[-1]) + + +class FV3Analysis(Task): + """ + General class for JEDI-based global FV3 analysis tasks + """ + @logit(logger, name="FV3Analysis") + def __init__(self, config: Dict[str, Any]): + """Constructor global atm analysis task + + This method will construct a global atm analysis task. + This includes: + - extending the task_config attribute AttrDict to include parameters required for this task + + Parameters + ---------- + config: Dict + dictionary object containing task configuration + + Returns + ---------- + None + """ + super().__init__(config) + + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + + _iau_times_iso = [] + for hour in self.task_config.IAUFHRS: + _iau_times_iso.append(to_isotime(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2)) + + # Extend task_config with variables that are repeatedly used across this class + self.task_config.update(AttrDict( + { + 'npz_ges': self.task_config.LEVS - 1, + 'npz_anl': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + 'ATM_WINDOW_BEGIN': _window_begin, + 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'OPREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", + 'APREFIX_ENS': f"enkf{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'iau_times_iso': _iau_times_iso, + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications + } + )) + + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) + + def initialize(self) -> None: + self.initialize() + + def execute(self) -> None: + super.execute() + + def finalize(self) -> None: + super.finalize() + + def clean(self): + super().clean() From e53614e0f967a074fb19da8f378b6c9222d3fdcd Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 16:29:35 +0000 Subject: [PATCH 04/45] templatize jedi testing --- dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml | 12 ++++++------ dev/parm/config/gfs/config.atmanl.j2 | 6 ++++-- dev/parm/config/gfs/config.atmensanl.j2 | 15 +++++++-------- dev/parm/config/gfs/yaml/defaults.yaml | 16 ++++++++-------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml index f7fa21d4560..78be147029a 100644 --- a/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml +++ b/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml @@ -6,17 +6,17 @@ base: DO_JEDIATMENS: "YES" DO_TEST_MODE: "NO" atmanl: - JCB_ALGO_YAML_VAR: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_3dvar_ufs_hybatmDA.yaml.j2" - JCB_ALGO_YAML_FV3INC: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_3dvar-fv3inc_ufs_hybatmDA.yaml.j2" LAYOUT_X_ATMANL: 4 LAYOUT_Y_ATMANL: 4 + VAR_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_3dvar_ufs_hybatmDA.yaml.j2" + FV3INC_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_3dvar-fv3inc_ufs_hybatmDA.yaml.j2" atmensanl: - JCB_ALGO_YAML_LETKF: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_ufs_hybatmDA.yaml.j2" - JCB_ALGO_YAML_OBS: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_observer_ufs_hybatmDA.yaml.j2" - JCB_ALGO_YAML_SOL: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_solver_ufs_hybatmDA.yaml.j2" - JCB_ALGO_YAML_FV3INC: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf-fv3inc_ufs_hybatmDA.yaml.j2" LAYOUT_X_ATMENSANL: 4 LAYOUT_Y_ATMENSANL: 4 + LETKF_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_lgetkf_ufs_hybatmDA.yaml.j2" + OBS_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_lgetkf-observer_ufs_hybatmDA.yaml.j2" + SOL_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_lgetkf-solver_ufs_hybatmDA.yaml.j2" + FV3INC_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_lgetkf-fv3inc_ufs_hybatmDA.yaml.j2" esfc: DONST: "NO" nsst: diff --git a/dev/parm/config/gfs/config.atmanl.j2 b/dev/parm/config/gfs/config.atmanl.j2 index 7ebe69be538..35f3833e3a6 100644 --- a/dev/parm/config/gfs/config.atmanl.j2 +++ b/dev/parm/config/gfs/config.atmanl.j2 @@ -6,14 +6,16 @@ echo "BEGIN: config.atmanl" export OBS_LIST_YAML="{{ OBS_LIST_YAML }}" -export JCB_ALGO_YAML_VAR="{{ JCB_ALGO_YAML_VAR }}" -export JCB_ALGO_YAML_FV3INC="{{ JCB_ALGO_YAML_FV3INC }}" export STATICB_TYPE="{{ STATICB_TYPE }}" export layout_x_atmanl="{{ LAYOUT_X_ATMANL }}" export layout_y_atmanl="{{ LAYOUT_Y_ATMANL }}" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" +export VAR_JEDI_TEST_YAML="{{ VAR_JEDI_TEST_YAML }}" +export FV3INC_JEDI_TEST_YAML="{{ FV3INC_JEDI_TEST_YAML }}" + export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_det_config.yaml.j2" +export BIAS_FILES_YAML="${PARMgfs}/gdas/atm/atm_bias_files.yaml.j2" echo "END: config.atmanl" diff --git a/dev/parm/config/gfs/config.atmensanl.j2 b/dev/parm/config/gfs/config.atmensanl.j2 index c8f333ca8f1..77544ca444b 100644 --- a/dev/parm/config/gfs/config.atmensanl.j2 +++ b/dev/parm/config/gfs/config.atmensanl.j2 @@ -6,18 +6,17 @@ echo "BEGIN: config.atmensanl" export OBS_LIST_YAML="{{ OBS_LIST_YAML }}" -export JCB_ALGO_YAML_LETKF="{{ JCB_ALGO_YAML_LETKF }}" -export JCB_ALGO_YAML_OBS="{{ JCB_ALGO_YAML_OBS }}" -export JCB_ALGO_YAML_SOL="{{ JCB_ALGO_YAML_SOL }}" -export JCB_ALGO_YAML_FV3INC="{{ JCB_ALGO_YAML_FV3INC }}" export layout_x_atmensanl="{{ LAYOUT_X_ATMENSANL }}" export layout_y_atmensanl="{{ LAYOUT_Y_ATMENSANL }}" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ens_jedi_config.yaml.j2" -export STAGE_CRTM_COEFF_YAML="${PARMgfs}/gdas/atm/atm_stage_crtm_coeff.yaml.j2" -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/atm/atm_stage_jedi_fix.yaml.j2" -export STAGE_BKG_YAML="${PARMgfs}/gdas/atm/atm_ens_stage_bkg.yaml.j2" +export LETKF_JEDI_TEST_YAML="{{ LETKF_JEDI_TEST_YAML }}" +export OBS_JEDI_TEST_YAML="{{ OBS_JEDI_TEST_YAML }}" +export SOL_JEDI_TEST_YAML="{{ SOL_JEDI_TEST_YAML }}" +export FV3INC_JEDI_TEST_YAML="{{ FV3INC_JEDI_TEST_YAML }}" + +export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ens_config.yaml.j2" +export BIAS_FILES_YAML="${PARMgfs}/gdas/atm/atm_bias_files.yaml.j2" echo "END: config.atmensanl" diff --git a/dev/parm/config/gfs/yaml/defaults.yaml b/dev/parm/config/gfs/yaml/defaults.yaml index b26ddde38d7..7eb39def757 100644 --- a/dev/parm/config/gfs/yaml/defaults.yaml +++ b/dev/parm/config/gfs/yaml/defaults.yaml @@ -29,25 +29,25 @@ base: NMEM_ENS_GFS_OFFSET: 20 atmanl: - OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" - JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" - JCB_ALGO_YAML_FV3INC: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar-fv3inc.yaml.j2" STATICB_TYPE: "gsibec" LAYOUT_X_ATMANL: 8 LAYOUT_Y_ATMANL: 8 IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 + OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" + VAR_JEDI_TEST_YAML: "" + FV3INC_JEDI_TEST_YAML: "" atmensanl: - OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" - JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" - JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" - JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" - JCB_ALGO_YAML_FV3INC: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2" LAYOUT_X_ATMENSANL: 8 LAYOUT_Y_ATMENSANL: 8 IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 + OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" + LETKF_JEDI_TEST_YAML: "" + OBS_JEDI_TEST_YAML: "" + SOL_JEDI_TEST_YAML: "" + FV3INC_JEDI_TEST_YAML: "" aeroanl: IO_LAYOUT_X: 1 From d37b4093298ff7c4178d66410e77b1251fe680cf Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 16:30:19 +0000 Subject: [PATCH 05/45] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index e572ab4cb4b..f3de4211bc6 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit e572ab4cb4bc2ccee2517426f3d20b919865de65 +Subproject commit f3de4211bc651ab699c4d75a8d7f5b25e21322e1 From f9f29a706b2a3e79849b8ac9d9a9c5fbcbf159ba Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 17:03:35 +0000 Subject: [PATCH 06/45] Remove ATM_ from window vars --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/fv3_analysis.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index f3de4211bc6..a3053d10103 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit f3de4211bc651ab699c4d75a8d7f5b25e21322e1 +Subproject commit a3053d10103a8a006e32cedbdc0fc18e9c67c5da diff --git a/ush/python/pygfs/task/fv3_analysis.py b/ush/python/pygfs/task/fv3_analysis.py index 08d7e7d58c1..f2e8bc414c3 100644 --- a/ush/python/pygfs/task/fv3_analysis.py +++ b/ush/python/pygfs/task/fv3_analysis.py @@ -45,8 +45,8 @@ def __init__(self, config: Dict[str, Any]): 'npz_ges': self.task_config.LEVS - 1, 'npz_anl': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, - 'ATM_WINDOW_BEGIN': _window_begin, - 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'WINDOW_BEGIN': _window_begin, + 'WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", 'OPREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX_ENS': f"enkf{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", From c010b2e8d5679c9e4ce1cffb4c898f0f8fdbc41f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 17:56:04 +0000 Subject: [PATCH 07/45] Separate obs list for different tests --- dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml | 2 ++ dev/parm/config/gcafs/yaml/defaults.yaml | 14 ++++++++------ sorc/gdas.cd | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml b/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml index 78be147029a..045a1bc8a4a 100644 --- a/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml +++ b/dev/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml @@ -8,11 +8,13 @@ base: atmanl: LAYOUT_X_ATMANL: 4 LAYOUT_Y_ATMANL: 4 + OBS_LIST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/atm_obs_list_ufs_hybatmDA.yaml.j2" VAR_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_3dvar_ufs_hybatmDA.yaml.j2" FV3INC_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_3dvar-fv3inc_ufs_hybatmDA.yaml.j2" atmensanl: LAYOUT_X_ATMENSANL: 4 LAYOUT_Y_ATMENSANL: 4 + OBS_LIST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/atm_obs_list_ufs_hybatmDA.yaml.j2" LETKF_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_lgetkf_ufs_hybatmDA.yaml.j2" OBS_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_lgetkf-observer_ufs_hybatmDA.yaml.j2" SOL_JEDI_TEST_YAML: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jedi-test_lgetkf-solver_ufs_hybatmDA.yaml.j2" diff --git a/dev/parm/config/gcafs/yaml/defaults.yaml b/dev/parm/config/gcafs/yaml/defaults.yaml index 2c8379195fe..814eaafc9c8 100644 --- a/dev/parm/config/gcafs/yaml/defaults.yaml +++ b/dev/parm/config/gcafs/yaml/defaults.yaml @@ -37,23 +37,25 @@ base: FHOUT_AERO: 3 atmanl: - JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar.yaml.j2" - JCB_ALGO_YAML_FV3INC: "${PARMgfs}/gdas/atm/jcb-prototype_3dvar-fv3inc.yaml.j2" STATICB_TYPE: "gsibec" LAYOUT_X_ATMANL: 8 LAYOUT_Y_ATMANL: 8 IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 + OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" + VAR_JEDI_TEST_YAML: "" + FV3INC_JEDI_TEST_YAML: "" atmensanl: - JCB_ALGO_YAML_LETKF: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf.yaml.j2" - JCB_ALGO_YAML_OBS: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_observer.yaml.j2" - JCB_ALGO_YAML_SOL: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2" - JCB_ALGO_YAML_FV3INC: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf-fv3inc.yaml.j2" LAYOUT_X_ATMENSANL: 8 LAYOUT_Y_ATMENSANL: 8 IO_LAYOUT_X: 1 IO_LAYOUT_Y: 1 + OBS_LIST_YAML: "${PARMgfs}/gdas/atm/atm_obs_list.yaml.j2" + LETKF_JEDI_TEST_YAML: "" + OBS_JEDI_TEST_YAML: "" + SOL_JEDI_TEST_YAML: "" + FV3INC_JEDI_TEST_YAML: "" aeroanl: IO_LAYOUT_X: 1 diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a3053d10103..968d8f9afa4 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a3053d10103a8a006e32cedbdc0fc18e9c67c5da +Subproject commit 968d8f9afa4e2c7862dc7e81fc4beae84254cbfe From 31b2bbc8185fdf87d217849f1948588bc15744ac Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 18:24:54 +0000 Subject: [PATCH 08/45] pynorms --- ush/python/pygfs/task/atm_analysis.py | 11 ++++++----- ush/python/pygfs/task/ensemble_recenter.py | 4 ++-- ush/python/pygfs/task/fv3_analysis.py | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index b91e12a40da..601e30cc40f 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -48,11 +48,11 @@ def __init__(self, config: Dict[str, Any]): _res_anl = int(self.task_config.CASE[1:]) _localization_type = 'bump' - + if self.task_config.DOHYBVAR: - _BERROR_YAML=f"atmosphere_background_error_hybrid_{self.task_config.STATICB_TYPE}_{_localization_type}" + _BERROR_YAML = f"atmosphere_background_error_hybrid_{self.task_config.STATICB_TYPE}_{_localization_type}" else: - _BERROR_YAML=f"atmosphere_background_error_static_{self.task_config.STATICB_TYPE}" + _BERROR_YAML = f"atmosphere_background_error_static_{self.task_config.STATICB_TYPE}" # Create a local dictionary that is repeatedly used across this class self.task_config.update(AttrDict( @@ -105,7 +105,7 @@ def initialize(self) -> None: for ob in self.task_config.observations: if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: bias_file_list.append(self.task_config.bias_files[ob]) - Jedi.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') + Jedi.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') # Initialize JEDI variational application logger.info(f"Initializing JEDI applications") @@ -181,4 +181,5 @@ def finalize(self) -> None: # Save files from COM logger.info(f"Saving files to COM") - FileHandler(self.task_config.save).sync() \ No newline at end of file + FileHandler(self.task_config.save).sync() + \ No newline at end of file diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index d814fda76ac..4bbb1672713 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -77,7 +77,7 @@ def initialize(self) -> None: # Stage files from COM logger.info(f"Staging files from COM") FileHandler(self.task_config.stage).sync() - + # Initialize JEDI ensemble increment recentering application logger.info(f"Initializing JEDI applications") self.jedi_dict['correction_increment'].initialize(self.task_config) @@ -121,6 +121,6 @@ def finalize(self) -> None: None """ - # Save output files to COM + # Save output files to COM logger.info(f"Saving output files to COM") FileHandler(self.task_config.stage).sync() diff --git a/ush/python/pygfs/task/fv3_analysis.py b/ush/python/pygfs/task/fv3_analysis.py index f2e8bc414c3..dabe41ee6a4 100644 --- a/ush/python/pygfs/task/fv3_analysis.py +++ b/ush/python/pygfs/task/fv3_analysis.py @@ -2,9 +2,9 @@ from logging import getLogger from typing import Any, Dict -from wxflow import (AttrDict, Task, +from wxflow import (AttrDict, Task, add_to_datetime, to_timedelta, to_isotime, - parse_j2yaml, + parse_j2yaml, logit) logger = getLogger(__name__.split('.')[-1]) From 2451bc770f8acda60622e0ef1663e880f9e1a0a3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 18:26:40 +0000 Subject: [PATCH 09/45] pynorms --- ush/python/pygfs/task/atm_analysis.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 601e30cc40f..c064a0c90d7 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -182,4 +182,3 @@ def finalize(self) -> None: # Save files from COM logger.info(f"Saving files to COM") FileHandler(self.task_config.save).sync() - \ No newline at end of file From 0e78550e4968ff2d52ff3b633f3f57789a31f1df Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 16 Sep 2025 18:42:57 +0000 Subject: [PATCH 10/45] bugfix --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/aero_analysis.py | 3 ++- ush/python/pygfs/task/aero_bmatrix.py | 3 ++- ush/python/pygfs/task/analysis_stats.py | 3 ++- ush/python/pygfs/task/fv3_analysis_calc.py | 3 ++- ush/python/pygfs/task/marine_analysis.py | 3 ++- ush/python/pygfs/task/marine_recenter.py | 3 ++- ush/python/pygfs/task/snow_analysis.py | 3 ++- ush/python/pygfs/task/snowens_analysis.py | 3 ++- 9 files changed, 17 insertions(+), 9 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 968d8f9afa4..6a53cc7c61f 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 968d8f9afa4e2c7862dc7e81fc4beae84254cbfe +Subproject commit 6a53cc7c61fe30ab1f0625a8a46bc0a44648cdc7 diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 28310ea1643..6946257d5a0 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -77,7 +77,8 @@ def __init__(self, config): # Create dictionary of Jedi objects expected_keys = ['aeroanlvar'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index cc9918d4d77..27ccb61d438 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -69,7 +69,8 @@ def __init__(self, config): # Create dictionary of Jedi objects expected_keys = ['aero_interpbkg', 'aero_diagb', 'aero_diffusion'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self: Task) -> None: diff --git a/ush/python/pygfs/task/analysis_stats.py b/ush/python/pygfs/task/analysis_stats.py index cfe34d3257f..086f289d3a1 100644 --- a/ush/python/pygfs/task/analysis_stats.py +++ b/ush/python/pygfs/task/analysis_stats.py @@ -74,7 +74,8 @@ def initialize(self) -> None: # Expected keys are what must be included from the JEDI config file. We can # then loop through ob space list from scripts/exglobal_analysis_stats.py expected_keys = ['aero', 'atmos', 'snow'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) logger.info(f"Copying files to {self.task_config.DATA}/stats") diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index a287c82ad4e..ba65dce0c60 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -70,7 +70,8 @@ def __init__(self, config): expected_keys.append('aero_addincrement') if self.task_config.DO_JEDISNOWDA: expected_keys.append('snow_addincrement') - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index fd7fa8d2427..b374c5f0ed5 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -81,7 +81,8 @@ def __init__(self, config): # Construct dictionary of JEDI objects, one for each JEDI application need for the analysis expected_keys = ['var', 'soca_incpostproc', 'soca_diag_stats'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML_DET, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML_DET, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self: Task) -> None: diff --git a/ush/python/pygfs/task/marine_recenter.py b/ush/python/pygfs/task/marine_recenter.py index 03fb5e2bcbc..87bc8e8c785 100644 --- a/ush/python/pygfs/task/marine_recenter.py +++ b/ush/python/pygfs/task/marine_recenter.py @@ -64,7 +64,8 @@ def __init__(self, config: Dict) -> None: # Construct dictionary of JEDI objects, one for each JEDI application need for the analysis expected_keys = ['gridgen', 'ens_handler'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self): diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 9f7b2a83ed5..8051e28f1de 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -81,7 +81,8 @@ def __init__(self, config: Dict[str, Any]): # Create JEDI object dictionary expected_keys = ['scf_to_ioda', 'snowanlvar'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 3d903bd118a..f513c5da7d1 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -82,7 +82,8 @@ def __init__(self, config: Dict[str, Any]): # Create JEDI object dictionary expected_keys = ['scf_to_ioda', 'snowanlvar', 'esnowanlensmean'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: From 8d8e4b1c18d647f27bfd06f31c4b226ecce96db8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 17 Sep 2025 14:56:25 +0000 Subject: [PATCH 11/45] Debugging and refactor analysis calc job --- jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI | 4 ++ sorc/gdas.cd | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 - ush/python/pygfs/task/atmens_analysis.py | 2 - ush/python/pygfs/task/fv3_analysis.py | 12 +++++ ush/python/pygfs/task/fv3_analysis_calc.py | 59 ++++++---------------- ush/python/pygfs/task/marine_bmat.py | 3 +- 7 files changed, 34 insertions(+), 50 deletions(-) diff --git a/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI b/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI index 5d900f4ac00..26944a8bbd1 100755 --- a/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI +++ b/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI @@ -27,6 +27,10 @@ YMD=${PDY} HH=${cyc} RUN=${RUN} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_CONF:COM_CONF_TMPL + +mkdir -m 775 -p "${COMOUT_CONF}" ############################################## # Run relevant script diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6a53cc7c61f..1430aa7ac7e 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6a53cc7c61fe30ab1f0625a8a46bc0a44648cdc7 +Subproject commit 1430aa7ac7e4f44001306cedff460d4fbd15db2a diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c064a0c90d7..0910f4f65a5 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -61,8 +61,6 @@ def __init__(self, config: Dict[str, Any]): 'npy_ges': _res + 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, - 'observations': parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'], - 'bias_files': parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'], 'BERROR_YAML': _BERROR_YAML, } )) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index d4392d3dd76..388f6c33e74 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -50,8 +50,6 @@ def __init__(self, config: Dict[str, Any]): { 'npx_ges': _res + 1, 'npy_ges': _res + 1, - 'observations': parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'], - 'bias_files': parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'], }) ) diff --git a/ush/python/pygfs/task/fv3_analysis.py b/ush/python/pygfs/task/fv3_analysis.py index dabe41ee6a4..0ebb2610a88 100644 --- a/ush/python/pygfs/task/fv3_analysis.py +++ b/ush/python/pygfs/task/fv3_analysis.py @@ -39,6 +39,16 @@ def __init__(self, config: Dict[str, Any]): for hour in self.task_config.IAUFHRS: _iau_times_iso.append(to_isotime(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2)) + if 'OBS_LIST_YAML' in self.task_config: + _observations = parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'] + else: + _observations = [] + + if 'BIAS_FILES_YAML' in self.task_config: + _bias_files = parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'] + else: + _bias_files = {} + # Extend task_config with variables that are repeatedly used across this class self.task_config.update(AttrDict( { @@ -53,6 +63,8 @@ def __init__(self, config: Dict[str, Any]): 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", 'iau_times_iso': _iau_times_iso, + 'observations': _observations, + 'bias_files': _bias_files, 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications } )) diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index ba65dce0c60..9eebea6a201 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -14,7 +14,7 @@ logger = getLogger(__name__.split('.')[-1]) -class FV3AnalysisCalc(Task): +class FV3AnalysisCalc(FV3Analysis): """ Class for analysis calculation """ @@ -39,7 +39,11 @@ def __init__(self, config): super().__init__(config) _res = int(self.task_config.CASE[1:]) - _res_anl = int(self.task_config.CASE_ANL[1:]) + if self.task_config.DOHYBVAR: + _res_anl = int(self.task_config.CASE_ENS[1:]) + else: + _res_anl = int(self.task_config.CASE[1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) # Create a local dictionary that is repeatedly used across this class @@ -47,17 +51,8 @@ def __init__(self, config): { 'npx_ges': _res + 1, 'npy_ges': _res + 1, - 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, - 'npz_anl': self.task_config.LEVS - 1, - 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", - 'ATM_WINDOW_BEGIN': _window_begin, - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX_ENS': f"enkf{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", } ) @@ -70,8 +65,7 @@ def __init__(self, config): expected_keys.append('aero_addincrement') if self.task_config.DO_JEDISNOWDA: expected_keys.append('snow_addincrement') - jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -92,6 +86,10 @@ def initialize(self) -> None: None """ + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.stage).sync() + # Initialize GDASApp JEDI addincrement application logger.info(f"Initializing GDASApp JEDI addincrement applications") self.jedi_dict['atm_addincrement'].initialize(self.task_config) @@ -100,17 +98,6 @@ def initialize(self) -> None: if self.task_config.DO_JEDISNOWDA: self.jedi_dict['snow_addincrement'].initialize(self.task_config) - # Stage fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_dict = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_dict).sync() - logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") - - # Stage background and increment files - logger.info(f"Staging background and increment files from COM") - fh_dict = parse_j2yaml(self.task_config.STAGE_YAML, self.task_config) - FileHandler(fh_dict).sync() - @logit(logger) def execute(self) -> None: """Compute analyses @@ -177,26 +164,6 @@ def finalize(self) -> None: None """ - # Copy analyses to COM - fh_dict = {'copy': []} - src_prefix = f"{self.task_config.DATA}/{self.task_config.GPREFIX}" - dest_prefix = f"{self.task_config.COMOUT_ATMOS_ANALYSIS}/{self.task_config.APREFIX}" - fh_dict['copy'].append([f"{src_prefix}atmf006.nc", - f"{dest_prefix}atmanl.nc"]) - fh_dict['copy'].append([f"{src_prefix}sfcf006.nc", - f"{dest_prefix}sfcanl.nc"]) - - # Copy YAMLs to COM - for app_name in self.jedi_dict.keys(): - src = os.path.join(self.task_config.DATA, - f"{app_name}.yaml") - dest = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, - f"{self.task_config.APREFIX}{app_name}.yaml") - fh_dict['copy'].append([src, dest]) - - # Call FileHandler - FileHandler(fh_dict).sync() - # Write analysis log file formatted_date = datetime.now().strftime("%a %b %d %H:%M:%S %Z%Y") log_file = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, f"{self.task_config.RUN}.t{self.task_config.cyc}z.loganl.txt") @@ -204,6 +171,10 @@ def finalize(self) -> None: with open(log_file, "w") as file: file.write(f"{message}\n") + # Save files from COM + logger.info(f"Saving files to COM") + FileHandler(self.task_config.save).sync() + @logit(logger) def insert_analysis_variables(valid_time, fn_anl: str, fn_bkg: str) -> None: diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index c9707a47469..5d4908a2d54 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -75,7 +75,8 @@ def __init__(self, config): # Create dictionary of Jedi objects expected_keys = ['gridgen', 'soca_diagb', 'soca_parameters_diffusion_vt', 'soca_setcorscales', 'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights', 'soca_chgres'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) + self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) @logit(logger) def initialize(self: Task) -> None: From 51a0a9faeecb9dea3f4525bcab0329f1b79ae7f4 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 17 Sep 2025 18:17:38 +0000 Subject: [PATCH 12/45] Debug --- dev/parm/config/gfs/config.analcalc_fv3jedi | 4 +--- sorc/gdas.cd | 2 +- ush/python/pygfs/task/ensemble_recenter.py | 2 +- ush/python/pygfs/task/fv3_analysis_calc.py | 3 ++- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dev/parm/config/gfs/config.analcalc_fv3jedi b/dev/parm/config/gfs/config.analcalc_fv3jedi index b0ede30dca5..233a5f694ac 100644 --- a/dev/parm/config/gfs/config.analcalc_fv3jedi +++ b/dev/parm/config/gfs/config.analcalc_fv3jedi @@ -14,9 +14,7 @@ export layout_y_analcalc_fv3jedi=2 # Get task specific resources source "${EXPDIR}/config.resources" analcalc_fv3jedi -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/atm/atm_stage_jedi_fix.yaml.j2" -export STAGE_YAML="${PARMgfs}/gdas/analcalc/analcalc_stage.yaml.j2" -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/analcalc/analcalc_jedi_config.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/analcalc/analcalc_config.yaml.j2" if [[ ${DOHYBVAR} = "YES" ]]; then export CASE_ANL=${CASE_ENS} diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 1430aa7ac7e..3b9d328507a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 1430aa7ac7e4f44001306cedff460d4fbd15db2a +Subproject commit 3b9d328507af50ec139261e7a30cdcf8d8fa3845 diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index 4bbb1672713..bf6efcdb983 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -123,4 +123,4 @@ def finalize(self) -> None: # Save output files to COM logger.info(f"Saving output files to COM") - FileHandler(self.task_config.stage).sync() + FileHandler(self.task_config.save).sync() diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index 9eebea6a201..b51a2c10b9e 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -5,11 +5,12 @@ import netCDF4 as nc import os from pprint import pformat -from pygfs.jedi import Jedi from wxflow import (AttrDict, FileHandler, Task, parse_j2yaml, to_timedelta, add_to_datetime, to_fv3time, to_isotime, logit) +from pygfs.jedi import Jedi +from pygfs.task.fv3_analysis import FV3Analysis logger = getLogger(__name__.split('.')[-1]) From 8fe4c2f288e2ea06c72d79d11c37ae5d3b0a2c24 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Sep 2025 16:26:23 +0000 Subject: [PATCH 13/45] Create methods to tar and compress diag files and bias corrections --- ush/python/pygfs/jedi/jedi.py | 85 ----------------- ush/python/pygfs/task/atm_analysis.py | 43 ++------- ush/python/pygfs/task/atmens_analysis.py | 24 +---- ush/python/pygfs/task/ensemble_recenter.py | 2 + ush/python/pygfs/task/fv3_analysis.py | 101 ++++++++++++++++++++- ush/python/pygfs/task/fv3_analysis_calc.py | 2 + 6 files changed, 115 insertions(+), 142 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 93cece28794..cf1a5319547 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -import os -import tarfile from logging import getLogger from typing import List, Dict, Any, Optional from pprint import pformat @@ -267,89 +265,6 @@ def clean_empty_obsspaces(self): if observers == []: raise WorkflowException(f"No observers found in JEDI input config") - @staticmethod - @logit(logger) - def remove_redundant(input_list: List) -> List: - """Remove reduncancies from list with possible redundant, non-mutable elements - - Parameters - ---------- - input_list : List - List with possible redundant, non-mutable elements - - Returns - ---------- - output_list : List - Input list but with redundancies removed - """ - - output_list = [] - for item in input_list: - if item not in output_list: - output_list.append(item) - - return output_list - - @staticmethod - @logit(logger) - def extract_tar_from_filehandler_dict(filehandler_dict) -> None: - """Extract tarballs from FileHandler input dictionary - - This method extracts files from tarballs specified in a FileHander - input dictionary for the 'copy' action. - - Parameters - ---------- - filehandler_dict - Input dictionary for FileHandler - - Returns - ---------- - None - """ - - for item in filehandler_dict['copy']: - # Use the filename from the destination entry if it's a file path - # Otherwise, it's a directory, so use the source entry filename - if os.path.isfile(item[1]): - filename = os.path.basename(item[1]) - else: - filename = os.path.basename(item[0]) - - # Check if file is a tar ball - if os.path.splitext(filename)[1] == '.tar': - tar_file = f"{os.path.dirname(item[1])}/{filename}" - - # Extract tarball - logger.info(f"Extract files from {tar_file}") - Jedi.extract_tar(tar_file) - - @staticmethod - @logit(logger) - def extract_tar(tar_file: str) -> None: - """Extract files from a tarball - - This method extract files from a tarball - - Parameters - ---------- - tar_file - path/name of tarball - - Returns - ---------- - None - """ - - # extract files from tar file - tar_path = os.path.dirname(tar_file) - try: - with tarfile.open(tar_file, "r") as tarball: - tarball.extractall(path=tar_path) - logger.info(f"Extract {tarball.getnames()}") - except Exception as e: - raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e - @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 0910f4f65a5..dbdec9d5875 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -48,7 +48,6 @@ def __init__(self, config: Dict[str, Any]): _res_anl = int(self.task_config.CASE[1:]) _localization_type = 'bump' - if self.task_config.DOHYBVAR: _BERROR_YAML = f"atmosphere_background_error_hybrid_{self.task_config.STATICB_TYPE}_{_localization_type}" else: @@ -59,8 +58,10 @@ def __init__(self, config: Dict[str, Any]): { 'npx_ges': _res + 1, 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config.LEVS - 1, 'BERROR_YAML': _BERROR_YAML, } )) @@ -103,7 +104,7 @@ def initialize(self) -> None: for ob in self.task_config.observations: if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: bias_file_list.append(self.task_config.bias_files[ob]) - Jedi.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') + FV3Analysis.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') # Initialize JEDI variational application logger.info(f"Initializing JEDI applications") @@ -145,37 +146,13 @@ def finalize(self) -> None: None """ - # Set paths of output tar files - diagtar = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}atmstat") - radtar = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS, f"{self.task_config.APREFIX}rad_varbc_params.tar") - - # Get lists of files to put in tarballs - diaglist = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) - satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) - tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) - - # Compress diag files - logger.info(f"Compressing {len(diaglist)} diag files") - for diagfile in diaglist: - with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: - f_out.writelines(f_in) - - # Create tarball of compressed diag files in COM - logger.debug(f"Creating tarball {diagtar} with {len(diaglist)} compressed diag files") - with tarfile.open(diagtar, "w") as archive: - for diagfile in diaglist: - diaggzip = f"{diagfile}.gz" - archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # Create tarball of radiance bias correction files - logger.info(f"Creating radiance bias correction tarball {radtar}") - with tarfile.open(radtar, 'w') as radbcor: - logger.info(f"Adding {radbcor.getnames()}") - for satfile in satlist: - radbcor.add(satfile, arcname=os.path.basename(satfile)) - for tlapfile in tlaplist: - # Change OPREFIX to APREFIX in tlapse file name when adding to tarball - radbcor.add(tlapfile, arcname=os.path.basename(tlapfile.replace(self.task_config.OPREFIX, self.task_config.APREFIX))) + # Compress and tar diag files in COM directory + self.tar_diag_files(self.task_config.COMOUT_ATMOS_ANALYSIS, + f"{self.task_config.APREFIX}atmstat") + + # Tar radiative bias correction files into COM directory + self.tar_radiative_bias_corrections(self.task_config.COMOUT_ATMOS_ANALYSIS, + f"{self.task_config.APREFIX}rad_varbc_params.tar") # Save files from COM logger.info(f"Saving files to COM") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 388f6c33e74..3162bf2ab09 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -50,6 +50,7 @@ def __init__(self, config: Dict[str, Any]): { 'npx_ges': _res + 1, 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, }) ) @@ -90,7 +91,7 @@ def initialize(self) -> None: for ob in self.task_config.observations: if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: bias_file_list.append(self.task_config.bias_files[ob]) - Jedi.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') + FV3Analysis.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') # initialize JEDI applications logger.info(f"Initializing JEDI LETKF observer application") @@ -150,24 +151,9 @@ def finalize(self) -> None: None """ - # Set paths of output tar files - diagtar = os.path.join(self.task_config.COMOUT_ATMOS_ANALYSIS_ENS, f"{self.task_config.APREFIX_ENS}atmensstat") - - # Get lists of files to put in tarballs - diaglist = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) - - # Compress diag files - logger.info(f"Compressing {len(diaglist)} diag files") - for diagfile in diaglist: - with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: - f_out.writelines(f_in) - - # Create tarball of compressed diag files in COM - logger.debug(f"Creating tarball {diagtar} with {len(diaglist)} compressed diag files") - with tarfile.open(diagtar, "w") as archive: - for diagfile in diaglist: - diaggzip = f"{diagfile}.gz" - archive.add(diaggzip, arcname=os.path.basename(diaggzip)) + # Compress and tar diag files in COM directory + self.tar_diag_files(self.task_config.COMOUT_ATMOS_ANALYSIS_ENS, + f"{self.task_config.APREFIX_ENS}atmstat") # Save files from COM logger.info(f"Saving files to COM") diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index bf6efcdb983..99ba51c1574 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -46,8 +46,10 @@ def __init__(self, config): { 'npx_ges': _res + 1, 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config.LEVS - 1, } )) diff --git a/ush/python/pygfs/task/fv3_analysis.py b/ush/python/pygfs/task/fv3_analysis.py index 0ebb2610a88..94ba43a8682 100644 --- a/ush/python/pygfs/task/fv3_analysis.py +++ b/ush/python/pygfs/task/fv3_analysis.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 from logging import getLogger +import os +import tarfile from typing import Any, Dict from wxflow import (AttrDict, Task, add_to_datetime, to_timedelta, to_isotime, @@ -43,7 +45,6 @@ def __init__(self, config: Dict[str, Any]): _observations = parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'] else: _observations = [] - if 'BIAS_FILES_YAML' in self.task_config: _bias_files = parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'] else: @@ -52,11 +53,9 @@ def __init__(self, config: Dict[str, Any]): # Extend task_config with variables that are repeatedly used across this class self.task_config.update(AttrDict( { - 'npz_ges': self.task_config.LEVS - 1, - 'npz_anl': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, 'WINDOW_BEGIN': _window_begin, 'WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications 'OPREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX_ENS': f"enkf{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", @@ -65,7 +64,6 @@ def __init__(self, config: Dict[str, Any]): 'iau_times_iso': _iau_times_iso, 'observations': _observations, 'bias_files': _bias_files, - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications } )) @@ -83,3 +81,96 @@ def finalize(self) -> None: def clean(self): super().clean() + + @logit(logger) + def tar_diag_files(self, comout, tarball_name): + """Compress and tar diag files into COM directory + + Parameters + ---------- + comout: str + path to COM output directory + tarball_name: str + name of output tar file + + Returns + ---------- + None + """ + + # Set paths of output tar files + diagtar = os.path.join(comout, tarball_name) + + # Get lists of files to put in tarballs + diaglist = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) + + # Compress diag files + logger.info(f"Compressing {len(diaglist)} diag files") + for diagfile in diaglist: + with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: + f_out.writelines(f_in) + + # Create tarball of compressed diag files in COM + logger.debug(f"Creating tarball {diagtar} with {len(diaglist)} compressed diag files") + with tarfile.open(diagtar, "w") as archive: + for diagfile in diaglist: + diaggzip = f"{diagfile}.gz" + archive.add(diaggzip, arcname=os.path.basename(diaggzip)) + + def tar_radiative_bias_corrections(self, comout, tarball_name): + """Tar radiative bias correction files and into COM directory + + Parameters + ---------- + comout: str + path to COM output directory + tarball_name: str + name of output tar file + + Returns + ---------- + None + """ + + # Set paths of output tar files + radtar = os.path.join(comout, tarball_name") + + # Get lists of files to put in tarballs + satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) + tlaplist = glob.glob(os.path.join(self.task_config.DATA, 'obs', '*tlapse.txt')) + + # Create tarball of radiance bias correction files + logger.info(f"Creating radiance bias correction tarball {radtar}") + with tarfile.open(radtar, 'w') as radbcor: + logger.info(f"Adding {radbcor.getnames()}") + for satfile in satlist: + radbcor.add(satfile, arcname=os.path.basename(satfile)) + for tlapfile in tlaplist: + # Change OPREFIX to APREFIX in tlapse file name when adding to tarball + radbcor.add(tlapfile, arcname=os.path.basename(tlapfile.replace(self.task_config.OPREFIX, self.task_config.APREFIX))) + + @staticmethod + @logit(logger) + def extract_tar(tar_file: str) -> None: + """Extract files from a tarball + + This method extract files from a tarball + + Parameters + ---------- + tar_file + path/name of tarball + + Returns + ---------- + None + """ + + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except Exception as e: + raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index b51a2c10b9e..e79969ad9e4 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -52,8 +52,10 @@ def __init__(self, config): { 'npx_ges': _res + 1, 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config.LEVS - 1, } ) From c65221e1787e1b41f2465679dc6ab66f748c8aa3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Sep 2025 16:26:56 +0000 Subject: [PATCH 14/45] Delete old analysis class --- ush/python/pygfs/task/analysis.py | 274 ------------------------------ 1 file changed, 274 deletions(-) delete mode 100644 ush/python/pygfs/task/analysis.py diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py deleted file mode 100644 index 1d8b38483b0..00000000000 --- a/ush/python/pygfs/task/analysis.py +++ /dev/null @@ -1,274 +0,0 @@ -#!/usr/bin/env python3 - -import os -import glob -import tarfile -from logging import getLogger -from pprint import pformat -from netCDF4 import Dataset -from typing import List, Dict, Any, Union, Optional - -from jcb import render -from wxflow import (parse_j2yaml, FileHandler, rm_p, logit, - Task, Executable, WorkflowException, to_fv3time, to_YMD, - Template, TemplateConstants) - -logger = getLogger(__name__.split('.')[-1]) - - -class Analysis(Task): - """Parent class for GDAS tasks - - The Analysis class is the parent class for all - Global Data Assimilation System (GDAS) tasks - directly related to peforming an analysis - """ - - def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) - # Store location of GDASApp jinja2 templates - self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') - # fix ocnres - self.task_config.OCNRES = f"{self.task_config.OCNRES :03d}" - - def initialize(self) -> None: - super().initialize() - - # all JEDI analyses need a JEDI config - self.task_config.jedi_config = self.get_jedi_config() - - # all analyses need to stage observations - obs_dict = self.get_obs_dict() - FileHandler(obs_dict).sync() - - # link jedi executable to run directory - self.link_jediexe() - - @logit(logger) - def get_jedi_config(self, algorithm: Optional[str] = None) -> Dict[str, Any]: - """Compile a dictionary of JEDI configuration from JEDIYAML template file - - Parameters - ---------- - algorithm (optional) : str - Name of the algorithm to use in the JEDI configuration. Will override the algorithm - set in the self.task_config.JCB_<>_YAML file - - Returns - ---------- - jedi_config : Dict - a dictionary containing the fully rendered JEDI yaml configuration - """ - - # generate JEDI YAML file - logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}") - - if 'JCB_BASE_YAML' in self.task_config.keys(): - # Step 1: fill templates of the jcb base YAML file - jcb_config = parse_j2yaml(self.task_config.JCB_BASE_YAML, self.task_config) - - # Step 2: (optional) fill templates of algorithm override YAML and merge - if 'JCB_ALGO_YAML' in self.task_config.keys(): - jcb_algo_config = parse_j2yaml(self.task_config.JCB_ALGO_YAML, self.task_config) - jcb_config = {**jcb_config, **jcb_algo_config} - - # If algorithm is present override the algorithm in the JEDI config - if algorithm: - jcb_config['algorithm'] = algorithm - - # Step 3: generate the JEDI Yaml using JCB driving YAML - jedi_config = render(jcb_config) - elif 'JEDIYAML' in self.task_config.keys(): - # Generate JEDI YAML file (without using JCB) - logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}") - jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, - searchpath=self.gdasapp_j2tmpl_dir) - logger.debug(f"JEDI config:\n{pformat(jedi_config)}") - else: - raise KeyError(f"Task config must contain JCB_BASE_YAML or JEDIYAML") - - logger.debug(f"JEDI config:\n{pformat(jedi_config)}") - - return jedi_config - - @logit(logger) - def get_obs_dict(self) -> Dict[str, Any]: - """Compile a dictionary of observation files to copy - - This method extracts 'observers' from the JEDI yaml and from that list, extracts a list of - observation files that are to be copied to the run directory - from the observation input directory - - Parameters - ---------- - - Returns - ---------- - obs_dict: Dict - a dictionary containing the list of observation files to copy for FileHandler - """ - - logger.info(f"Extracting a list of observation files from Jedi config file") - observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') - logger.debug(f"observations:\n{pformat(observations)}") - - copylist = [] - for ob in observations['observers']: - obfile = ob['obs space']['obsdatain']['engine']['obsfile'] - basename = os.path.basename(obfile) - copylist.append([os.path.join(self.task_config['COM_OBS'], basename), obfile]) - obs_dict = { - 'mkdir': [os.path.join(self.task_config['DATA'], 'obs')], - 'copy': copylist - } - return obs_dict - - @logit(logger) - def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: List) -> None: - """Add cubed-sphere increments to cubed-sphere backgrounds - - Parameters - ---------- - inc_file_tmpl : str - template of the FV3 increment file of the form: 'filetype.tile{tilenum}.nc' - bkg_file_tmpl : str - template of the FV3 background file of the form: 'filetype.tile{tilenum}.nc' - incvars : List - List of increment variables to add to the background - """ - - for itile in range(1, self.task_config.ntiles + 1): - inc_path = inc_file_tmpl.format(tilenum=itile) - bkg_path = bkg_file_tmpl.format(tilenum=itile) - with Dataset(inc_path, mode='r') as incfile, Dataset(bkg_path, mode='a') as rstfile: - for vname in incvars: - increment = incfile.variables[vname][:] - bkg = rstfile.variables[vname][:] - anl = bkg + increment - rstfile.variables[vname][:] = anl[:] - try: - rstfile.variables[vname].delncattr('checksum') # remove the checksum so fv3 does not complain - except (AttributeError, RuntimeError): - pass # checksum is missing, move on - - @logit(logger) - def link_jediexe(self) -> None: - """ - - This method links a JEDI executable to the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - exe_src = self.task_config.JEDIEXE - - # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - logger.info(f"Link executable {exe_src} to DATA/") - logger.warn("Linking is not permitted per EE2.") - exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - - return exe_dest - - @staticmethod - @logit(logger) - def tgz_diags(statfile: str, diagdir: str) -> None: - """tar and gzip the diagnostic files resulting from a JEDI analysis. - - Parameters - ---------- - statfile : str | os.PathLike - Path to the output .tar.gz .tgz file that will contain the diag*.nc files e.g. atmstat.tgz - diagdir : str | os.PathLike - Directory containing JEDI diag files - """ - - # get list of diag files to put in tarball - diags = glob.glob(os.path.join(diagdir, 'diags', 'diag*nc')) - diags.extend(glob.glob(os.path.join(diagdir, 'diags', 'diag*nc4'))) - - logger.info(f"Compressing {len(diags)} diag files to {statfile}") - - # Open tar.gz file for writing - with tarfile.open(statfile, "w:gz") as tgz: - # Add diag files to tarball - for diagfile in diags: - tgz.add(diagfile, arcname=os.path.basename(diagfile)) - - -@logit(logger) -def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: - """ - Recursively search through a nested dictionary and return the value for the target key. - This returns the first target key it finds. So if a key exists in a subsequent - nested dictionary, it will not be found. - - Parameters - ---------- - nested_dict : Dict - Dictionary to search - target_key : str - Key to search for - - Returns - ------- - Any - Value of the target key - - Raises - ------ - KeyError - If key is not found in dictionary - - TODO: if this gives issues due to landing on an incorrect key in the nested - dictionary, we will have to implement a more concrete method to search for a key - given a more complete address. See resolved conversations in PR 2387 - - # Example usage: - nested_dict = { - 'a': { - 'b': { - 'c': 1, - 'd': { - 'e': 2, - 'f': 3 - } - }, - 'g': 4 - }, - 'h': { - 'i': 5 - }, - 'j': { - 'k': 6 - } - } - - user_key = input("Enter the key to search for: ") - result = find_value_in_nested_dict(nested_dict, user_key) - """ - - if not isinstance(nested_dict, dict): - raise TypeError(f"Input is not of type(dict)") - - result = nested_dict.get(target_key) - if result is not None: - return result - - for value in nested_dict.values(): - if isinstance(value, dict): - try: - result = find_value_in_nested_dict(value, target_key) - if result is not None: - return result - except KeyError: - pass - - raise KeyError(f"Key '{target_key}' not found in the nested dictionary") From fb90bbcc413e62a5da1f43ca7a3343f3c0d50b07 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Sep 2025 16:30:44 +0000 Subject: [PATCH 15/45] FV3Analysis -> Analysis --- ush/python/pygfs/__init__.py | 1 - ush/python/pygfs/task/{fv3_analysis.py => analysis.py} | 2 +- ush/python/pygfs/task/atm_analysis.py | 4 ++-- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- ush/python/pygfs/task/ensemble_recenter.py | 4 ++-- ush/python/pygfs/task/fv3_analysis_calc.py | 4 ++-- 6 files changed, 9 insertions(+), 10 deletions(-) rename ush/python/pygfs/task/{fv3_analysis.py => analysis.py} (99%) diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index d0ab9838ce7..394a93e8a47 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,7 +2,6 @@ import os from .task.analysis import Analysis -from .task.fv3_analysis import FV3Analysis from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis from .task.aero_bmatrix import AerosolBMatrix diff --git a/ush/python/pygfs/task/fv3_analysis.py b/ush/python/pygfs/task/analysis.py similarity index 99% rename from ush/python/pygfs/task/fv3_analysis.py rename to ush/python/pygfs/task/analysis.py index 94ba43a8682..2fdb3f36a2b 100644 --- a/ush/python/pygfs/task/fv3_analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -12,7 +12,7 @@ logger = getLogger(__name__.split('.')[-1]) -class FV3Analysis(Task): +class Analysis(Task): """ General class for JEDI-based global FV3 analysis tasks """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index dbdec9d5875..516fccca5c8 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -11,13 +11,13 @@ add_to_datetime, to_timedelta, parse_j2yaml, logit, save_as_yaml) -from pygfs.task.fv3_analysis import FV3Analysis +from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmAnalysis(FV3Analysis): +class AtmAnalysis(Analysis): """ Class for JEDI-based global atm deterministic analysis tasks """ diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 3162bf2ab09..e7e616ddf77 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -13,13 +13,13 @@ parse_j2yaml, logit, Template, TemplateConstants) -from pygfs.task.fv3_analysis import FV3Analysis +from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class AtmEnsAnalysis(FV3Analysis): +class AtmEnsAnalysis(Analysis): """ Class for JEDI-based global atmens analysis tasks """ diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index 99ba51c1574..f282ae0cb1d 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -8,13 +8,13 @@ add_to_datetime, to_timedelta, to_isotime, to_YMD, parse_j2yaml, logit) -from pygfs.task.fv3_analysis import FV3Analysis +from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class EnsembleRecenter(FV3Analysis): +class EnsembleRecenter(Analysis): """ Class for JEDI-based ensemble increment recentering """ diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index e79969ad9e4..4b512266a75 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -10,12 +10,12 @@ to_timedelta, add_to_datetime, to_fv3time, to_isotime, logit) from pygfs.jedi import Jedi -from pygfs.task.fv3_analysis import FV3Analysis +from pygfs.task.analysis import Analysis logger = getLogger(__name__.split('.')[-1]) -class FV3AnalysisCalc(FV3Analysis): +class FV3AnalysisCalc(Analysis): """ Class for analysis calculation """ From af8b8200290127e519d3078aa149ab87dd892bdf Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Sep 2025 16:38:40 +0000 Subject: [PATCH 16/45] tweaks --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 3b9d328507a..46734b16312 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 3b9d328507af50ec139261e7a30cdcf8d8fa3845 +Subproject commit 46734b1631279207f00181b151b33730f301de56 diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 2fdb3f36a2b..c9495267bd6 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -173,4 +173,4 @@ def extract_tar(tar_file: str) -> None: tarball.extractall(path=tar_path) logger.info(f"Extract {tarball.getnames()}") except Exception as e: - raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e + raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e \ No newline at end of file From 3dc622007376b7b81568da390f29ace0f968e2d8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Sep 2025 17:16:20 +0000 Subject: [PATCH 17/45] Update --- ush/python/pygfs/task/analysis.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index c9495267bd6..5a76b8eb1ad 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -4,7 +4,7 @@ import os import tarfile from typing import Any, Dict -from wxflow import (AttrDict, Task, +from wxflow import (AttrDict, Task, WorkflowException, add_to_datetime, to_timedelta, to_isotime, parse_j2yaml, logit) @@ -14,9 +14,9 @@ class Analysis(Task): """ - General class for JEDI-based global FV3 analysis tasks + General class for JEDI-based global analysis tasks """ - @logit(logger, name="FV3Analysis") + @logit(logger, name="Analysis") def __init__(self, config: Dict[str, Any]): """Constructor global atm analysis task From d201d8b23e347ed346bc19d01e1c7d11e6740507 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 19 Sep 2025 13:40:56 +0000 Subject: [PATCH 18/45] Each task reads its own TASK_CONFIG_YAML --- ush/python/pygfs/jedi/jedi.py | 85 ++++++++++++++++++++++ ush/python/pygfs/task/analysis.py | 76 ++++++++++++------- ush/python/pygfs/task/atm_analysis.py | 22 ++---- ush/python/pygfs/task/atmens_analysis.py | 27 ++----- ush/python/pygfs/task/ensemble_recenter.py | 15 ++-- ush/python/pygfs/task/fv3_analysis_calc.py | 20 ++--- 6 files changed, 162 insertions(+), 83 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index cf1a5319547..7c627198760 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import os +import tarfile from logging import getLogger from typing import List, Dict, Any, Optional from pprint import pformat @@ -265,6 +267,89 @@ def clean_empty_obsspaces(self): if observers == []: raise WorkflowException(f"No observers found in JEDI input config") + @staticmethod + @logit(logger) + def remove_redundant(input_list: List) -> List: + """Remove reduncancies from list with possible redundant, non-mutable elements + + Parameters + ---------- + input_list : List + List with possible redundant, non-mutable elements + + Returns + ---------- + output_list : List + Input list but with redundancies removed + """ + + output_list = [] + for item in input_list: + if item not in output_list: + output_list.append(item) + + return output_list + + @staticmethod + @logit(logger) + def extract_tar_from_filehandler_dict(filehandler_dict) -> None: + """Extract tarballs from FileHandler input dictionary + + This method extracts files from tarballs specified in a FileHander + input dictionary for the 'copy' action. + + Parameters + ---------- + filehandler_dict + Input dictionary for FileHandler + + Returns + ---------- + None + """ + + for item in filehandler_dict['copy']: + # Use the filename from the destination entry if it's a file path + # Otherwise, it's a directory, so use the source entry filename + if os.path.isfile(item[1]): + filename = os.path.basename(item[1]) + else: + filename = os.path.basename(item[0]) + + # Check if file is a tar ball + if os.path.splitext(filename)[1] == '.tar': + tar_file = f"{os.path.dirname(item[1])}/{filename}" + + # Extract tarball + logger.info(f"Extract files from {tar_file}") + extract_tar(tar_file) + + +@logit(logger) +def extract_tar(tar_file: str) -> None: + """Extract files from a tarball + + This method extract files from a tarball + + Parameters + ---------- + tar_file + path/name of tarball + + Returns + ---------- + None + """ + + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except Exception as e: + raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e + @logit(logger) def find_value_in_nested_dict(nested_dict: Dict, target_key: str) -> Any: diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 5a76b8eb1ad..f6b90752064 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 +import glob +import gzip from logging import getLogger import os import tarfile @@ -67,9 +69,6 @@ def __init__(self, config: Dict[str, Any]): } )) - # Extend task_config with content of config yaml for this task - self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) - def initialize(self) -> None: self.initialize() @@ -79,11 +78,31 @@ def execute(self) -> None: def finalize(self) -> None: super.finalize() - def clean(self): + def clean(self) -> None: super().clean() @logit(logger) - def tar_diag_files(self, comout, tarball_name): + def untar_bias_corrections(self) -> None: + """Extract bias correction files from tarballs + This method will extract bias correction files from tarballs + + Parameters + ---------- + None + + Returns + ---------- + None + """ + + bias_file_list = [] + for ob in self.task_config.observations: + if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: + bias_file_list.append(self.task_config.bias_files[ob]) + extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') + + @logit(logger) + def tar_diag_files(self, comout: str, tarball_name: str) -> None: """Compress and tar diag files into COM directory Parameters @@ -117,7 +136,8 @@ def tar_diag_files(self, comout, tarball_name): diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - def tar_radiative_bias_corrections(self, comout, tarball_name): + @logit(logger) + def tar_radiative_bias_corrections(self, comout: str, tarball_name: str) -> None: """Tar radiative bias correction files and into COM directory Parameters @@ -133,7 +153,7 @@ def tar_radiative_bias_corrections(self, comout, tarball_name): """ # Set paths of output tar files - radtar = os.path.join(comout, tarball_name") + radtar = os.path.join(comout, tarball_name) # Get lists of files to put in tarballs satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) @@ -149,28 +169,28 @@ def tar_radiative_bias_corrections(self, comout, tarball_name): # Change OPREFIX to APREFIX in tlapse file name when adding to tarball radbcor.add(tlapfile, arcname=os.path.basename(tlapfile.replace(self.task_config.OPREFIX, self.task_config.APREFIX))) - @staticmethod - @logit(logger) - def extract_tar(tar_file: str) -> None: - """Extract files from a tarball - This method extract files from a tarball +@logit(logger) +def extract_tar(tar_file: str) -> None: + """Extract files from a tarball - Parameters - ---------- - tar_file - path/name of tarball + This method extract files from a tarball - Returns - ---------- - None - """ + Parameters + ---------- + tar_file + path/name of tarball + + Returns + ---------- + None + """ - # extract files from tar file - tar_path = os.path.dirname(tar_file) - try: - with tarfile.open(tar_file, "r") as tarball: - tarball.extractall(path=tar_path) - logger.info(f"Extract {tarball.getnames()}") - except Exception as e: - raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e \ No newline at end of file + # extract files from tar file + tar_path = os.path.dirname(tar_file) + try: + with tarfile.open(tar_file, "r") as tarball: + tarball.extractall(path=tar_path) + logger.info(f"Extract {tarball.getnames()}") + except Exception as e: + raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e \ No newline at end of file diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 516fccca5c8..97bc63839f4 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -1,18 +1,10 @@ #!/usr/bin/env python3 -import os -import glob -import gzip -import tarfile from logging import getLogger -from pprint import pformat -from typing import Any, Dict -from wxflow import (AttrDict, FileHandler, - add_to_datetime, to_timedelta, - parse_j2yaml, - logit, save_as_yaml) from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi +from typing import Any, Dict +from wxflow import AttrDict, FileHandler, parse_j2yaml, logit logger = getLogger(__name__.split('.')[-1]) @@ -62,10 +54,14 @@ def __init__(self, config: Dict[str, Any]): 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, 'BERROR_YAML': _BERROR_YAML, } )) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) + # Create dictionary of Jedi objects expected_keys = ['atmanlvar', 'atmanlfv3inc'] self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @@ -100,11 +96,7 @@ def initialize(self) -> None: # Extract bias corrections from tar files logger.info(f"Extracting bias corrections from tar files") - bias_file_list = [] - for ob in self.task_config.observations: - if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: - bias_file_list.append(self.task_config.bias_files[ob]) - FV3Analysis.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') + self.untar_bias_corrections() # Initialize JEDI variational application logger.info(f"Initializing JEDI applications") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index e7e616ddf77..b04c2ebbe68 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -1,20 +1,10 @@ #!/usr/bin/env python3 -import os -import glob -import gzip -import tarfile from logging import getLogger -from pprint import pformat -from typing import Dict, Any - -from wxflow import (AttrDict, FileHandler, Task, - add_to_datetime, to_timedelta, to_YMD, - parse_j2yaml, - logit, - Template, TemplateConstants) from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi +from typing import Dict, Any +from wxflow import AttrDict, FileHandler, parse_j2yaml, logit logger = getLogger(__name__.split('.')[-1]) @@ -51,9 +41,13 @@ def __init__(self, config: Dict[str, Any]): 'npx_ges': _res + 1, 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, }) ) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) + # Create dictionary of JEDI objects expected_keys = ['atmensanlobs', 'atmensanlsol', 'atmensanlfv3inc', 'atmensanlletkf'] self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @@ -87,11 +81,7 @@ def initialize(self) -> None: # Extract bias corrections from tar files logger.info(f"Extracting bias corrections from tar files") - bias_file_list = [] - for ob in self.task_config.observations: - if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: - bias_file_list.append(self.task_config.bias_files[ob]) - FV3Analysis.extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') + self.untar_bias_corrections() # initialize JEDI applications logger.info(f"Initializing JEDI LETKF observer application") @@ -158,6 +148,3 @@ def finalize(self) -> None: # Save files from COM logger.info(f"Saving files to COM") FileHandler(self.task_config.save).sync() - - def clean(self): - super().clean() diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index f282ae0cb1d..a339567981d 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -1,15 +1,10 @@ #!/usr/bin/env python3 -from datetime import timedelta from logging import getLogger -import os -from pprint import pformat -from wxflow import (AttrDict, FileHandler, Task, Executable, Template, TemplateConstants, - add_to_datetime, to_timedelta, to_isotime, to_YMD, - parse_j2yaml, - logit) from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi +from typing import Dict, Any +from wxflow import AttrDict, FileHandler, parse_j2yaml, logit logger = getLogger(__name__.split('.')[-1]) @@ -19,7 +14,7 @@ class EnsembleRecenter(Analysis): Class for JEDI-based ensemble increment recentering """ @logit(logger, name="EnsembleRecenter") - def __init__(self, config): + def __init__(self, config: Dict[str, Any]): """Constructor for atmospheric ensemble increment recentering task This method will construct an ensemble increment recentering task @@ -50,9 +45,13 @@ def __init__(self, config): 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, } )) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) + # Create dictionary of Jedi objects expected_keys = ['correction_increment', 'ensemble_recenter'] self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index 4b512266a75..efef420b2ea 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -4,13 +4,10 @@ from logging import getLogger import netCDF4 as nc import os -from pprint import pformat -from wxflow import (AttrDict, FileHandler, Task, - parse_j2yaml, - to_timedelta, add_to_datetime, to_fv3time, to_isotime, - logit) from pygfs.jedi import Jedi from pygfs.task.analysis import Analysis +from typing import Dict, Any +from wxflow import AttrDict, FileHandler, to_fv3time, parse_j2yaml, logit logger = getLogger(__name__.split('.')[-1]) @@ -20,7 +17,7 @@ class FV3AnalysisCalc(Analysis): Class for analysis calculation """ @logit(logger, name="FV3AnalysisCalc") - def __init__(self, config): + def __init__(self, config: Dict[str, Any]): """Constructor for analysis calculation task This method will construct an analysis calculation @@ -45,10 +42,8 @@ def __init__(self, config): else: _res_anl = int(self.task_config.CASE[1:]) - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + self.task_config.update(AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, @@ -56,11 +51,12 @@ def __init__(self, config): 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, } - ) + )) - # Extend task_config with local_dict - self.task_config = AttrDict(**self.task_config, **local_dict) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) # Create dictionary of Jedi objects expected_keys = ['atm_addincrement'] From d833fdb85705d63afa260e7c42eb2245f4fb7bf8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 22 Sep 2025 18:15:56 +0000 Subject: [PATCH 19/45] Refactor snow --- dev/parm/config/gfs/config.snowanl.j2 | 7 +- jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE | 2 - sorc/gdas.cd | 2 +- ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/snow_analysis.py | 153 ++++----------------- ush/python/pygfs/task/snowens_analysis.py | 157 ++++------------------ 6 files changed, 50 insertions(+), 273 deletions(-) diff --git a/dev/parm/config/gfs/config.snowanl.j2 b/dev/parm/config/gfs/config.snowanl.j2 index 83f1a9ae78a..8faaab0fa06 100644 --- a/dev/parm/config/gfs/config.snowanl.j2 +++ b/dev/parm/config/gfs/config.snowanl.j2 @@ -12,12 +12,7 @@ source "${EXPDIR}/config.resources" snowanl export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x" export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/apply_incr_nml.j2" -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_det_jedi_config.yaml.j2" -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/snow/snow_stage_jedi_fix.yaml.j2" -export STAGE_BKG_YAML="${PARMgfs}/gdas/snow/snow_det_stage_bkg.yaml.j2" -export STAGE_BERROR_YAML="${PARMgfs}/gdas/snow/snow_stage_berror.yaml.j2" -export STAGE_GTS_YAML="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" -export STAGE_IMS_SCF2IODA_YAML="${PARMgfs}/gdas/snow/snow_stage_ims_scf2ioda.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_det_config.yaml.j2" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE index 1298d011aff..0bd01398c96 100755 --- a/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE @@ -24,8 +24,6 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL \ COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL -mkdir -m 775 -p "${COMIN_ATMOS_ANALYSIS_PREV}" - ############################################################### # Run relevant script diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 46734b16312..e09c546d367 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 46734b1631279207f00181b151b33730f301de56 +Subproject commit e09c546d367e0373c5591945a1134a12aa9b4741 diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index f6b90752064..0f495473e73 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -50,7 +50,7 @@ def __init__(self, config: Dict[str, Any]): if 'BIAS_FILES_YAML' in self.task_config: _bias_files = parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'] else: - _bias_files = {} + _bias_files = AttrDict # Extend task_config with variables that are repeatedly used across this class self.task_config.update(AttrDict( diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 8051e28f1de..1a82bac4422 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -9,19 +9,15 @@ import tarfile import numpy as np from netCDF4 import Dataset - -from wxflow import (AttrDict, - FileHandler, +from pygfs.task.analysis import Analysis +from pygfs.jedi import Jedi +from wxflow import (AttrDict, Executable, FileHandler, WorkflowException, to_fv3time, to_YMD, to_YMDH, to_timedelta, add_to_datetime, to_julian, rm_p, cp, parse_j2yaml, save_as_yaml, Jinja, - Task, - logit, - Executable, - WorkflowException) -from pygfs.jedi import Jedi + logit) logger = getLogger(__name__.split('.')[-1]) @@ -31,7 +27,7 @@ class SnowAnalysis(Task): Class for JEDI-based global snow analysis tasks """ - @logit(logger, name="SnowAnalysis") + @logit(logger, name="Analysis") def __init__(self, config: Dict[str, Any]): """Constructor global snow analysis task @@ -52,32 +48,20 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) _res = int(self.task_config['CASE'][1:]) - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - # fix ocnres - self.task_config.OCNRES = f"{self.task_config.OCNRES:03d}" - - # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + # Extend task_config with variables repeatedly used across this class + self.task_config.update(AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, - 'SNOW_WINDOW_BEGIN': _window_begin, - 'SNOW_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", - 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'snow_obsdatain_path': os.path.join(self.task_config.DATA, 'obs'), - 'snow_obsdataout_path': os.path.join(self.task_config.DATA, 'diags'), - 'snow_bkg_path': os.path.join('.', 'bkg/'), - 'res': _res, + 'OCNRES': f"{self.task_config.OCNRES:03d}" } - ) + )) - # Extend task_config with local_dict - self.task_config = AttrDict(**self.task_config, **local_dict) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) # Create JEDI object dictionary expected_keys = ['scf_to_ioda', 'snowanlvar'] @@ -106,47 +90,12 @@ def initialize(self) -> None: None """ - # stage backgrounds - logger.info(f"Staging background files from {self.task_config.STAGE_BKG_YAML}") - bkg_staging_dict = parse_j2yaml(self.task_config.STAGE_BKG_YAML, self.task_config) - FileHandler(bkg_staging_dict).sync() - logger.debug(f"Background files:\n{pformat(bkg_staging_dict)}") - - # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - obs_dict = self.jedi_dict['snowanlvar'].render_jcb(self.task_config, 'snow_obs_staging') - FileHandler(obs_dict).sync() - logger.debug(f"Observation files:\n{pformat(obs_dict)}") - - # stage GTS bufr2ioda mapping YAML files - logger.info(f"Staging GTS bufr2ioda mapping YAML files from {self.task_config.STAGE_GTS_YAML}") - gts_mapping_list = parse_j2yaml(self.task_config.STAGE_GTS_YAML, self.task_config) - FileHandler(gts_mapping_list).sync() - - # stage FV3-JEDI fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_dict = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_dict).sync() - logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") - - # staging B error files - logger.info("Stage files for static background error") - berror_staging_dict = parse_j2yaml(self.task_config.STAGE_BERROR_YAML, self.task_config) - FileHandler(berror_staging_dict).sync() - logger.debug(f"Background error files:\n{pformat(berror_staging_dict)}") - - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config.DATA, 'anl'), - os.path.join(self.task_config.DATA, 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - - # if 00z, do SCF preprocessing + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.data_in).sync() + + # if 00z, do initialize application for SCF preprocessing if self.task_config.cyc == 0: - ims_scf_to_ioda_staging_dict = parse_j2yaml(self.task_config.STAGE_IMS_SCF2IODA_YAML, self.task_config) - FileHandler(ims_scf_to_ioda_staging_dict).sync() self.jedi_dict['scf_to_ioda'].initialize(self.task_config) # initialize JEDI variational application @@ -186,67 +135,13 @@ def finalize(self) -> None: Instance of the SnowAnalysis object """ - # ---- tar up diags - # path of output tar statfile - snowstat = os.path.join(self.task_config.COMOUT_SNOW_ANALYSIS, f"{self.task_config.APREFIX}snowstat.tgz") - - # get list of diag files to put in tarball - diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) - - logger.info(f"Compressing {len(diags)} diag files to {snowstat}") - - # gzip the files first - logger.debug(f"Gzipping {len(diags)} diag files") - for diagfile in diags: - with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: - f_out.writelines(f_in) - - # open tar file for writing - logger.debug(f"Creating tar file {snowstat} with {len(diags)} gzipped diag files") - with tarfile.open(snowstat, "w|gz") as archive: - for diagfile in diags: - diaggzip = f"{diagfile}.gz" - archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # get list of yamls to copy to ROTDIR - yamls = glob.glob(os.path.join(self.task_config.DATA, '*snow*yaml')) - - # copy full YAML from executable to ROTDIR - for src in yamls: - yaml_base = os.path.splitext(os.path.basename(src))[0] - dest_yaml_name = f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{yaml_base}.yaml" - dest = os.path.join(self.task_config.COMOUT_CONF, dest_yaml_name) - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() - - logger.info("Copy analysis to COM") - bkgtimes = [] - if self.task_config.DOIAU: - # need both beginning and middle of window - bkgtimes.append(self.task_config.SNOW_WINDOW_BEGIN) - bkgtimes.append(self.task_config.current_cycle) - anllist = [] - for bkgtime in bkgtimes: - template = f'{to_fv3time(bkgtime)}.sfc_data.tile{{tilenum}}.nc' - for itile in range(1, self.task_config.ntiles + 1): - filename = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', filename) - dest = os.path.join(self.task_config.COMOUT_SNOW_ANALYSIS, filename) - anllist.append([src, dest]) - FileHandler({'copy': anllist}).sync() + # Compress and tar diag files into COM directory + self.tar_diag_files(self.task_config.COMOUT_SNOW_ANALYSIS, + f"{self.task_config.APREFIX}snowstat.tgz") - logger.info('Copy increments to COM') - template = f'snowinc.{to_fv3time(self.task_config.current_cycle)}.sfc_data.tile{{tilenum}}.nc' - inclist = [] - for itile in range(1, self.task_config.ntiles + 1): - filename = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', filename) - dest = os.path.join(self.task_config.COMOUT_SNOW_ANALYSIS, filename) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + # Save files to COM + logger.info(f"Saving files to COM") + FileHandler(self.task_config.data_out).sync() @logit(logger) def add_increments(self) -> None: @@ -263,7 +158,7 @@ def add_increments(self) -> None: bkgtimes = [] if self.task_config.DOIAU: # want analysis at beginning and middle of window - bkgtimes.append(self.task_config.SNOW_WINDOW_BEGIN) + bkgtimes.append(self.task_config.WINDOW_BEGIN) bkgtimes.append(self.task_config.current_cycle) anllist = [] for bkgtime in bkgtimes: @@ -278,7 +173,7 @@ def add_increments(self) -> None: if self.task_config.DOIAU: logger.info("Copying increments to beginning of window") template_in = f'snowinc.{to_fv3time(self.task_config.current_cycle)}.sfc_data.tile{{tilenum}}.nc' - template_out = f'snowinc.{to_fv3time(self.task_config.SNOW_WINDOW_BEGIN)}.sfc_data.tile{{tilenum}}.nc' + template_out = f'snowinc.{to_fv3time(self.task_config.WINDOW_BEGIN)}.sfc_data.tile{{tilenum}}.nc' inclist = [] for itile in range(1, self.task_config.ntiles + 1): filename_in = template_in.format(tilenum=itile) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index f513c5da7d1..fe3b1b873ce 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -52,33 +52,23 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) _res = int(self.task_config['CASE_ENS'][1:]) - self.task_config['CASE'] = self.task_config['CASE_ENS'] - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - # fix ocnres - self.task_config.OCNRES = f"{self.task_config.OCNRES :03d}" + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + # Extend task_config with variables repeatedly used across this class + self.task_config.update(AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, - 'SNOW_WINDOW_BEGIN': _window_begin, - 'SNOW_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", - 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'snow_obsdatain_path': os.path.join(self.task_config.DATA, 'obs'), - 'snow_obsdataout_path': os.path.join(self.task_config.DATA, 'diags'), - 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), - 'res': _res, + 'CASE': self.task_config.CASE_ENS, + 'OCNRES': f"{self.task_config.OCNRES:03d}" } - ) + )) - # Extend task_config with local_dict - self.task_config = AttrDict(**self.task_config, **local_dict) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) # Create JEDI object dictionary expected_keys = ['scf_to_ioda', 'snowanlvar', 'esnowanlensmean'] @@ -107,17 +97,10 @@ def initialize(self) -> None: None """ - # stage backgrounds - logger.info(f"Staging background files from {self.task_config.STAGE_BKG_YAML}") - bkg_staging_dict = parse_j2yaml(self.task_config.STAGE_BKG_YAML, self.task_config) - FileHandler(bkg_staging_dict).sync() - logger.debug(f"Background files:\n{pformat(bkg_staging_dict)}") - - # stage orography - logger.info(f"Staging orography files from {self.task_config.STAGE_OROG_YAML}") - orog_staging_dict = parse_j2yaml(self.task_config.STAGE_OROG_YAML, self.task_config) - FileHandler(orog_staging_dict).sync() - logger.debug(f"Orography files:\n{pformat(orog_staging_dict)}") + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.data_in).sync() + # note JEDI will try to read the orog files for each member, let's just symlink logger.info("Linking orography files for each member") oro_files = glob.glob(os.path.join(self.task_config.DATA, 'orog', 'ens', '*')) @@ -128,52 +111,14 @@ def initialize(self) -> None: # need to symlink orography files for the ensmean too dest = os.path.join(self.task_config.DATA, 'bkg', 'ensmean') for oro_file in oro_files: - os.symlink(oro_file, os.path.join(dest, os.path.basename(oro_file))) - - # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - obs_dict = self.jedi_dict['snowanlvar'].render_jcb(self.task_config, 'snow_obs_staging') - FileHandler(obs_dict).sync() - logger.debug(f"Observation files:\n{pformat(obs_dict)}") - - # stage GTS bufr2ioda mapping YAML files - logger.info(f"Staging GTS bufr2ioda mapping YAML files from {self.task_config.STAGE_GTS_YAML}") - gts_mapping_list = parse_j2yaml(self.task_config.STAGE_GTS_YAML, self.task_config) - FileHandler(gts_mapping_list).sync() - - # stage FV3-JEDI fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_dict = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_dict).sync() - logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") - - # staging B error files - logger.info("Stage files for static background error") - berror_staging_dict = parse_j2yaml(self.task_config.STAGE_BERROR_YAML, self.task_config) - FileHandler(berror_staging_dict).sync() - logger.debug(f"Background error files:\n{pformat(berror_staging_dict)}") - - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config.DATA, 'anl'), - os.path.join(self.task_config.DATA, 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - - # if 00z, do SCF preprocessing - if self.task_config.cyc == 0: - ims_scf_to_ioda_staging_dict = parse_j2yaml(self.task_config.STAGE_IMS_SCF2IODA_YAML, self.task_config) - FileHandler(ims_scf_to_ioda_staging_dict).sync() - self.jedi_dict['scf_to_ioda'].initialize(self.task_config) + os.symlink(oro_file, os.path.join(dest, os.path.basename(oro_file)))) - # initialize JEDI variational application - logger.info(f"Initializing JEDI variational DA application") + # Initialize JEDI applications + logger.info(f"Initializing JEDI applications") self.jedi_dict['snowanlvar'].initialize(self.task_config, clean_empty_obsspaces=False) - - # initialize ensemble mean computation - logger.info(f"Initializing JEDI ensemble mean application") self.jedi_dict['esnowanlensmean'].initialize(self.task_config) + if self.task_config.cyc == 0: + self.jedi_dict['scf_to_ioda'].initialize(self.task_config) @logit(logger) def execute(self, jedi_dict_key: str) -> None: @@ -208,69 +153,13 @@ def finalize(self) -> None: Instance of the SnowEnsAnalysis object """ - # ---- tar up diags - # path of output tar statfile - snowstat = os.path.join(self.task_config.COMOUT_SNOW_ANALYSIS, f"{self.task_config.APREFIX}snowstat.tgz") - - # get list of diag files to put in tarball - diags = glob.glob(os.path.join(self.task_config.DATA, 'diags', 'diag*nc')) - - logger.info(f"Compressing {len(diags)} diag files to {snowstat}") - - # gzip the files first - logger.debug(f"Gzipping {len(diags)} diag files") - for diagfile in diags: - with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: - f_out.writelines(f_in) - - # open tar file for writing - logger.debug(f"Creating tar file {snowstat} with {len(diags)} gzipped diag files") - with tarfile.open(snowstat, "w|gz") as archive: - for diagfile in diags: - diaggzip = f"{diagfile}.gz" - archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # get list of yamls to copy to ROTDIR - yamls = glob.glob(os.path.join(self.task_config.DATA, '*snow*yaml')) - - # copy full YAML from executable to ROTDIR - for src in yamls: - yaml_base = os.path.splitext(os.path.basename(src))[0] - dest_yaml_name = f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.{yaml_base}.yaml" - dest = os.path.join(self.task_config.COMOUT_CONF, dest_yaml_name) - logger.debug(f"Copying {src} to {dest}") - yaml_copy = { - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() + # Compress and tar diag files into COM directory + self.tar_diag_files(self.task_config.COMOUT_SNOW_ANALYSIS, + f"{self.task_config.APREFIX}snowstat.tgz") - logger.info("Copy analysis to COM") - bkgtimes = [] - if self.task_config.DOIAU: - # need both beginning and middle of window - bkgtimes.append(self.task_config.SNOW_WINDOW_BEGIN) - bkgtimes.append(self.task_config.current_cycle) - anllist = [] - for mem in range(1, self.task_config.NMEM_ENS + 1): - for bkgtime in bkgtimes: - template = f'{to_fv3time(bkgtime)}.sfc_data.tile{{tilenum}}.nc' - for itile in range(1, self.task_config.ntiles + 1): - filename = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', f"mem{mem:03d}", filename) - COMOUT_SNOW_ANALYSIS = self.task_config.COMOUT_SNOW_ANALYSIS.replace('ensstat', f"mem{mem:03d}") - dest = os.path.join(COMOUT_SNOW_ANALYSIS, filename) - anllist.append([src, dest]) - FileHandler({'copy': anllist}).sync() - - logger.info('Copy increments to COM') - template = f'snowinc.{to_fv3time(self.task_config.current_cycle)}.sfc_data.tile{{tilenum}}.nc' - inclist = [] - for itile in range(1, self.task_config.ntiles + 1): - filename = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', filename) - dest = os.path.join(self.task_config.COMOUT_SNOW_ANALYSIS, filename) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + # Save files to COM + logger.info(f"Saving files to COM") + FileHandler(self.task_config.data_out).sync() @logit(logger) def add_increments(self) -> None: From 8392ba7e93cf81c7f82fa214b52f38fcffde2477 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 24 Sep 2025 16:54:29 +0000 Subject: [PATCH 20/45] Finish snow refactor --- dev/parm/config/gfs/config.esnowanl.j2 | 12 ++------- sorc/gdas.cd | 2 +- ush/python/pygfs/task/atm_analysis.py | 4 +-- ush/python/pygfs/task/atmens_analysis.py | 4 +-- ush/python/pygfs/task/ensemble_recenter.py | 4 +-- ush/python/pygfs/task/fv3_analysis_calc.py | 4 +-- ush/python/pygfs/task/snow_analysis.py | 8 +++--- ush/python/pygfs/task/snowens_analysis.py | 29 ++++++---------------- 8 files changed, 23 insertions(+), 44 deletions(-) diff --git a/dev/parm/config/gfs/config.esnowanl.j2 b/dev/parm/config/gfs/config.esnowanl.j2 index 4da6cf78473..2dbf77949b4 100644 --- a/dev/parm/config/gfs/config.esnowanl.j2 +++ b/dev/parm/config/gfs/config.esnowanl.j2 @@ -8,21 +8,13 @@ echo "BEGIN: config.esnowanl" # Get task specific resources source "${EXPDIR}/config.resources" esnowanl -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_ens_jedi_config.yaml.j2" -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/snow/snow_stage_jedi_fix.yaml.j2" -export STAGE_OROG_YAML="${PARMgfs}/gdas/snow/snow_stage_orog.yaml.j2" -export STAGE_BERROR_YAML="${PARMgfs}/gdas/snow/snow_stage_berror.yaml.j2" -export STAGE_BKG_YAML="${PARMgfs}/gdas/snow/snow_ens_stage_bkg.yaml.j2" -export STAGE_IMS_SCF2IODA_YAML="${PARMgfs}/gdas/snow/snow_stage_ims_scf2ioda.yaml.j2" -export STAGE_GTS_YAML="${PARMgfs}/gdas/snow/obs/config/bufr2ioda_mapping.yaml.j2" -export SAVE_YAML="${PARMgfs}/gdas/snow/snow_ens_save.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_ens_config.yaml.j2" +export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2" # Name of the executable that applies increment to bkg and its namelist template export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x" export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/ens_apply_incr_nml.j2" -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_ens_jedi_config.yaml.j2" - export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index e09c546d367..c621722b2c9 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit e09c546d367e0373c5591945a1134a12aa9b4741 +Subproject commit c621722b2c95a77de0e6d8a64918735ad312bc36 diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 97bc63839f4..8c7aefd3254 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -92,7 +92,7 @@ def initialize(self) -> None: # Stage files from COM logger.info(f"Staging files from COM") - FileHandler(self.task_config.stage).sync() + FileHandler(self.task_config.data_in).sync() # Extract bias corrections from tar files logger.info(f"Extracting bias corrections from tar files") @@ -148,4 +148,4 @@ def finalize(self) -> None: # Save files from COM logger.info(f"Saving files to COM") - FileHandler(self.task_config.save).sync() + FileHandler(self.task_config.data_out).sync() diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index b04c2ebbe68..7187af21e01 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -77,7 +77,7 @@ def initialize(self) -> None: # Stage files from COM logger.info(f"Staging files from COM") - FileHandler(self.task_config.stage).sync() + FileHandler(self.task_config.data_in).sync() # Extract bias corrections from tar files logger.info(f"Extracting bias corrections from tar files") @@ -147,4 +147,4 @@ def finalize(self) -> None: # Save files from COM logger.info(f"Saving files to COM") - FileHandler(self.task_config.save).sync() + FileHandler(self.task_config.data_out).sync() diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index a339567981d..8151fb93756 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -77,7 +77,7 @@ def initialize(self) -> None: # Stage files from COM logger.info(f"Staging files from COM") - FileHandler(self.task_config.stage).sync() + FileHandler(self.task_config.data_in).sync() # Initialize JEDI ensemble increment recentering application logger.info(f"Initializing JEDI applications") @@ -124,4 +124,4 @@ def finalize(self) -> None: # Save output files to COM logger.info(f"Saving output files to COM") - FileHandler(self.task_config.save).sync() + FileHandler(self.task_config.data_out).sync() diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index efef420b2ea..57d54f7c321 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -87,7 +87,7 @@ def initialize(self) -> None: # Stage files from COM logger.info(f"Staging files from COM") - FileHandler(self.task_config.stage).sync() + FileHandler(self.task_config.data_in).sync() # Initialize GDASApp JEDI addincrement application logger.info(f"Initializing GDASApp JEDI addincrement applications") @@ -172,7 +172,7 @@ def finalize(self) -> None: # Save files from COM logger.info(f"Saving files to COM") - FileHandler(self.task_config.save).sync() + FileHandler(self.task_config.data_out).sync() @logit(logger) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 1a82bac4422..b9f9c895008 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -22,7 +22,7 @@ logger = getLogger(__name__.split('.')[-1]) -class SnowAnalysis(Task): +class SnowAnalysis(Analysis): """ Class for JEDI-based global snow analysis tasks """ @@ -56,7 +56,8 @@ def __init__(self, config: Dict[str, Any]): 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, - 'OCNRES': f"{self.task_config.OCNRES:03d}" + 'OCNRES': f"{self.task_config.OCNRES:03d}", + 'snow_bkg_path': os.path.join('.', 'bkg/'), } )) @@ -65,8 +66,7 @@ def __init__(self, config: Dict[str, Any]): # Create JEDI object dictionary expected_keys = ['scf_to_ioda', 'snowanlvar'] - jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index fe3b1b873ce..6e88569a935 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -9,7 +9,8 @@ import tarfile import numpy as np from netCDF4 import Dataset - +from pygfs.task.analysis import Analysis +from pygfs.jedi import Jedi from wxflow import (AttrDict, FileHandler, to_fv3time, to_YMD, to_YMDH, to_timedelta, add_to_datetime, @@ -17,16 +18,14 @@ rm_p, cp, parse_j2yaml, save_as_yaml, Jinja, - Task, logit, Executable, WorkflowException) -from pygfs.jedi import Jedi logger = getLogger(__name__.split('.')[-1]) -class SnowEnsAnalysis(Task): +class SnowEnsAnalysis(Analysis): """ Class for JEDI-based global snow ensemble analysis tasks """ @@ -63,7 +62,8 @@ def __init__(self, config: Dict[str, Any]): 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, 'CASE': self.task_config.CASE_ENS, - 'OCNRES': f"{self.task_config.OCNRES:03d}" + 'OCNRES': f"{self.task_config.OCNRES:03d}", + 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), } )) @@ -72,8 +72,7 @@ def __init__(self, config: Dict[str, Any]): # Create JEDI object dictionary expected_keys = ['scf_to_ioda', 'snowanlvar', 'esnowanlensmean'] - jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @logit(logger) def initialize(self) -> None: @@ -101,18 +100,6 @@ def initialize(self) -> None: logger.info(f"Staging files from COM") FileHandler(self.task_config.data_in).sync() - # note JEDI will try to read the orog files for each member, let's just symlink - logger.info("Linking orography files for each member") - oro_files = glob.glob(os.path.join(self.task_config.DATA, 'orog', 'ens', '*')) - for mem in range(1, self.task_config.NMEM_ENS + 1): - dest = os.path.join(self.task_config.DATA, 'bkg', f"mem{mem:03}") - for oro_file in oro_files: - os.symlink(oro_file, os.path.join(dest, os.path.basename(oro_file))) - # need to symlink orography files for the ensmean too - dest = os.path.join(self.task_config.DATA, 'bkg', 'ensmean') - for oro_file in oro_files: - os.symlink(oro_file, os.path.join(dest, os.path.basename(oro_file)))) - # Initialize JEDI applications logger.info(f"Initializing JEDI applications") self.jedi_dict['snowanlvar'].initialize(self.task_config, clean_empty_obsspaces=False) @@ -174,7 +161,7 @@ def add_increments(self) -> None: if self.task_config.DOIAU: logger.info("Copying increments to beginning of window") template_in = f'snowinc.{to_fv3time(self.task_config.current_cycle)}.sfc_data.tile{{tilenum}}.nc' - template_out = f'snowinc.{to_fv3time(self.task_config.SNOW_WINDOW_BEGIN)}.sfc_data.tile{{tilenum}}.nc' + template_out = f'snowinc.{to_fv3time(self.task_config.WINDOW_BEGIN)}.sfc_data.tile{{tilenum}}.nc' inclist = [] for itile in range(1, self.task_config.ntiles + 1): filename_in = template_in.format(tilenum=itile) @@ -187,7 +174,7 @@ def add_increments(self) -> None: bkgtimes = [] if self.task_config.DOIAU: # need both beginning and middle of window - bkgtimes.append(self.task_config.SNOW_WINDOW_BEGIN) + bkgtimes.append(self.task_config.WINDOW_BEGIN) bkgtimes.append(self.task_config.current_cycle) # loop over members From e9a1994d996fa0525629e53eaa58b1b821a2b446 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 24 Sep 2025 16:54:59 +0000 Subject: [PATCH 21/45] Finish snow refactor --- dev/parm/config/gfs/config.snowanl.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/parm/config/gfs/config.snowanl.j2 b/dev/parm/config/gfs/config.snowanl.j2 index 8faaab0fa06..0fd94aa0a72 100644 --- a/dev/parm/config/gfs/config.snowanl.j2 +++ b/dev/parm/config/gfs/config.snowanl.j2 @@ -13,6 +13,7 @@ export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x" export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/apply_incr_nml.j2" export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_det_config.yaml.j2" +export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" From 7d2243fc51256511946e24d033f63fcfd35866a7 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 24 Sep 2025 17:14:02 +0000 Subject: [PATCH 22/45] debug --- ush/python/pygfs/task/snowens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 6e88569a935..b5016f71162 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -142,7 +142,7 @@ def finalize(self) -> None: # Compress and tar diag files into COM directory self.tar_diag_files(self.task_config.COMOUT_SNOW_ANALYSIS, - f"{self.task_config.APREFIX}snowstat.tgz") + f"{self.task_config.APREFIX_ENS}snowstat.tgz") # Save files to COM logger.info(f"Saving files to COM") From d5978c6d8a6b880ba36924c57cd1ef4a832e3ec8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 25 Sep 2025 15:37:44 +0000 Subject: [PATCH 23/45] First draft of aero refactor --- dev/parm/config/gcafs/config.aeroanl.j2 | 16 +-- dev/parm/config/gfs/config.aeroanl.j2 | 18 +-- jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 4 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/aero_analysis.py | 141 +++++++----------------- ush/python/pygfs/task/aero_bmatrix.py | 57 ++++------ ush/python/pygfs/task/analysis.py | 9 +- 7 files changed, 77 insertions(+), 170 deletions(-) diff --git a/dev/parm/config/gcafs/config.aeroanl.j2 b/dev/parm/config/gcafs/config.aeroanl.j2 index e97b6d088a6..ddfaca39576 100644 --- a/dev/parm/config/gcafs/config.aeroanl.j2 +++ b/dev/parm/config/gcafs/config.aeroanl.j2 @@ -18,22 +18,14 @@ case ${CASE} in exit 4 esac export CASE_ANL -export JCB_ALGO_YAML_VAR=${PARMgfs}/gdas/aero/jcb-prototype_3dvar.yaml.j2 -export STATICB_TYPE='diffusion' -export BERROR_YAML="aero_background_error_static_${STATICB_TYPE}" -export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_det_jedi_config.yaml.j2" -export STAGE_CRTM_COEFF_YAML="${PARMgfs}/gdas/aero/aero_stage_crtm_coeff.yaml.j2" -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/aero/aero_stage_jedi_fix.yaml.j2" -export STAGE_YAML="${PARMgfs}/gdas/aero/aero_det_stage.yaml.j2" -export SAVE_YAML="${PARMgfs}/gdas/aero/aero_det_save.yaml.j2" +export STATICB_TYPE='diffusion' -export AERO_BMATRIX_RESCALE_YAML="aero_gen_bmatrix_rescale_default.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_bmat_config.yaml.j2" +export OBS_LIST_YAML="${PARMgfs}/gdas/aero/aero_obs_list.yaml.j2" +export BIAS_FILES_YAML="${PARMgfs}/gdas/aero/aero_bias_files.yaml.j2" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" -export aero_bkg_times="3,6,9" - echo "END: config.aeroanl" diff --git a/dev/parm/config/gfs/config.aeroanl.j2 b/dev/parm/config/gfs/config.aeroanl.j2 index f7d21986930..ddfaca39576 100644 --- a/dev/parm/config/gfs/config.aeroanl.j2 +++ b/dev/parm/config/gfs/config.aeroanl.j2 @@ -18,24 +18,14 @@ case ${CASE} in exit 4 esac export CASE_ANL -export JCB_ALGO_YAML_VAR=${PARMgfs}/gdas/aero/jcb-prototype_3dvar.yaml.j2 + export STATICB_TYPE='diffusion' -export BERROR_YAML="aero_background_error_static_${STATICB_TYPE}" -export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_det_jedi_config.yaml.j2" -export STAGE_CRTM_COEFF_YAML="${PARMgfs}/gdas/aero/aero_stage_crtm_coeff.yaml.j2" -export STAGE_JEDI_FIX_YAML="${PARMgfs}/gdas/aero/aero_stage_jedi_fix.yaml.j2" -export STAGE_YAML="${PARMgfs}/gdas/aero/aero_det_stage.yaml.j2" -export SAVE_YAML="${PARMgfs}/gdas/aero/aero_det_save.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_bmat_config.yaml.j2" +export OBS_LIST_YAML="${PARMgfs}/gdas/aero/aero_obs_list.yaml.j2" +export BIAS_FILES_YAML="${PARMgfs}/gdas/aero/aero_bias_files.yaml.j2" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" -if [[ "${DOIAU}" == "YES" ]]; then - export aero_bkg_times="3,6,9" -else - export aero_bkg_times="6," # Trailing comma is necessary so this is treated as a list -fi - echo "END: config.aeroanl" diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index a0f1a04a45b..545087aaaff 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -18,6 +18,10 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_CONF:COM_CONF_TMPL \ COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL +mkdir -m 755 -p "${COMOUT_CHEM_ANALYSIS}" +mkdir -m 755 -p "${COMOUT_ATMOS_RESTART}" +mkdir -m 755 -p "${COMOUT_CONF}" + ############################################################### # Run relevant script diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c621722b2c9..081a825c27b 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c621722b2c95a77de0e6d8a64918735ad312bc36 +Subproject commit 081a825c27b5f1ff101a1eab950bdb17a8c3505d diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 6946257d5a0..cc663e3bb86 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -1,22 +1,18 @@ #!/usr/bin/env python3 import os -import glob -import gzip -import tarfile from logging import getLogger -from pprint import pformat from netCDF4 import Dataset from typing import Dict, List - -from wxflow import (AttrDict, - FileHandler, - add_to_datetime, to_fv3time, to_timedelta, - to_fv3time, - Task, - YAMLFile, parse_j2yaml, - logit) from pygfs.jedi import Jedi +from wxflow import ( + AttrDict, + FileHandler, + to_fv3time, + Task, + YAMLFile, parse_j2yaml, + logit +) import numpy as np logger = getLogger(__name__.split('.')[-1]) @@ -48,32 +44,30 @@ def __init__(self, config): _res = int(self.task_config['CASE'][1:]) _res_anl = int(self.task_config['CASE_ANL'][1:]) - _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + if self.task_config.DOIAU: + _anl_time = self.task_config.WINDOW_BEGIN + else: + _anl_time = self.task_config.current_cycle + + # Extend task_config with variables repeatedly used across this class + self.task_config.update(AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config['LEVS'] - 1, - 'AERO_WINDOW_BEGIN': _window_begin, - 'AERO_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", - 'aero_bkg_fhr': [fh - 3 for fh in self.task_config['aero_bkg_times']], - 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gcdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'aero_obsdatain_path': f"{self.task_config.DATA}/obs/", - 'aero_obsdataout_path': f"{self.task_config.DATA}/diags/", - 'BKG_TSTEP': "PT3H" # FGAT + 'npz': self.task_config.LEVS - 1, + 'BKG_TSTEP': "PT3H", # FGAT + 'BERROR_YAML': f'aero_background_error_static_{self.task_config.STATICB_TYPE}', + 'anl_time': _anl_time, } ) - # Extend task_config with local_dict - self.task_config = AttrDict(**self.task_config, **local_dict) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) # Create dictionary of Jedi objects expected_keys = ['aeroanlvar'] @@ -96,45 +90,13 @@ def initialize(self) -> None: - creating output directories """ - # stage observations - logger.info(f"Staging list of observation files generated from JEDI config") - obs_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_obs_staging') - FileHandler(obs_dict).sync() - logger.debug(f"Observation files:\n{pformat(obs_dict)}") + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.data_in).sync() - # # stage bias corrections - logger.info(f"Staging list of bias correction files") - bias_dict = self.jedi_dict['aeroanlvar'].render_jcb(self.task_config, 'aero_bias_staging') - - if bias_dict['copy'] is None: - logger.info(f"No bias correction files to stage") - else: - try: - bias_dict['copy'] = Jedi.remove_redundant(bias_dict['copy']) - FileHandler(bias_dict).sync() - logger.debug(f"Bias correction files:\n{pformat(bias_dict)}") - - # extract bias corrections - Jedi.extract_tar_from_filehandler_dict(bias_dict) - except FileNotFoundError: - logger.error(f"Bias correction files or directories do not exist:\n{pformat(bias_dict)}") - - # stage CRTM fix files - logger.info(f"Staging CRTM fix files from {self.task_config.STAGE_CRTM_COEFF_YAML}") - crtm_fix_dict = parse_j2yaml(self.task_config.STAGE_CRTM_COEFF_YAML, self.task_config) - FileHandler(crtm_fix_dict).sync() - logger.debug(f"CRTM fix files:\n{pformat(crtm_fix_dict)}") - - # stage fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_dict = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_dict).sync() - logger.debug(f"JEDI fix files:\n{pformat(jedi_fix_dict)}") - - # stage files from COM and create working directories - logger.info(f"Staging files prescribed from {self.task_config.STAGE_YAML}") - stage_dict = parse_j2yaml(self.task_config.STAGE_YAML, self.task_config) - FileHandler(stage_dict).sync() + # Extract bias corrections from tar files + logger.info(f"Extracting bias corrections from tar files") + self.untar_bias_corrections() # initialize JEDI variational application logger.info(f"Initializing JEDI variational DA application") @@ -169,49 +131,22 @@ def finalize(self) -> None: - moving the increment files to the ROTDIR """ - # ---- tar up diags - # path of output tar statfile - logger.info('Preparing observation space diagnostics for archiving') - aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat.tgz") - - # get list of diag files to put in tarball - diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc')) - - # gzip the files first - for diagfile in diags: - logger.info(f'Adding {diagfile} to tar file') - with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: - f_out.writelines(f_in) # ---- add increments to RESTART files logger.info('Adding increments to RESTART files') self._add_fms_cube_sphere_increments() - # tar up bias correction files - bfile = f"{self.task_config.APREFIX}aero_varbc_params.tar" - aertar = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, bfile) - - # get lists of aerosol bias correction files to add to tarball - satlist = glob.glob(os.path.join(self.task_config.DATA, 'bc', '*satbias*nc')) - - # copy files back to COM - logger.info(f"Copying files to COM based on {self.task_config.SAVE_YAML}") - save_dict = parse_j2yaml(self.task_config.SAVE_YAML, self.task_config) - FileHandler(save_dict).sync() - - # tar aerosol bias correction files to ROTDIR - logger.info(f"Creating aerosol bias correction tar file {aertar}") - with tarfile.open(aertar, 'w') as aerbcor: - for satfile in satlist: - aerbcor.add(satfile, arcname=os.path.basename(satfile)) - logger.info(f"Add {aerbcor.getnames()}") - - # open tar file for writing - with tarfile.open(aerostat, "w|gz") as archive: - for diagfile in diags: - diaggzip = f"{diagfile}.gz" - archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - logger.info(f'Saved diags to {aerostat}') + # Compress and tar diag files in COM directory + self.tar_diag_files(self.task_config.COMOUT_CHEM_ANALYSIS, + f"{self.task_config['APREFIX']}aerostat.tgz") + + # Tar radiative bias correction files into COM directory + self.tar_radiative_bias_corrections(self.task_config.COMOUT_CHEM_ANALYSIS, + f"{self.task_config.APREFIX}aero_varbc_params.tar") + + # Save files from COM + logger.info(f"Saving files to COM") + FileHandler(self.task_config.data_out).sync() def clean(self): super().clean() diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 27ccb61d438..18fbc514cf7 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -1,18 +1,14 @@ #!/usr/bin/env python3 -import os from logging import getLogger -from typing import List, Dict - -from wxflow import (AttrDict, FileHandler, - add_to_datetime, to_timedelta, - parse_j2yaml, logit, Task) +from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi +from wxflow import AttrDict, FileHandler, add_to_datetime, to_timedelta, parse_j2yaml, logit logger = getLogger(__name__.split('.')[-1]) -class AerosolBMatrix(Task): +class AerosolBMatrix(Analysis): """ Class for global aerosol BMatrix tasks """ @@ -40,32 +36,24 @@ def __init__(self, config): _res_anl = int(self.task_config['CASE_ANL'][1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - # fix ocnres - self.task_config.OCNRES = f"{self.task_config.OCNRES:03d}" - - # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( + # Extend task_config with variables repeatedly used across this class + self.task_config.update(AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config['LEVS'] - 1, - 'AERO_WINDOW_BEGIN': _window_begin, - 'AERO_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", - 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), - 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'aero_obsdatain_path': f"{self.task_config.DATA}/obs/", - 'aero_obsdataout_path': f"{self.task_config.DATA}/diags/", + 'npz': self.task_config.LEVS - 1, + 'BERROR_YAML': f'aero_background_error_static_{self.task_config.STATICB_TYPE}', + 'BERROR_DATA_DIR': f'{FIXgfs}/gdas/aero/clim_b', + 'AERO_BMATRIX_RESCALE_YAML': 'aero_gen_bmatrix_rescale_default.yaml.j2', } - ) + )) - # task_config is everything that this task should need - self.task_config = AttrDict(**self.task_config, **local_dict) + # Extend task_config with content of config yaml for this task + self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) # Create dictionary of Jedi objects expected_keys = ['aero_interpbkg', 'aero_diagb', 'aero_diffusion'] @@ -91,17 +79,12 @@ def initialize(self: Task) -> None: None """ - # stage fix files - logger.info(f"Staging JEDI fix files from {self.task_config.STAGE_JEDI_FIX_YAML}") - jedi_fix_list = parse_j2yaml(self.task_config.STAGE_JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_list).sync() - - # stage files from COM and create working directories - logger.info(f"Staging files from COM and creating working directories {self.task_config.STAGE_YAML}") - stage_dict = parse_j2yaml(self.task_config.STAGE_YAML, self.task_config) - FileHandler(stage_dict).sync() + # Stage files from COM + logger.info(f"Staging files from COM") + FileHandler(self.task_config.data_in).sync() # initialize JEDI applications + logger.info(f"Initializing JEDI applications") self.jedi_dict['aero_interpbkg'].initialize(self.task_config) self.jedi_dict['aero_diagb'].initialize(self.task_config) self.jedi_dict['aero_diffusion'].initialize(self.task_config) @@ -142,7 +125,7 @@ def finalize(self) -> None: - copying YAMLs to COM """ - # save files to COMOUT - logger.info(f"Saving files to COMOUT based on {self.task_config.SAVE_YAML}") - save_dict = parse_j2yaml(self.task_config.SAVE_YAML, self.task_config) - FileHandler(save_dict).sync() + + # Save files to COM + logger.info(f"Saving files to COM") + FileHandler(self.task_config.data_out).sync() diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 0f495473e73..ce01c146458 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -38,10 +38,11 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _next_cycle = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H")) - _iau_times_iso = [] + _iau_times = [] for hour in self.task_config.IAUFHRS: - _iau_times_iso.append(to_isotime(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2)) + _iau_times.append(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2) if 'OBS_LIST_YAML' in self.task_config: _observations = parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'] @@ -57,13 +58,15 @@ def __init__(self, config: Dict[str, Any]): { 'WINDOW_BEGIN': _window_begin, 'WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'next_cycle': _next_cycle, 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications 'OPREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX_ENS': f"enkf{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'iau_times_iso': _iau_times_iso, + 'OCNRES': f"{self.task_config.OCNRES:03d}", + 'iau_times': _iau_times, 'observations': _observations, 'bias_files': _bias_files, } From 74bc70690f583afc5d86d013342ab66714a86ead Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 26 Sep 2025 14:31:12 +0000 Subject: [PATCH 24/45] Debug --- dev/parm/config/gcafs/config.aeroanl.j2 | 4 +++- dev/parm/config/gcafs/config.aeroanlgenb | 7 +++---- dev/parm/config/gfs/config.aeroanl.j2 | 8 +++++++- dev/parm/config/gfs/config.aeroanlgenb | 6 +++--- jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX | 5 ++++- sorc/gdas.cd | 2 +- ush/python/pygfs/task/aero_analysis.py | 17 +++++++++++------ ush/python/pygfs/task/aero_bmatrix.py | 7 +++---- ush/python/pygfs/task/analysis.py | 16 +++++++++++++--- ush/python/pygfs/task/atm_analysis.py | 1 + ush/python/pygfs/task/atmens_analysis.py | 1 + 11 files changed, 50 insertions(+), 24 deletions(-) diff --git a/dev/parm/config/gcafs/config.aeroanl.j2 b/dev/parm/config/gcafs/config.aeroanl.j2 index ddfaca39576..fb4a3de649f 100644 --- a/dev/parm/config/gcafs/config.aeroanl.j2 +++ b/dev/parm/config/gcafs/config.aeroanl.j2 @@ -21,11 +21,13 @@ export CASE_ANL export STATICB_TYPE='diffusion' -export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_bmat_config.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_det_config.yaml.j2" export OBS_LIST_YAML="${PARMgfs}/gdas/aero/aero_obs_list.yaml.j2" export BIAS_FILES_YAML="${PARMgfs}/gdas/aero/aero_bias_files.yaml.j2" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" +export aero_bkg_times="3,6,9" + echo "END: config.aeroanl" diff --git a/dev/parm/config/gcafs/config.aeroanlgenb b/dev/parm/config/gcafs/config.aeroanlgenb index fc4fd2243fd..d9c8dbc2862 100644 --- a/dev/parm/config/gcafs/config.aeroanlgenb +++ b/dev/parm/config/gcafs/config.aeroanlgenb @@ -8,11 +8,10 @@ echo "BEGIN: config.aeroanlgenb" # Get task specific resources source "${EXPDIR}/config.resources" aeroanlgenb -export JEDI_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_bmat_jedi_config.yaml.j2" -export STAGE_YAML="${PARMgfs}/gdas/aero/aero_bmat_stage.yaml.j2" -export SAVE_YAML="${PARMgfs}/gdas/aero/aero_bmat_save.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_bmat_config.yaml.j2" +export OBS_LIST_YAML="${PARMgfs}/gdas/aero/aero_obs_list.yaml.j2" +export BIAS_FILES_YAML="${PARMgfs}/gdas/aero/aero_bias_files.yaml.j2" -export RESCALE_YAML="${PARMgfs}/gdas/jcb-gdas/aero/algorithm/aero_gen_bmatrix_rescale_default.yaml.j2" export aero_diffusion_iter=200 export aero_diffusion_horiz_len=300e3 export aero_diffusion_fixed_val=20.0 diff --git a/dev/parm/config/gfs/config.aeroanl.j2 b/dev/parm/config/gfs/config.aeroanl.j2 index ddfaca39576..ae4efd8bcc4 100644 --- a/dev/parm/config/gfs/config.aeroanl.j2 +++ b/dev/parm/config/gfs/config.aeroanl.j2 @@ -21,11 +21,17 @@ export CASE_ANL export STATICB_TYPE='diffusion' -export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_bmat_config.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_det_config.yaml.j2" export OBS_LIST_YAML="${PARMgfs}/gdas/aero/aero_obs_list.yaml.j2" export BIAS_FILES_YAML="${PARMgfs}/gdas/aero/aero_bias_files.yaml.j2" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" +if [[ "${DOIAU}" == "YES" ]]; then + export aero_bkg_times="3,6,9" +else + export aero_bkg_times="6," # Trailing comma is necessary so this is treated as a list +fi + echo "END: config.aeroanl" diff --git a/dev/parm/config/gfs/config.aeroanlgenb b/dev/parm/config/gfs/config.aeroanlgenb index 7fadfe33da2..d9c8dbc2862 100644 --- a/dev/parm/config/gfs/config.aeroanlgenb +++ b/dev/parm/config/gfs/config.aeroanlgenb @@ -8,9 +8,9 @@ echo "BEGIN: config.aeroanlgenb" # Get task specific resources source "${EXPDIR}/config.resources" aeroanlgenb -export JEDI_CONFIG_YAML"${PARMgfs}/gdas/aero/aero_bmat_jedi_config.yaml.j2" -export STAGE_YAML"${PARMgfs}/gdas/aero/aero_bmat_stage.yaml.j2" -export SAVE_YAML"${PARMgfs}/gdas/aero/aero_bmat_save.yaml.j2" +export TASK_CONFIG_YAML="${PARMgfs}/gdas/aero/aero_bmat_config.yaml.j2" +export OBS_LIST_YAML="${PARMgfs}/gdas/aero/aero_obs_list.yaml.j2" +export BIAS_FILES_YAML="${PARMgfs}/gdas/aero/aero_bias_files.yaml.j2" export aero_diffusion_iter=200 export aero_diffusion_horiz_len=300e3 diff --git a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX index 2f73504b9af..971f66d3782 100755 --- a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX +++ b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @@ -13,9 +13,12 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanl # Generate COM variables from templates YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ - COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \ + COMOUT_CONF:COM_CONF_TMPL + mkdir -p "${COMOUT_CHEM_BMAT}" +mkdir -p "${COMOUT_CONF}" ############################################################### # Run relevant script diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 081a825c27b..9a78290fe9c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 081a825c27b5f1ff101a1eab950bdb17a8c3505d +Subproject commit 9a78290fe9cd8d029418b04ab2697fc9ed59c75f diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index cc663e3bb86..2010e04ac14 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -4,12 +4,12 @@ from logging import getLogger from netCDF4 import Dataset from typing import Dict, List +from pygfs.task.analysis import Analysis from pygfs.jedi import Jedi from wxflow import ( AttrDict, FileHandler, - to_fv3time, - Task, + to_fv3time, to_timedelta, YAMLFile, parse_j2yaml, logit ) @@ -18,7 +18,7 @@ logger = getLogger(__name__.split('.')[-1]) -class AerosolAnalysis(Task): +class AerosolAnalysis(Analysis): """ Class for JEDI-based global aerosol analysis tasks """ @@ -50,6 +50,10 @@ def __init__(self, config): else: _anl_time = self.task_config.current_cycle + _bkg_times = [] + for hour in self.task_config.aero_bkg_times: + _bkg_times.append(self.task_config.WINDOW_BEGIN + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H")/2 ) + # Extend task_config with variables repeatedly used across this class self.task_config.update(AttrDict( { @@ -62,17 +66,18 @@ def __init__(self, config): 'npz': self.task_config.LEVS - 1, 'BKG_TSTEP': "PT3H", # FGAT 'BERROR_YAML': f'aero_background_error_static_{self.task_config.STATICB_TYPE}', + 'AERO_BMATRIX_RESCALE_YAML': 'aero_gen_bmatrix_rescale_default.yaml.j2', 'anl_time': _anl_time, + 'bkg_times': _bkg_times, } - ) + )) # Extend task_config with content of config yaml for this task self.task_config.update(parse_j2yaml(self.task_config.TASK_CONFIG_YAML, self.task_config)) # Create dictionary of Jedi objects expected_keys = ['aeroanlvar'] - jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, expected_keys) @logit(logger) def initialize(self) -> None: diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 18fbc514cf7..651b9eb863a 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -47,7 +47,7 @@ def __init__(self, config): 'npz_anl': self.task_config['LEVS'] - 1, 'npz': self.task_config.LEVS - 1, 'BERROR_YAML': f'aero_background_error_static_{self.task_config.STATICB_TYPE}', - 'BERROR_DATA_DIR': f'{FIXgfs}/gdas/aero/clim_b', + 'BERROR_DATA_DIR': f'{self.task_config.FIXgfs}/gdas/aero/clim_b', 'AERO_BMATRIX_RESCALE_YAML': 'aero_gen_bmatrix_rescale_default.yaml.j2', } )) @@ -57,11 +57,10 @@ def __init__(self, config): # Create dictionary of Jedi objects expected_keys = ['aero_interpbkg', 'aero_diagb', 'aero_diffusion'] - jedi_config_dict = parse_j2yaml(self.task_config.JEDI_CONFIG_YAML, self.task_config) - self.jedi_dict = Jedi.get_jedi_dict(jedi_config_dict, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.jedi_config, self.task_config, expected_keys) @logit(logger) - def initialize(self: Task) -> None: + def initialize(self) -> None: """Initialize a global aerosol B-matrix This method will initialize a global aerosol B-Matrix. diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index ce01c146458..230110aeff4 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -37,34 +37,44 @@ def __init__(self, config: Dict[str, Any]): """ super().__init__(config) + # Get assimilation window times _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) _next_cycle = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H")) + # Get specific assimilation times within the assimulation window _iau_times = [] for hour in self.task_config.IAUFHRS: _iau_times.append(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2) + # Get observations list from obs list yaml if 'OBS_LIST_YAML' in self.task_config: _observations = parse_j2yaml(self.task_config.OBS_LIST_YAML, self.task_config)['observations'] else: _observations = [] + + # Get bias correction dict from bias files yaml if 'BIAS_FILES_YAML' in self.task_config: _bias_files = parse_j2yaml(self.task_config.BIAS_FILES_YAML, self.task_config)['bias_files'] else: _bias_files = AttrDict + # Set prefix needed for GPREFIX, depedning on the model + if self.task_config.NET == 'gcafs': + _da_prefix = 'gcdas' + else: + _da_prefix = 'gdas' + # Extend task_config with variables that are repeatedly used across this class self.task_config.update(AttrDict( { 'WINDOW_BEGIN': _window_begin, 'WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", 'next_cycle': _next_cycle, - 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications 'OPREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX': f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", 'APREFIX_ENS': f"enkf{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'GPREFIX_ENS': f"enkfgdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'GPREFIX': f"{_da_prefix}.t{self.task_config.previous_cycle.hour:02d}z.", + 'GPREFIX_ENS': f"enkf{_da_prefix}.t{self.task_config.previous_cycle.hour:02d}z.", 'OCNRES': f"{self.task_config.OCNRES:03d}", 'iau_times': _iau_times, 'observations': _observations, diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 8c7aefd3254..2d01d6b0627 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -55,6 +55,7 @@ def __init__(self, config: Dict[str, Any]): 'npy_anl': _res_anl + 1, 'npz_anl': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications 'BERROR_YAML': _BERROR_YAML, } )) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 7187af21e01..e1cdceeddcc 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -42,6 +42,7 @@ def __init__(self, config: Dict[str, Any]): 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, + 'BKG_TSTEP': "PT1H", # Placeholder for 4D applications }) ) From 5d991a347a9da5823f83d40720c8bc962b54b618 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 26 Sep 2025 15:05:59 +0000 Subject: [PATCH 25/45] Revert some stuff --- dev/parm/config/gfs/config.atmanl.j2 | 8 ++++++++ dev/parm/config/gfs/config.ecen_fv3jedi | 6 ++++++ ush/python/pygfs/task/atm_analysis.py | 8 ++------ ush/python/pygfs/task/ensemble_recenter.py | 2 +- ush/python/pygfs/task/fv3_analysis_calc.py | 5 +---- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/dev/parm/config/gfs/config.atmanl.j2 b/dev/parm/config/gfs/config.atmanl.j2 index 35f3833e3a6..bbec1e3817e 100644 --- a/dev/parm/config/gfs/config.atmanl.j2 +++ b/dev/parm/config/gfs/config.atmanl.j2 @@ -18,4 +18,12 @@ export FV3INC_JEDI_TEST_YAML="{{ FV3INC_JEDI_TEST_YAML }}" export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_det_config.yaml.j2" export BIAS_FILES_YAML="${PARMgfs}/gdas/atm/atm_bias_files.yaml.j2" +export LOCALIZATION_TYPE="bump" + +if [[ ${DOHYBVAR} = "YES" ]]; then + export CASE_ANL=${CASE_ENS} +else + export CASE_ANL=${CASE} +fi + echo "END: config.atmanl" diff --git a/dev/parm/config/gfs/config.ecen_fv3jedi b/dev/parm/config/gfs/config.ecen_fv3jedi index 4c121e4f317..768140b2654 100644 --- a/dev/parm/config/gfs/config.ecen_fv3jedi +++ b/dev/parm/config/gfs/config.ecen_fv3jedi @@ -16,4 +16,10 @@ source "${EXPDIR}/config.resources" ecen_fv3jedi export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ecen_config.yaml.j2" +if [[ ${DOHYBVAR} = "YES" ]]; then + export CASE_ANL=${CASE_ENS} +else + export CASE_ANL=${CASE} +fi + echo "END: config.ecen_fv3jedi" diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 2d01d6b0627..aa9be4b3756 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -34,14 +34,10 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) _res = int(self.task_config.CASE[1:]) - if self.task_config.DOHYBVAR: - _res_anl = int(self.task_config.CASE_ENS[1:]) - else: - _res_anl = int(self.task_config.CASE[1:]) + _res_anl = int(self.task_config.CASE_ANL[1:]) - _localization_type = 'bump' if self.task_config.DOHYBVAR: - _BERROR_YAML = f"atmosphere_background_error_hybrid_{self.task_config.STATICB_TYPE}_{_localization_type}" + _BERROR_YAML = f"atmosphere_background_error_hybrid_{self.task_config.STATICB_TYPE}_{self.task_config.LOCALIZATION_TYPE}" else: _BERROR_YAML = f"atmosphere_background_error_static_{self.task_config.STATICB_TYPE}" diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index 8151fb93756..debb831f9f9 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -34,7 +34,7 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) _res = int(self.task_config.CASE[1:]) - _res_anl = int(self.task_config.CASE_ENS[1:]) + _res_anl = int(self.task_config.CASE_ANL[1:]) # Create a local dictionary that is repeatedly used across this class self.task_config.update(AttrDict( diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index 57d54f7c321..572bf765f28 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -37,10 +37,7 @@ def __init__(self, config: Dict[str, Any]): super().__init__(config) _res = int(self.task_config.CASE[1:]) - if self.task_config.DOHYBVAR: - _res_anl = int(self.task_config.CASE_ENS[1:]) - else: - _res_anl = int(self.task_config.CASE[1:]) + _res_anl = int(self.task_config['CASE_ANL'][1:]) # Create a local dictionary that is repeatedly used across this class self.task_config.update(AttrDict( From e5b37d1f1874bf3a09c0a665acca3934ffc8e253 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 26 Sep 2025 15:16:16 +0000 Subject: [PATCH 26/45] pynorms --- ush/python/pygfs/task/aero_analysis.py | 2 +- ush/python/pygfs/task/analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 2010e04ac14..b3ec22b13b6 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -52,7 +52,7 @@ def __init__(self, config): _bkg_times = [] for hour in self.task_config.aero_bkg_times: - _bkg_times.append(self.task_config.WINDOW_BEGIN + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H")/2 ) + _bkg_times.append( self.task_config.WINDOW_BEGIN + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2 ) # Extend task_config with variables repeatedly used across this class self.task_config.update(AttrDict( diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 230110aeff4..e07d3b75442 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -206,4 +206,4 @@ def extract_tar(tar_file: str) -> None: tarball.extractall(path=tar_path) logger.info(f"Extract {tarball.getnames()}") except Exception as e: - raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e \ No newline at end of file + raise WorkflowException(f"An error occurred while extracting {tar_file}:\n{e}") from e From 0e82fc51190c7e4cb2f947c76191b8a9353706e9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 26 Sep 2025 15:19:47 +0000 Subject: [PATCH 27/45] pynorms and shellnorms --- dev/parm/config/gfs/config.atmanl.j2 | 1 + ush/python/pygfs/task/aero_analysis.py | 2 +- ush/python/pygfs/task/analysis.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dev/parm/config/gfs/config.atmanl.j2 b/dev/parm/config/gfs/config.atmanl.j2 index bbec1e3817e..a2ac0e99a77 100644 --- a/dev/parm/config/gfs/config.atmanl.j2 +++ b/dev/parm/config/gfs/config.atmanl.j2 @@ -21,6 +21,7 @@ export BIAS_FILES_YAML="${PARMgfs}/gdas/atm/atm_bias_files.yaml.j2" export LOCALIZATION_TYPE="bump" if [[ ${DOHYBVAR} = "YES" ]]; then + # shellcheck disable=SC2153 export CASE_ANL=${CASE_ENS} else export CASE_ANL=${CASE} diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index b3ec22b13b6..51ee9b83873 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -52,7 +52,7 @@ def __init__(self, config): _bkg_times = [] for hour in self.task_config.aero_bkg_times: - _bkg_times.append( self.task_config.WINDOW_BEGIN + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2 ) + _bkg_times.append(self.task_config.WINDOW_BEGIN + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2) # Extend task_config with variables repeatedly used across this class self.task_config.update(AttrDict( diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index e07d3b75442..41054869b03 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -98,7 +98,7 @@ def clean(self) -> None: def untar_bias_corrections(self) -> None: """Extract bias correction files from tarballs This method will extract bias correction files from tarballs - + Parameters ---------- None From 77e567ce299be43b9455c2879ed46ffb5a4528cd Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 26 Sep 2025 19:09:13 +0000 Subject: [PATCH 28/45] debug --- dev/parm/config/gfs/config.atmanl.j2 | 5 ++--- dev/parm/config/gfs/config.atmensanl.j2 | 9 ++++----- sorc/gdas.cd | 2 +- ush/python/pygfs/task/analysis.py | 6 +++--- ush/python/pygfs/task/snow_analysis.py | 1 - ush/python/pygfs/task/snowens_analysis.py | 1 - 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/dev/parm/config/gfs/config.atmanl.j2 b/dev/parm/config/gfs/config.atmanl.j2 index a2ac0e99a77..d0e0e8d0f88 100644 --- a/dev/parm/config/gfs/config.atmanl.j2 +++ b/dev/parm/config/gfs/config.atmanl.j2 @@ -6,15 +6,14 @@ echo "BEGIN: config.atmanl" export OBS_LIST_YAML="{{ OBS_LIST_YAML }}" +export VAR_JEDI_TEST_YAML="{{ VAR_JEDI_TEST_YAML }}" +export FV3INC_JEDI_TEST_YAML="{{ FV3INC_JEDI_TEST_YAML }}" export STATICB_TYPE="{{ STATICB_TYPE }}" export layout_x_atmanl="{{ LAYOUT_X_ATMANL }}" export layout_y_atmanl="{{ LAYOUT_Y_ATMANL }}" export io_layout_x="{{ IO_LAYOUT_X }}" export io_layout_y="{{ IO_LAYOUT_Y }}" -export VAR_JEDI_TEST_YAML="{{ VAR_JEDI_TEST_YAML }}" -export FV3INC_JEDI_TEST_YAML="{{ FV3INC_JEDI_TEST_YAML }}" - export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_det_config.yaml.j2" export BIAS_FILES_YAML="${PARMgfs}/gdas/atm/atm_bias_files.yaml.j2" diff --git a/dev/parm/config/gfs/config.atmensanl.j2 b/dev/parm/config/gfs/config.atmensanl.j2 index 77544ca444b..d6b869151ab 100644 --- a/dev/parm/config/gfs/config.atmensanl.j2 +++ b/dev/parm/config/gfs/config.atmensanl.j2 @@ -6,15 +6,14 @@ echo "BEGIN: config.atmensanl" export OBS_LIST_YAML="{{ OBS_LIST_YAML }}" -export layout_x_atmensanl="{{ LAYOUT_X_ATMENSANL }}" -export layout_y_atmensanl="{{ LAYOUT_Y_ATMENSANL }}" -export io_layout_x="{{ IO_LAYOUT_X }}" -export io_layout_y="{{ IO_LAYOUT_Y }}" - export LETKF_JEDI_TEST_YAML="{{ LETKF_JEDI_TEST_YAML }}" export OBS_JEDI_TEST_YAML="{{ OBS_JEDI_TEST_YAML }}" export SOL_JEDI_TEST_YAML="{{ SOL_JEDI_TEST_YAML }}" export FV3INC_JEDI_TEST_YAML="{{ FV3INC_JEDI_TEST_YAML }}" +export layout_x_atmensanl="{{ LAYOUT_X_ATMENSANL }}" +export layout_y_atmensanl="{{ LAYOUT_Y_ATMENSANL }}" +export io_layout_x="{{ IO_LAYOUT_X }}" +export io_layout_y="{{ IO_LAYOUT_Y }}" export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ens_config.yaml.j2" export BIAS_FILES_YAML="${PARMgfs}/gdas/atm/atm_bias_files.yaml.j2" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 9a78290fe9c..567dad0b169 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 9a78290fe9cd8d029418b04ab2697fc9ed59c75f +Subproject commit 567dad0b1692b4b05ffdd1aea9b374ee50df10bb diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 41054869b03..ae6eb831593 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -42,9 +42,9 @@ def __init__(self, config: Dict[str, Any]): _next_cycle = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H")) # Get specific assimilation times within the assimulation window - _iau_times = [] + _iau_times_iso = [] for hour in self.task_config.IAUFHRS: - _iau_times.append(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _iau_times_iso.append(to_isotime(_window_begin + to_timedelta(f"{str(hour)}H") - to_timedelta(f"{self.task_config.assim_freq}H") / 2)) # Get observations list from obs list yaml if 'OBS_LIST_YAML' in self.task_config: @@ -76,7 +76,7 @@ def __init__(self, config: Dict[str, Any]): 'GPREFIX': f"{_da_prefix}.t{self.task_config.previous_cycle.hour:02d}z.", 'GPREFIX_ENS': f"enkf{_da_prefix}.t{self.task_config.previous_cycle.hour:02d}z.", 'OCNRES': f"{self.task_config.OCNRES:03d}", - 'iau_times': _iau_times, + 'iau_times_iso': _iau_times_iso, 'observations': _observations, 'bias_files': _bias_files, } diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index b9f9c895008..28b7fb44dc3 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -56,7 +56,6 @@ def __init__(self, config: Dict[str, Any]): 'npy_ges': _res + 1, 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, - 'OCNRES': f"{self.task_config.OCNRES:03d}", 'snow_bkg_path': os.path.join('.', 'bkg/'), } )) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index b5016f71162..8d38a1ef2ab 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -62,7 +62,6 @@ def __init__(self, config: Dict[str, Any]): 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, 'CASE': self.task_config.CASE_ENS, - 'OCNRES': f"{self.task_config.OCNRES:03d}", 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), } )) From e6caa8630c31b6e6309a98e7137155978b72ec31 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 26 Sep 2025 19:23:20 +0000 Subject: [PATCH 29/45] debug --- ush/python/pygfs/task/atmens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index e1cdceeddcc..82be16d6759 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -144,7 +144,7 @@ def finalize(self) -> None: # Compress and tar diag files in COM directory self.tar_diag_files(self.task_config.COMOUT_ATMOS_ANALYSIS_ENS, - f"{self.task_config.APREFIX_ENS}atmstat") + f"{self.task_config.APREFIX_ENS}atmensstat") # Save files from COM logger.info(f"Saving files to COM") From 3efc23a1c3513d7a6f68881d1dd9fb6e207199ea Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 29 Sep 2025 14:27:31 +0000 Subject: [PATCH 30/45] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 567dad0b169..82f729dae1e 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 567dad0b1692b4b05ffdd1aea9b374ee50df10bb +Subproject commit 82f729dae1ebdb1ffbc4b5b2c83df1ece7d2b8f1 From 82b77a5f80ab7d2b47de5a5fbc1b8188350aee86 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 29 Sep 2025 15:10:28 +0000 Subject: [PATCH 31/45] update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 82f729dae1e..7397b5474fa 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 82f729dae1ebdb1ffbc4b5b2c83df1ece7d2b8f1 +Subproject commit 7397b5474fa8448bb7631b22b34d6b8f3c54b369 From 60b377097c61a6b21bf91f53e61ba6afe6586d1a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 30 Sep 2025 20:40:10 +0000 Subject: [PATCH 32/45] bugfixes --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/snow_analysis.py | 7 ++++--- ush/python/pygfs/task/snowens_analysis.py | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 7397b5474fa..2000d14fc99 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 7397b5474fa8448bb7631b22b34d6b8f3c54b369 +Subproject commit 2000d14fc99989fc56fa809ebdb08e8ed0a90a7f diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index ae6eb831593..0141260b53d 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -180,7 +180,7 @@ def tar_radiative_bias_corrections(self, comout: str, tarball_name: str) -> None radbcor.add(satfile, arcname=os.path.basename(satfile)) for tlapfile in tlaplist: # Change OPREFIX to APREFIX in tlapse file name when adding to tarball - radbcor.add(tlapfile, arcname=os.path.basename(tlapfile.replace(self.task_config.OPREFIX, self.task_config.APREFIX))) + radbcor.add(tlapfile, arcname=os.path.basename(tlapfile.replace(self.task_config.GPREFIX, self.task_config.APREFIX))) @logit(logger) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index c61568ac870..f3941c7262d 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -51,8 +51,9 @@ def __init__(self, config: Dict[str, Any]): # if 00z, do SCF preprocessing _ims_file = os.path.join(self.task_config.FIXgfs, 'gdas', 'obs', 'ims', - f'IMS_4km_to_{self.task_config.CASE}.mx{self.task_config.OCNRES}.nc'), - if task_config.cyc == 0 and os.path.exists(_ims_file): + f'IMS_4km_to_{self.task_config.CASE}.mx{self.task_config.OCNRES}.nc') + logger.info(f"Checking for IMS file: {_ims_file}") + if self.task_config.cyc == 0 and os.path.exists(_ims_file): _DO_IMS_SCF = True else: _DO_IMS_SCF = False @@ -65,7 +66,7 @@ def __init__(self, config: Dict[str, Any]): 'npz_ges': self.task_config.LEVS - 1, 'npz': self.task_config.LEVS - 1, 'snow_bkg_path': os.path.join('.', 'bkg/'), - _ims_file: _ims_file, + 'ims_file': _ims_file, 'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done } )) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index cc5fa58dc0e..09b802a567a 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -54,8 +54,8 @@ def __init__(self, config: Dict[str, Any]): # if 00z, do SCF preprocessing _ims_file = os.path.join(self.task_config.FIXgfs, 'gdas', 'obs', 'ims', - f'IMS_4km_to_{self.task_config.CASE}.mx{self.task_config.OCNRES}.nc'), - if task_config.cyc == 0 and os.path.exists(_ims_file): + f'IMS_4km_to_{self.task_config.CASE}.mx{self.task_config.OCNRES}.nc') + if self.task_config.cyc == 0 and os.path.exists(_ims_file): _DO_IMS_SCF = True else: _DO_IMS_SCF = False From 2f54294166aed805af479383bd351c50d3b9de19 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 30 Sep 2025 20:42:48 +0000 Subject: [PATCH 33/45] Make no JEDI obs a warning, not an exception --- ush/python/pygfs/jedi/jedi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/jedi/jedi.py b/ush/python/pygfs/jedi/jedi.py index 7c627198760..2cb1d5a1aff 100644 --- a/ush/python/pygfs/jedi/jedi.py +++ b/ush/python/pygfs/jedi/jedi.py @@ -263,9 +263,9 @@ def clean_empty_obsspaces(self): observers.clear() observers.extend(cleaned_observers) - # If no observers left in list, raise error + # Warn if no observers left in list if observers == []: - raise WorkflowException(f"No observers found in JEDI input config") + logger.warning(f"No observers found in JEDI input config") @staticmethod @logit(logger) From b2284ab770308f1e9028ac717543a0887ec25b8d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 1 Oct 2025 13:10:55 +0000 Subject: [PATCH 34/45] Update comments --- ush/python/pygfs/task/aero_analysis.py | 20 +++++++------------- ush/python/pygfs/task/aero_bmatrix.py | 8 +++----- ush/python/pygfs/task/atm_analysis.py | 17 ++++++----------- ush/python/pygfs/task/atmens_analysis.py | 14 +++++--------- ush/python/pygfs/task/ensemble_recenter.py | 7 +++---- ush/python/pygfs/task/fv3_analysis_calc.py | 8 ++++---- ush/python/pygfs/task/snow_analysis.py | 16 +++++----------- ush/python/pygfs/task/snowens_analysis.py | 14 ++++---------- 8 files changed, 37 insertions(+), 67 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 51ee9b83873..d5408e34e68 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -85,14 +85,9 @@ def initialize(self) -> None: This method will initialize a global aerosol analysis using JEDI. This includes: - - initialize JEDI applications - - staging observation files - - staging bias correction files - - staging CRTM fix files - - staging FV3-JEDI fix files - - staging B error files - - staging model backgrounds - - creating output directories + - stage input files from COM and create output directories + - extract bias corrections from tar files + - initialize JEDI application """ # Stage files from COM @@ -129,11 +124,10 @@ def finalize(self) -> None: This method will finalize a global aerosol analysis using JEDI. This includes: - - tarring up output diag files and place in ROTDIR - - copying the generated YAML file from initialize to the ROTDIR - - copying the guess files to the ROTDIR - - applying the increments to the original RESTART files - - moving the increment files to the ROTDIR + - apply increments to the original RESTART files + - compress and tar output diag files in COM + - tar radiative bias correction files in COM + - save output files and YAMLs to COM """ diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 651b9eb863a..511b7dcdcf5 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -65,9 +65,8 @@ def initialize(self) -> None: This method will initialize a global aerosol B-Matrix. This includes: - - staging the determinstic backgrounds - - staging fix files - - initializing the JEDI applications + - stage input files from COM and create output directories + - initialize JEDI applications Parameters ---------- @@ -120,8 +119,7 @@ def finalize(self) -> None: This method will finalize a global aerosol bmatrix using JEDI. This includes: - - copying the bmatrix files to COM - - copying YAMLs to COM + - save output files and YAMLs to COM """ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index aa9be4b3756..0ea0724ef03 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -69,14 +69,9 @@ def initialize(self) -> None: This method will initialize a global atm analysis. This includes: + - stage input files from COM and create output directories + - extract bias corrections from tar files - initialize JEDI applications - - staging observation files - - staging bias correction files - - staging CRTM fix files - - staging FV3-JEDI fix files - - staging B error files - - staging model backgrounds - - creating output directories Parameters ---------- @@ -88,7 +83,7 @@ def initialize(self) -> None: """ # Stage files from COM - logger.info(f"Staging files from COM") + logger.info(f"Staging files from COM and creating output directories") FileHandler(self.task_config.data_in).sync() # Extract bias corrections from tar files @@ -122,9 +117,9 @@ def finalize(self) -> None: This method will finalize a global atm analysis using JEDI. This includes: - - tar output diag files and place in ROTDIR - - copy the generated YAML file from initialize to the ROTDIR - - copy the updated bias correction files to ROTDIR + - compress and tar output diag files in COM + - tar radiative bias correction files and place in COM + - save output files and YAMLs to COM Parameters ---------- diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 82be16d6759..778cfaea8a3 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -59,13 +59,9 @@ def initialize(self) -> None: This method will initialize a global atmens analysis. This includes: - - initialize JEDI LETKF observer and FV3 increment converter applications - - staging observation files - - staging bias correction files - - staging CRTM fix files - - staging FV3-JEDI fix files - - staging model backgrounds - - creating output directories + - stage input files from COM and create output directories + - extract bias corrections from tar files + - initialize JEDI applications Parameters ---------- @@ -130,8 +126,8 @@ def finalize(self) -> None: This method will finalize a global atmens analysis using JEDI. This includes: - - tar output diag files and place in ROTDIR - - copy the generated YAML file from initialize to the ROTDIR + - compress and tar output diag files and place in COM + - save output files and YAMLs to COM Parameters ---------- diff --git a/ush/python/pygfs/task/ensemble_recenter.py b/ush/python/pygfs/task/ensemble_recenter.py index debb831f9f9..54ba1d12339 100644 --- a/ush/python/pygfs/task/ensemble_recenter.py +++ b/ush/python/pygfs/task/ensemble_recenter.py @@ -62,9 +62,8 @@ def initialize(self) -> None: This method will initialize the ensemble increment recentering task. This includes: - - initializing the JEDI recentering application - - staging JEDI fix files - - staging backgrounds and increments + - stage input files from COM and create output directories + - initialize JEDI applications Parameters ---------- @@ -111,7 +110,7 @@ def finalize(self) -> None: This method will finalize the ensemble increment recentering task. This includes: - - Move correction increment files to the comrot directory + - save output files and YAMLs to COM Parameters ---------- diff --git a/ush/python/pygfs/task/fv3_analysis_calc.py b/ush/python/pygfs/task/fv3_analysis_calc.py index 572bf765f28..5dc5000ed51 100644 --- a/ush/python/pygfs/task/fv3_analysis_calc.py +++ b/ush/python/pygfs/task/fv3_analysis_calc.py @@ -69,9 +69,8 @@ def initialize(self) -> None: This method will initialize the analysis calculation task. This includes: - - initializing the JEDI addincrement application - - staging JEDI fix files - - staging backgrounds and increments + - stage input files from COM and create output directories + - initialize JEDI applications Parameters ---------- @@ -149,7 +148,8 @@ def finalize(self) -> None: This method will finalize the analysis calculation task. This includes: - - Move analysis files to the comrot directory + - write analysis log file + - save output files and YAMLs to COM Parameters ---------- diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index f3941c7262d..3a106d62f6d 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -84,12 +84,8 @@ def initialize(self) -> None: This method will initialize a global snow analysis. This includes: - - initialize JEDI application - - staging model backgrounds - - staging observation files - - staging FV3-JEDI fix files - - staging B error files - - creating output directories + - stage input files from COM and create output directories + - initialize JEDI applications Parameters ---------- @@ -101,7 +97,7 @@ def initialize(self) -> None: """ # Stage files from COM - logger.info(f"Staging files from COM") + logger.info(f"Staging files from COM and creating output directories") FileHandler(self.task_config.data_in).sync() # initialize JEDI variational application @@ -132,10 +128,8 @@ def execute(self, jedi_dict_key: str) -> None: def finalize(self) -> None: """Performs closing actions of the Snow analysis task This method: - - tar and gzip the output diag files and place in COM/ - - copy the generated YAML file from initialize to the COM/ - - copy the analysis files to the COM/ - - copy the increment files to the COM/ + - compress and tar output diag files in COM + - save output files and YAMLs to COM Parameters ---------- diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 09b802a567a..32b3cc27cc8 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -86,12 +86,8 @@ def initialize(self) -> None: This method will initialize a global snow ensemble analysis. This includes: + - stage input files from COM and create output directories - initialize JEDI applications - - staging model backgrounds - - staging observation files - - staging FV3-JEDI fix files - - staging B error files - - creating output directories Parameters ---------- @@ -103,7 +99,7 @@ def initialize(self) -> None: """ # Stage files from COM - logger.info(f"Staging files from COM") + logger.info(f"Staging files from COM and creating output directories") FileHandler(self.task_config.data_in).sync() # Initialize JEDI applications @@ -135,10 +131,8 @@ def execute(self, jedi_dict_key: str) -> None: def finalize(self) -> None: """Performs closing actions of the Snow analysis task This method: - - tar and gzip the output diag files and place in COM/ - - copy the generated YAML file from initialize to the COM/ - - copy the analysis files to the COM/ - - copy the increment files to the COM/ + - compress and tar output diag files in COM + - save output files and YAMLs to COM Parameters ---------- From d894cab342589d03cf8b17abf0bb6219e68757b8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 1 Oct 2025 13:59:48 +0000 Subject: [PATCH 35/45] update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 2000d14fc99..8db3630d7cc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 2000d14fc99989fc56fa809ebdb08e8ed0a90a7f +Subproject commit 8db3630d7cc84c1536cc901b774f97cb61364ab7 From 22eb7691eb173c58cb5325267c82963c1b3ed055 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 1 Oct 2025 14:03:23 +0000 Subject: [PATCH 36/45] pynorms --- ush/python/pygfs/task/snow_analysis.py | 2 +- ush/python/pygfs/task/snowens_analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 3a106d62f6d..f7f66194624 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -67,7 +67,7 @@ def __init__(self, config: Dict[str, Any]): 'npz': self.task_config.LEVS - 1, 'snow_bkg_path': os.path.join('.', 'bkg/'), 'ims_file': _ims_file, - 'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done + 'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done } )) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 32b3cc27cc8..f9a4267a45a 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -69,7 +69,7 @@ def __init__(self, config: Dict[str, Any]): 'npz': self.task_config.LEVS - 1, 'CASE': self.task_config.CASE_ENS, 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), - 'DO_IMS_SCF': False, # Boolean to decide if IMS snow cover processing is done + 'DO_IMS_SCF': False, # Boolean to decide if IMS snow cover processing is done } )) From 5b6caf15a3997d84a554f65711d324e5080a77ad Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 3 Oct 2025 13:38:45 +0000 Subject: [PATCH 37/45] Bugfix --- sorc/gdas.cd | 2 +- ush/python/pygfs/task/snow_analysis.py | 3 +-- ush/python/pygfs/task/snowens_analysis.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 8db3630d7cc..ef3ba848711 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 8db3630d7cc84c1536cc901b774f97cb61364ab7 +Subproject commit ef3ba848711c1b5750cfa38d3b5fd91f6d4caac3 diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index f7f66194624..c9bb3ad778b 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -50,8 +50,7 @@ def __init__(self, config: Dict[str, Any]): _res = int(self.task_config['CASE'][1:]) # if 00z, do SCF preprocessing - _ims_file = os.path.join(self.task_config.FIXgfs, 'gdas', 'obs', 'ims', - f'IMS_4km_to_{self.task_config.CASE}.mx{self.task_config.OCNRES}.nc') + _ims_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc') logger.info(f"Checking for IMS file: {_ims_file}") if self.task_config.cyc == 0 and os.path.exists(_ims_file): _DO_IMS_SCF = True diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index f9a4267a45a..7de3d33b2ba 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -53,8 +53,7 @@ def __init__(self, config: Dict[str, Any]): _res = int(self.task_config['CASE_ENS'][1:]) # if 00z, do SCF preprocessing - _ims_file = os.path.join(self.task_config.FIXgfs, 'gdas', 'obs', 'ims', - f'IMS_4km_to_{self.task_config.CASE}.mx{self.task_config.OCNRES}.nc') + _ims_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc') if self.task_config.cyc == 0 and os.path.exists(_ims_file): _DO_IMS_SCF = True else: @@ -69,7 +68,8 @@ def __init__(self, config: Dict[str, Any]): 'npz': self.task_config.LEVS - 1, 'CASE': self.task_config.CASE_ENS, 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), - 'DO_IMS_SCF': False, # Boolean to decide if IMS snow cover processing is done + 'ims_file': _ims_file, + 'DO_IMS_SCF': _DO_IMS_SCF , # Boolean to decide if IMS snow cover processing is done } )) From f1185c873ac0b4b0b67def534ee953e3e6d62a2e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 3 Oct 2025 13:42:53 +0000 Subject: [PATCH 38/45] pynorms --- ush/python/pygfs/task/snowens_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/snowens_analysis.py b/ush/python/pygfs/task/snowens_analysis.py index 7de3d33b2ba..7a9d0c7acff 100644 --- a/ush/python/pygfs/task/snowens_analysis.py +++ b/ush/python/pygfs/task/snowens_analysis.py @@ -69,7 +69,7 @@ def __init__(self, config: Dict[str, Any]): 'CASE': self.task_config.CASE_ENS, 'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'), 'ims_file': _ims_file, - 'DO_IMS_SCF': _DO_IMS_SCF , # Boolean to decide if IMS snow cover processing is done + 'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done } )) From d5ee55cb00b217aaffeb9c22ed30888270691a61 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 7 Oct 2025 21:36:42 +0000 Subject: [PATCH 39/45] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index ef3ba848711..6953b91edfc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit ef3ba848711c1b5750cfa38d3b5fd91f6d4caac3 +Subproject commit 6953b91edfcc30943f5814e7c9af66fd9d44bcc0 From c132905955a1199654489d0058bddf93ce2d8ef6 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 7 Oct 2025 21:45:06 +0000 Subject: [PATCH 40/45] confict --- .github/CODEOWNERS | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3b7b060f522..25b75a93742 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -220,17 +220,10 @@ ush/python/pygfs/task/atm_analysis.py @DavidNew-NOAA @RussTreadon-NOAA ush/python/pygfs/task/atmens_analysis.py @DavidNew-NOAA @RussTreadon-NOAA ush/python/pygfs/task/bmatrix.py @DavidNew-NOAA ush/python/pygfs/task/gfs_forecast.py @aerorahul -<<<<<<< HEAD ush/python/pygfs/task/marine_analysis.py @guillaumevernieres @AndrewEichmann-NOAA @DavidNew-NOAA ush/python/pygfs/task/marine_bmat.py @guillaumevernieres @AndrewEichmann-NOAA @DavidNew-NOAA ush/python/pygfs/task/marine_letkf.py @guillaumevernieres @AndrewEichmann-NOAA @DavidNew-NOAA -ush/python/pygfs/task/oceanice_products.py @aerorahul @GwenChen-NOAA @AminIlia-NOAA @ChristopherHill-NOAA -======= -ush/python/pygfs/task/marine_analysis.py @guillaumevernieres @AndrewEichmann-NOAA -ush/python/pygfs/task/marine_bmat.py @guillaumevernieres @AndrewEichmann-NOAA -ush/python/pygfs/task/marine_letkf.py @guillaumevernieres @AndrewEichmann-NOAA -ush/python/pygfs/task/oceanice_products.py @aerorahul @JesseMeng-NOAA @ChristopherHill-NOAA ->>>>>>> develop +ush/python/pygfs/task/oceanice_products.py @aerorahul @JesseMeng-NOAA @ChristopherHill-NOAA @DavidNew-NOAA ush/python/pygfs/task/offline_analysis.py @CoryMartin-NOAA ush/python/pygfs/task/snow_analysis.py @jiaruidong2017 @DavidNew-NOAA ush/python/pygfs/task/snowens_analysis.py @jiaruidong2017 @DavidNew-NOAA From 23c63c5407a607335f3f97be0232af55433f68bd Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 7 Oct 2025 22:02:41 +0000 Subject: [PATCH 41/45] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6953b91edfc..d4931dff209 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6953b91edfcc30943f5814e7c9af66fd9d44bcc0 +Subproject commit d4931dff2090272e49e19400c9a5bfa2e5ba6ac2 From cf011f7b0947ccdc8e8ac56ac486fd8dd42f5fa6 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 7 Oct 2025 22:16:58 +0000 Subject: [PATCH 42/45] Update obs list --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index d4931dff209..97de79cf4d8 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit d4931dff2090272e49e19400c9a5bfa2e5ba6ac2 +Subproject commit 97de79cf4d853b0b4928b1bc063732749933170d From cb02a943a887255757e48ea4e82104d81380992e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 8 Oct 2025 00:20:08 +0000 Subject: [PATCH 43/45] Warn on missing bias correction files --- ush/python/pygfs/task/analysis.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 0141260b53d..11aa8f1f41c 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -112,7 +112,11 @@ def untar_bias_corrections(self) -> None: for ob in self.task_config.observations: if ob in self.task_config.bias_files and not self.task_config.bias_files[ob] in bias_file_list: bias_file_list.append(self.task_config.bias_files[ob]) - extract_tar(f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}') + bias_file_path = f'{self.task_config.DATA}/obs/{self.task_config.GPREFIX}{self.task_config.bias_files[ob]}' + if os.path.exists(bias_file_path): + extract_tar(bias_file_path) + else: + logger.warning(f"Bias correction file {bias_file_path} does not exist and will be skipped") @logit(logger) def tar_diag_files(self, comout: str, tarball_name: str) -> None: From 894a3bf1eac3855701178b5eae8de7e8c97bf413 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 8 Oct 2025 00:23:27 +0000 Subject: [PATCH 44/45] pynorms --- ush/python/pygfs/task/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 11aa8f1f41c..1998e5a6c04 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -116,7 +116,7 @@ def untar_bias_corrections(self) -> None: if os.path.exists(bias_file_path): extract_tar(bias_file_path) else: - logger.warning(f"Bias correction file {bias_file_path} does not exist and will be skipped") + logger.warning(f"Bias correction file {bias_file_path} does not exist and will be skipped") @logit(logger) def tar_diag_files(self, comout: str, tarball_name: str) -> None: From 4e7e2d001508641b91a2390ba4606b6fbf69784c Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 8 Oct 2025 13:29:49 +0000 Subject: [PATCH 45/45] Update gdas hash to develop --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 97de79cf4d8..6b00f289cf8 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 97de79cf4d853b0b4928b1bc063732749933170d +Subproject commit 6b00f289cf8333133a55bb5ac534a0ddfed74980