From 7601bf086a344b70bf1834ee7281ab6ed2fbb383 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 11 Jun 2025 17:54:14 +0000 Subject: [PATCH 01/39] Initial commit --- dev/parm/config/gfs/config.marineanl.j2 | 2 + scripts/exgdas_global_ocean_analysis_ecen.py | 25 ++++ sorc/gdas.cd | 2 +- sorc/link_workflow.sh | 4 +- ush/python/pygfs/task/marine_recenter.py | 134 +++++++++++++++++++ 5 files changed, 163 insertions(+), 4 deletions(-) create mode 100755 scripts/exgdas_global_ocean_analysis_ecen.py create mode 100644 ush/python/pygfs/task/marine_recenter.py diff --git a/dev/parm/config/gfs/config.marineanl.j2 b/dev/parm/config/gfs/config.marineanl.j2 index bc16ff509ec..4490ecedf82 100644 --- a/dev/parm/config/gfs/config.marineanl.j2 +++ b/dev/parm/config/gfs/config.marineanl.j2 @@ -22,6 +22,8 @@ export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_me export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export MARINE_DET_STAGE_FILES_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_files_stage.yaml.j2" export MARINE_DET_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_finalize.yaml.j2" +export MARINE_ECEN_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ecen_stage.yaml.j2" +export MARINE_ECEN_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ecen_finalize.yaml.j2" export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" export MARINE_JCB_GDAS_OBS="${PARMgfs}/gdas/jcb-gdas/observations/marine" diff --git a/scripts/exgdas_global_ocean_analysis_ecen.py b/scripts/exgdas_global_ocean_analysis_ecen.py new file mode 100755 index 00000000000..ea781ae65de --- /dev/null +++ b/scripts/exgdas_global_ocean_analysis_ecen.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# exgdas_global_marine_analysis_ecen.py +# This script creates an MarineRecenter class +# and runs the initialize, execute, and finalize methods +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.marine_recenter import MarineRecenter + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the aerosol analysis task + MarineRecen = MarineRecenter(config) + MarineRecen.initialize() + MarineRecen.execute('gridgen') + MarineRecen.execute('gdas_ens_handler') + MarineRecen.run() + MarineRecen.finalize() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6f32980dd3f..531f066f493 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6f32980dd3fa83e840ca35ce8d7252f894b44530 +Subproject commit 531f066f4938e53559d9d8a17790d2a76b240bc2 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 4993e655994..6e67daffe9a 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -160,9 +160,7 @@ done cd "${HOMEgfs}/scripts" || exit 8 if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then - declare -a gdas_scripts=(exglobal_prep_ocean_obs.py - exgdas_global_marine_analysis_ecen.py - ) + declare -a gdas_scripts=(exglobal_prep_ocean_obs.py) for gdas_script in "${gdas_scripts[@]}"; do ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/scripts/${gdas_script}" . done diff --git a/ush/python/pygfs/task/marine_recenter.py b/ush/python/pygfs/task/marine_recenter.py new file mode 100644 index 00000000000..f7b3be037dc --- /dev/null +++ b/ush/python/pygfs/task/marine_recenter.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +from logging import getLogger +import os +from typing import Dict +import pygfs.utils.marine_da_utils as mdau +from pygfs.jedi import Jedi +from wxflow import (AttrDict, FileHandler, Task, + add_to_datetime, to_timedelta, to_fv3time, to_isotime, + parse_j2yaml, + logit) + +logger = getLogger(__name__.split('.')[-1]) + + +class MarineRecenter(Task): + """ + Class for global ocean analysis recentering task + """ + + @logit(logger, name="MarineRecenter") + def __init__(self, config: Dict) -> None: + """Constructor for ocean recentering task + Parameters: + ------------ + config: Dict + configuration, namely evironment variables + Returns: + -------- + None + """ + + super().__init__(config) + + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) + if self.task_config.DOIAU: + # forecast initialized at the begining of the DA window + _cice_rst_date = to_fv3time(_window_begin) + else: + # forecast initialized at the middle of the DA window + _cice_rst_date = to_fv3time(self.task_config.current_cycle) + + local_dict = AttrDict( + { + 'PARMsoca': os.path.join(self.task_config.PARMgfs, 'gdas', 'soca'), + 'MARINE_WINDOW_BEGIN': _window_begin, + 'MARINE_WINDOW_END': _window_end, + 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, + 'MARINE_WINDOW_BEGIN_ISO': to_isotime(_window_begin), + 'MARINE_WINDOW_END_ISO': to_isotime(_window_end), + 'MARINE_WINDOW_MIDDLE_ISO': to_isotime(self.task_config.current_cycle), + 'MARINE_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", + 'ENSPERT_RELPATH': _enspert_relpath, + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'cice_rst_date': _cice_rst_date + } + ) + + # Extend task_config with local_dict + self.task_config.update(local_dict) + + # Construct dictionary of JEDI objects, one for each JEDI application need for the analysis + expected_keys = ['gridgen', 'ens_handler'] + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML_ANALYSIS, self.task_config, expected_keys) + + @logit(logger) + def initialize(self): + """Method initialize for ocean recentering task + Parameters: + ------------ + None + Returns: + -------- + None + """ + + # stage fix files + logger.info(f"Staging SOCA fix files from {self.task_config.SOCA_INPUT_FIX_DIR}") + soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) + FileHandler(soca_fix_list).sync() + + # prepare the MOM6 input.nml + mdau.prep_input_nml(self.task_config) + + # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) + logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}") + soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) + FileHandler(soca_utility_list).sync() + + # stage backgrounds + bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) + FileHandler(bkg_list).sync() + + # stage the ensemble members and CICE restarts + logger.info("---------------- Stage ensemble members and CICE restarts") + soca_ens_list = parse_j2yaml(self.task_config.MARINE_ECEN_STAGE_YAML_TMPL, self.task_config) + FileHandler(soca_ens_list).sync() + + @logit(logger) + def execute(self, jedi_dict_key: str) -> None: + """Execute JEDI application of marine analysis + + Parameters + ---------- + jedi_dict_key + key specifying particular Jedi object in self.jedi_dict + + Returns + ---------- + None + """ + + self.jedi_dict[jedi_dict_key].execute() + + @logit(logger) + def finalize(self): + """Method finalize for ocean recentering task + Parameters: + ------------ + None + Returns: + -------- + None + """ + + # Save recentered increments and ensemble statistics + logger.info("---------------- Save recentered increments and ensemble statistics") + soca_ens_list = parse_j2yaml(self.task_config.MARINE_ECEN_FINALIZE_YAML_TMPL, self.task_config) + FileHandler(soca_ens_list).sync() + + From 0174f1401a63b7ed007801fc8f21453a04a825a1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 11 Jun 2025 18:44:39 +0000 Subject: [PATCH 02/39] pynorms --- ush/python/pygfs/task/marine_recenter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/marine_recenter.py b/ush/python/pygfs/task/marine_recenter.py index f7b3be037dc..ced599b4667 100644 --- a/ush/python/pygfs/task/marine_recenter.py +++ b/ush/python/pygfs/task/marine_recenter.py @@ -129,6 +129,4 @@ def finalize(self): # Save recentered increments and ensemble statistics logger.info("---------------- Save recentered increments and ensemble statistics") soca_ens_list = parse_j2yaml(self.task_config.MARINE_ECEN_FINALIZE_YAML_TMPL, self.task_config) - FileHandler(soca_ens_list).sync() - - + FileHandler(soca_ens_list).sync() \ No newline at end of file From ae46e1edf93b8381b4e880e9128d668a934c6fe6 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 11 Jun 2025 18:46:32 +0000 Subject: [PATCH 03/39] pynorms round 2 --- ush/python/pygfs/task/marine_recenter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_recenter.py b/ush/python/pygfs/task/marine_recenter.py index ced599b4667..a3af87f882e 100644 --- a/ush/python/pygfs/task/marine_recenter.py +++ b/ush/python/pygfs/task/marine_recenter.py @@ -129,4 +129,5 @@ def finalize(self): # Save recentered increments and ensemble statistics logger.info("---------------- Save recentered increments and ensemble statistics") soca_ens_list = parse_j2yaml(self.task_config.MARINE_ECEN_FINALIZE_YAML_TMPL, self.task_config) - FileHandler(soca_ens_list).sync() \ No newline at end of file + FileHandler(soca_ens_list).sync() + \ No newline at end of file From 30aa61cd18eb039051e40661cc11e3974964c7ab Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 26 Jun 2025 15:09:53 +0000 Subject: [PATCH 04/39] Update --- dev/parm/config/gfs/config.marineanl.j2 | 1 + scripts/exgdas_global_marine_analysis_ecen.py | 24 +++++++++++++++++++ ush/python/pygfs/__init__.py | 1 + ush/python/pygfs/task/marine_recenter.py | 9 ++++++- 4 files changed, 34 insertions(+), 1 deletion(-) create mode 100755 scripts/exgdas_global_marine_analysis_ecen.py diff --git a/dev/parm/config/gfs/config.marineanl.j2 b/dev/parm/config/gfs/config.marineanl.j2 index 4490ecedf82..fc9bc3a2222 100644 --- a/dev/parm/config/gfs/config.marineanl.j2 +++ b/dev/parm/config/gfs/config.marineanl.j2 @@ -7,6 +7,7 @@ echo "BEGIN: config.marineanl" export JEDI_CONFIG_YAML_BMAT="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" export JEDI_CONFIG_YAML_ANALYSIS="${PARMgfs}/gdas/soca_analysis_jedi_config.yaml.j2" +export JEDI_CONFIG_YAML_ECEN="${PARMgfs}/gdas/soca/soca_ecen_jedi_config.yaml.j2" export JCB_ALGO_YAML_VAR="{{ JCB_ALGO_YAML_VAR }}" export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" diff --git a/scripts/exgdas_global_marine_analysis_ecen.py b/scripts/exgdas_global_marine_analysis_ecen.py new file mode 100755 index 00000000000..e339d790016 --- /dev/null +++ b/scripts/exgdas_global_marine_analysis_ecen.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# exgdas_global_marine_analysis_ecen.py +# This script creates an MarineRecenter class +# and runs the initialize, run, and finalize methods +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.marine_recenter import MarineRecenter + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # + MarineRecen = MarineRecenter(config) + MarineRecen.initialize() + MarineRecen.execute('gridgen') + MarineRecen.execute('ens_handler') + MarineRecen.finalize() diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index 12c75bdad8f..98edc71f5ab 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -17,6 +17,7 @@ from .task.gfs_forecast import GFSForecast from .utils import marine_da_utils from .task.fetch import Fetch +from .task.marine_recenter import MarineRecenter __docformat__ = "restructuredtext" __version__ = "0.1.0" diff --git a/ush/python/pygfs/task/marine_recenter.py b/ush/python/pygfs/task/marine_recenter.py index a3af87f882e..92686dc1fc6 100644 --- a/ush/python/pygfs/task/marine_recenter.py +++ b/ush/python/pygfs/task/marine_recenter.py @@ -64,7 +64,7 @@ def __init__(self, config: Dict) -> None: # Construct dictionary of JEDI objects, one for each JEDI application need for the analysis expected_keys = ['gridgen', 'ens_handler'] - self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML_ANALYSIS, self.task_config, expected_keys) + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML_ECEN, self.task_config, expected_keys) @logit(logger) def initialize(self): @@ -99,6 +99,13 @@ def initialize(self): soca_ens_list = parse_j2yaml(self.task_config.MARINE_ECEN_STAGE_YAML_TMPL, self.task_config) FileHandler(soca_ens_list).sync() + # initialize JEDI applications + logger.info(f"Initializing SOCA gridgen application") + self.jedi_dict['gridgen'].initialize(self.task_config) + + logger.info(f"Initializing SOCA ensemble handler") + self.jedi_dict['ens_handler'].initialize(self.task_config) + @logit(logger) def execute(self, jedi_dict_key: str) -> None: """Execute JEDI application of marine analysis From 9db049cbd67c4bb0bfc05648eb1fc134c8072b7a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 26 Jun 2025 15:10:24 +0000 Subject: [PATCH 05/39] Update --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8bf3e1c46c0..1c3478927b8 100644 --- a/.gitignore +++ b/.gitignore @@ -140,7 +140,6 @@ sorc/ocnicepost.fd #------------------------------ # jobs symlinks # scripts symlinks -scripts/exgdas_global_marine_analysis_ecen.py scripts/exglobal_prep_ocean_obs.py # ush symlinks ush/bufr2ioda_insitu_profile_argo.py From 27c07f79cb40a1226291caec678bf51a4dccb41d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 7 Jul 2025 14:27:13 +0000 Subject: [PATCH 06/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 2ef5c6eba3a..1665f166291 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 2ef5c6eba3a72d8f3e322aab0440800a677422cc +Subproject commit 1665f166291bc9acdb8bfa1274384f98469569df From 63eefd91377fdd1312aa91adc1d22036f5f7cdd6 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 9 Jul 2025 18:28:11 +0000 Subject: [PATCH 07/39] Debugging --- dev/workflow/applications/gfs_cycled.py | 13 +++++++------ sorc/gdas.cd | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dev/workflow/applications/gfs_cycled.py b/dev/workflow/applications/gfs_cycled.py index 5f49b9383e1..3da7f10a45e 100644 --- a/dev/workflow/applications/gfs_cycled.py +++ b/dev/workflow/applications/gfs_cycled.py @@ -255,12 +255,7 @@ def get_task_names(self): task_names[run] += ['anal', 'analcalc'] if options['do_jediocnvar']: - task_names[run] += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] - if options['do_letkf_ocn']: - task_names[run] += ['marineanlletkf'] - if options['do_hybvar']: - task_names[run] += ['ocnanalecen'] - task_names[run] += ['marineanlchkpt', 'marineanlfinal'] + task_names[run] += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar', 'marineanlchkpt', 'marineanlfinal'] task_names[run] += ['sfcanl'] @@ -393,6 +388,12 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 + if options['do_jediocnvar']: + if options['do_letkf_ocn']: + task_names[run] += ['marineanlletkf'] + if options['do_hybvar']: + task_names[run] += ['ocnanalecen'] + else: task_names[run] += ['eobs', 'eupd', 'ecen'] task_names[run].append('echgres') if 'gdas' in run else 0 diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 1665f166291..d13072e0c2d 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 1665f166291bc9acdb8bfa1274384f98469569df +Subproject commit d13072e0c2d3e3815b66ae4807b9e947e45b62e0 From f013ae5c115c6063468d093a7c8f27833afd72ad Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Jul 2025 13:38:38 +0000 Subject: [PATCH 08/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index d13072e0c2d..a24efa19f3e 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit d13072e0c2d3e3815b66ae4807b9e947e45b62e0 +Subproject commit a24efa19f3e8c68b31bc62e8d9e0b14cf418e22e From 14531095dbf65bd678fe9ea086047c18aabbdd16 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Jul 2025 13:39:38 +0000 Subject: [PATCH 09/39] Refactor marine bmat finalize --- dev/parm/config/gfs/config.marineanl.j2 | 1 + ush/python/pygfs/task/marine_analysis.py | 2 +- ush/python/pygfs/task/marine_bmat.py | 87 +++--------------------- 3 files changed, 12 insertions(+), 78 deletions(-) diff --git a/dev/parm/config/gfs/config.marineanl.j2 b/dev/parm/config/gfs/config.marineanl.j2 index fc9bc3a2222..d6231762aed 100644 --- a/dev/parm/config/gfs/config.marineanl.j2 +++ b/dev/parm/config/gfs/config.marineanl.j2 @@ -20,6 +20,7 @@ export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yam export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2" export MARINE_UTILITY_YAML_TMPL="${PARMgfs}/gdas/soca/soca_utils_stage.yaml.j2" export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_mem.yaml.j2" +export MARINE_BMAT_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_bmat_finalize.yaml.j2" export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export MARINE_DET_STAGE_FILES_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_files_stage.yaml.j2" export MARINE_DET_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_finalize.yaml.j2" diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index ae2c159677b..baf40416b77 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -194,7 +194,7 @@ def finalize(self: Task) -> None: """ # Save output files to COM - logger.info(f"Copy files from {self.task_config.DATA} to {self.task_config.COMOUT_OCEAN_ANALYSIS}") + logger.info(f"Copy files to ROTDIR") soca_finalize_list = parse_j2yaml(self.task_config.MARINE_DET_FINALIZE_YAML_TMPL, self.task_config) FileHandler(soca_finalize_list).sync() diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index b2e30caa5a2..875500a8f18 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -62,6 +62,9 @@ def __init__(self, config): 'MARINE_WINDOW_MIDDLE': self.task_config.current_cycle, 'MARINE_WINDOW_END': _window_end, 'MARINE_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", + 'MARINE_WINDOW_BEGIN_ISO': to_isotime(_window_begin), + 'MARINE_WINDOW_MIDDLE_ISO': to_isotime(self.task_config.current_cycle), + 'MARINE_WINDOW_END_ISO': to_isotime(_window_end), 'ENSPERT_RELPATH': _enspert_relpath, 'CALC_SCALE_EXEC': _calc_scale_exec, 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", @@ -215,84 +218,14 @@ def finalize(self: Task) -> None: None """ - APREFIX = self.task_config.APREFIX - window_begin_iso = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') - window_middle_iso = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') - window_end_iso = self.task_config.MARINE_WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ') - - # Copy the soca grid if it was created - grid_file = os.path.join(self.task_config.DATA, 'soca_gridspec.nc') - if os.path.exists(grid_file): - logger.info(f"Copying the soca grid file to the ROTDIR") - FileHandler({'copy': [[grid_file, - os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, 'soca_gridspec.nc')]]}).sync() - - # Copy the diffusion coefficient files to the ROTDIR - logger.info(f"Copying the diffusion coefficient files to the ROTDIR") - diffusion_coeff_list = [] - for diff_type in ['hz', 'vt']: - src = os.path.join(self.task_config.DATAstaticb, f"{diff_type}_ocean.nc") - dest = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, - f"{APREFIX}{diff_type}_ocean.nc") - diffusion_coeff_list.append([src, dest]) - - FileHandler({'copy': diffusion_coeff_list}).sync() - - # Copy diag B files to ROTDIR - logger.info(f"Copying diag B files to the ROTDIR") - diagb_list = [] - # TODO(AFE) the two renames are to accomodate yaml settings in var task, which should changed # ocean diag B - os.rename(os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.incr.{window_end_iso}.nc"), + os.rename(os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.incr.{self.task_config.MARINE_WINDOW_END_ISO}.nc"), os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.nc")) - src = os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.nc") - dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, f"{APREFIX}ocean.bkgerr_ens_stddev.nc") - diagb_list.append([src, dst]) - - # ice diag B - os.rename(os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.incr.{window_end_iso}.nc"), - os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.nc")) - src = os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.nc") - dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, f"{APREFIX}ice.bkgerr_ens_stddev.nc") - diagb_list.append([src, dst]) - - if self.task_config.DOHYBVAR_OCN == "YES" or self.task_config.NMEM_ENS >= 2: - src = os.path.join(self.task_config.DATAstaticb, f"ocn.ssh_recentering_error.incr.{window_begin_iso}.nc") - dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, f"{APREFIX}ocean.recentering_error.nc") - diagb_list.append([src, dst]) + os.rename(os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.incr.{self.task_config.MARINE_WINDOW_END_ISO}.nc"), + os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.nc")) - src = os.path.join(self.task_config.DATAstaticb, f"ice.ssh_recentering_error.incr.{window_begin_iso}.nc") - dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, f"{APREFIX}ice.recentering_error.nc") - diagb_list.append([src, dst]) - - FileHandler({'copy': diagb_list}).sync() - - # Copy the ensemble perturbation diagnostics to the ROTDIR - if self.task_config.DOHYBVAR_OCN == "YES" or self.task_config.NMEM_ENS >= 2: - weight_list = [] - src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc") - dst = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, - f"{APREFIX}ocean.ens_weights.nc") - weight_list.append([src, dst]) - - src = os.path.join(self.task_config.DATA, f"ice.ens_weights.incr.{window_middle_iso}.nc") - dst = os.path.join(self.task_config.COMOUT_ICE_BMATRIX, - f"{APREFIX}ice.ens_weights.nc") - weight_list.append([src, dst]) - - # Copy the ssh diagnostics - for string in ['ssh_steric_stddev', 'ssh_unbal_stddev', 'ssh_total_stddev', 'steric_explained_variance']: - weight_list.append([os.path.join(self.task_config.DATA, 'staticb', f'ocn.{string}.incr.{window_begin_iso}.nc'), - os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, f'{APREFIX}ocean.{string}.nc')]) - - FileHandler({'copy': weight_list}).sync() - - # Copy the YAML files to the OCEAN ROTDIR - yamls = glob.glob(os.path.join(self.task_config.DATA, '*.yaml')) - yaml_list = [] - for yaml_file in yamls: - dest = os.path.join(self.task_config.COMOUT_OCEAN_BMATRIX, - f"{APREFIX}{os.path.basename(yaml_file)}") - yaml_list.append([yaml_file, dest]) - FileHandler({'copy': yaml_list}).sync() + # Save output files to COM + logger.info(f"Copy files to ROTDIR") + soca_finalize_list = parse_j2yaml(self.task_config.MARINE_BMAT_FINALIZE_YAML_TMPL, self.task_config) + FileHandler(soca_finalize_list).sync() From 78a87a677b552d9b8bb42b3e22bd5787f3a12b2e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Jul 2025 13:42:35 +0000 Subject: [PATCH 10/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a24efa19f3e..c2358ed3787 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a24efa19f3e8c68b31bc62e8d9e0b14cf418e22e +Subproject commit c2358ed3787f5692e4adc4e8647fee18ccbc7655 From b0d3c2c0b5d5f038268675806a1a489ff009bf05 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 10 Jul 2025 15:32:24 +0000 Subject: [PATCH 11/39] Debugging --- dev/workflow/applications/gfs_cycled.py | 10 +++++----- dev/workflow/rocoto/gfs_tasks.py | 2 +- jobs/JGLOBAL_MARINE_BMAT | 8 +++++--- sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_bmat.py | 9 +++------ 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/dev/workflow/applications/gfs_cycled.py b/dev/workflow/applications/gfs_cycled.py index 477947d3250..6015f2aad6e 100644 --- a/dev/workflow/applications/gfs_cycled.py +++ b/dev/workflow/applications/gfs_cycled.py @@ -388,11 +388,11 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 - if options['do_jediocnvar']: - if options['do_letkf_ocn']: - task_names[run] += ['marineanlletkf'] - if options['do_hybvar']: - task_names[run] += ['ocnanalecen'] + if options['do_jediocnvar']: + if options['do_letkf_ocn']: + task_names[run] += ['marineanlletkf'] + if options['do_hybvar']: + task_names[run] += ['ocnanalecen'] else: task_names[run] += ['eobs', 'eupd', 'ecen'] diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index 2e85f83e131..ffece506552 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -865,7 +865,7 @@ def marineanlchkpt(self): dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlvar'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f'{self.run}_ocnanalecen'} + dep_dict = {'type': 'task', 'name': f'enkf{self.run}_ocnanalecen'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_mergensst']: data = f'&ROTDIR;/{self.run}.@Y@m@d/@H/atmos/{self.run}.t@Hz.sfcanl.nc' diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index b495f912576..0e0c614d2d6 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -48,10 +48,12 @@ RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \ - COMOUT_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL + COMOUT_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \ + COMOUT_CONF:COM_CONF_TMPL -mkdir -p "${COMOUT_OCEAN_BMATRIX}" -mkdir -p "${COMOUT_ICE_BMATRIX}" +mkdir -m 775 -p "${COMOUT_OCEAN_BMATRIX}" +mkdir -m 775 -p "${COMOUT_ICE_BMATRIX}" +mkdir -m 775 -p "${COMOUT_CONF}" ############################################################### # Run relevant script diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c2358ed3787..c987254c7d8 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c2358ed3787f5692e4adc4e8647fee18ccbc7655 +Subproject commit c987254c7d8718098e25458bdf70baabf19f8475 diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 875500a8f18..46490319713 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -5,14 +5,11 @@ from logging import getLogger import pygfs.utils.marine_da_utils as mdau -from wxflow import (AttrDict, - FileHandler, - add_to_datetime, to_timedelta, +from wxflow import (AttrDict, FileHandler, Executable, Task, + add_to_datetime, to_timedelta, to_isotime, chdir, parse_j2yaml, save_as_yaml, - logit, - Executable, - Task) + logit) from pygfs.jedi import Jedi From 6e77d0a5003716afdd5ac06612ec51932c10ff18 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Sun, 13 Jul 2025 20:22:57 +0000 Subject: [PATCH 12/39] Debugging --- dev/workflow/applications/gfs_cycled.py | 10 +++++----- dev/workflow/rocoto/gfs_tasks.py | 6 +++--- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN | 2 +- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 14 ++++++++++---- ush/python/pygfs/task/marine_letkf.py | 3 ++- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/dev/workflow/applications/gfs_cycled.py b/dev/workflow/applications/gfs_cycled.py index 6015f2aad6e..b1fa137e69c 100644 --- a/dev/workflow/applications/gfs_cycled.py +++ b/dev/workflow/applications/gfs_cycled.py @@ -388,17 +388,17 @@ def get_task_names(self): task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 + else: + task_names[run] += ['eobs', 'eupd', 'ecen'] + task_names[run].append('echgres') if 'gdas' in run else 0 + task_names[run] += ['ediag'] + if options['do_jediocnvar']: if options['do_letkf_ocn']: task_names[run] += ['marineanlletkf'] if options['do_hybvar']: task_names[run] += ['ocnanalecen'] - else: - task_names[run] += ['eobs', 'eupd', 'ecen'] - task_names[run].append('echgres') if 'gdas' in run else 0 - task_names[run] += ['ediag'] - task_names[run].append('esnowanl') if options['do_jedisnowda'] else 0 task_names[run].append('efcs') if 'gdas' in run else 0 task_names[run].append('epos') if 'gdas' in run else 0 diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index ffece506552..4e947b71b06 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -705,9 +705,9 @@ def prepoceanobs(self): def marineanlletkf(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} + dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf', '')}_prepoceanobs"} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmat'} + dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf', '')}_marinebmat"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -838,7 +838,7 @@ def ocnanalecen(self): # can run in parallel with marinebmat deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmatinit'} + dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf', '')}_marinebmatinit"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN index 63b5775e85c..daf72870893 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN @@ -1,6 +1,6 @@ #!/bin/bash -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATAjob="${DATAROOT//enkf/}/${RUN//enkf/}marineanalysis.${PDY:-}${cyc}" export DATA="${DATAjob}/${jobid}" export DATAens="${DATAjob}/ensdata" diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index e8c3b1f1c2e..c347e41cd19 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,6 +1,7 @@ #!/bin/bash -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +# TODO: use ${RUN}, ${DATAROOT} rather than ${RUN//enkf/}, ${DATAROOT//enkf/} once https://github.com/NOAA-EMC/global-workflow/issues/3767 is fixed +export DATAjob="${DATAROOT//enkf/}/${RUN//enkf/}marineanalysis.${PDY:-}${cyc}" export DATA="${DATAjob}/${jobid}" # Create the directory to hold ensemble perturbations export DATAens="${DATAjob}/ensdata" @@ -19,11 +20,13 @@ gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} export GDUMP="gdas" export GDUMP_ENS="enkf${GDUMP}" -export OPREFIX="${RUN}.t${cyc}z." + +# TODO: use ${RUN} rather than ${RUN//enkf/} once https://github.com/NOAA-EMC/global-workflow/issues/3767 is fixed +export OPREFIX="${RUN//enkf/}.t${cyc}z." ec_offset=${NMEM_ENS_GFS_OFFSET:-20} export NMEM_ENS_MAX=${NMEM_ENS} -if [[ "${RUN}" == "gfs" && "${DOHYBVAR_OCN}" == "YES" ]]; then +if [[ "${RUN//enkf/}" == "gfs" && "${DOHYBVAR_OCN}" == "YES" ]]; then NMEM_ENS=${NMEM_ENS_GFS} export mem_offset=$((ec_offset * cyc/6)) else @@ -34,8 +37,11 @@ RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ +# TODO: remove explicit RUN specification once https://github.com/NOAA-EMC/global-workflow/issues/3767 is fixed +RUN="${GDUMP}" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OBS:COM_OBS_TMPL \ + +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \ COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 4e549e6acf8..f09fc7f2972 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -102,7 +102,8 @@ def initialize(self): 'ROTDIR', 'RUN', 'WINDOW_BEGIN', - 'WINDOW_MIDDLE'] + 'WINDOW_MIDDLE', + 'DATAens'] for key in keys: stageconfig[key] = self.task_config[key] From 3134e16092625dfc53a9445414f1d04e2eb6fd21 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 14 Jul 2025 16:18:17 +0000 Subject: [PATCH 13/39] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c987254c7d8..62e3b84f093 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c987254c7d8718098e25458bdf70baabf19f8475 +Subproject commit 62e3b84f093cc33dc0ec0aa0ca8e34d18ef4b1a8 From 7891b63bc721133944997f88f7d900e1d76224b1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 14 Jul 2025 16:19:52 +0000 Subject: [PATCH 14/39] Updates --- dev/workflow/rocoto/gfs_tasks.py | 6 +++--- parm/archive/enkf.yaml.j2 | 3 +++ parm/archive/gdas.yaml.j2 | 3 --- ush/python/pygfs/task/marine_bmat.py | 11 +++++------ 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index 4e947b71b06..0008c58b2a9 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -865,7 +865,7 @@ def marineanlchkpt(self): dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlvar'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f'enkf{self.run}_ocnanalecen'} + dep_dict = {'type': 'task', 'name': f"enkf{self.run}_ocnanalecen"} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_mergensst']: data = f'&ROTDIR;/{self.run}.@Y@m@d/@H/atmos/{self.run}.t@Hz.sfcanl.nc' @@ -2792,7 +2792,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.run}_esfc'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf", "")}_ocnanalecen'} + dep_dict = {'type': 'task', 'name': f'{self.run}_ocnanalecen'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} @@ -2976,7 +2976,7 @@ def earc_tars(self): dep_dict = {'type': 'metatask', 'name': f'{self.run}_ecmn'} deps.append(rocoto.add_dependency(dep_dict)) if self._base.get('DOHYBVAR_OCN', True): - dep_dict = {'type': 'task', 'name': f'gfs_ocnanalecen'} + dep_dict = {'type': 'task', 'name': f'enkfgfs_ocnanalecen'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'gfs_marineanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 70506f38295..e6aa8d71ba5 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -34,6 +34,9 @@ enkf: {% set steps = ["eobs", "eupd"] %} {% endif %} {% endif %} + {% if DOHYBVAR_OCN %} + {% set steps = steps + ["ocnanalecen"] %} + {% endif %} {% for step in steps %} - "logs/{{ cycle_YMDH }}/{{ RUN }}_{{ step }}.log" diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 612929b13a9..c1a01531d44 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -27,9 +27,6 @@ gdas: - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlvar.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlfinal.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlchkpt.log" - {% if DOHYBVAR_OCN %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}_ocnanalecen.log" - {% endif %} {% endif %} {% if DO_AERO_ANL %} - "logs/{{ cycle_YMDH }}/gdas_aeroanlgenb.log" diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 46490319713..7975c3bea8c 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -114,6 +114,7 @@ def initialize(self: Task) -> None: # stage backgrounds # TODO(G): Check ocean backgrounds dates for consistency + logger.info(f"Staging SOCA backgrounds") bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) FileHandler(bkg_list).sync() @@ -142,14 +143,12 @@ def initialize(self: Task) -> None: # stage ensemble members for the hybrid background error if self.task_config.DOHYBVAR_OCN == "YES" or self.task_config.NMEM_ENS >= 2: logger.debug(f"Stage ensemble members for the hybrid background error") - mdau.stage_ens_mem(self.task_config) + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, self.task_config) + FileHandler(letkf_stage_list).sync() # create the symbolic link to the static B-matrix directory - link_target = os.path.join(self.task_config.DATAstaticb) - link_name = os.path.join(self.task_config.DATA, 'staticb') - if os.path.exists(link_name): - os.remove(link_name) - os.symlink(link_target, link_name) + FileHandler({'link': [[self.task_config.DATAstaticb, + os.path.join(self.task_config.DATA, 'staticb')]]}).sync() @logit(logger) def execute(self) -> None: From 7230339ed335f3ffdfb8a3187d94ea56712a0a67 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 14 Jul 2025 16:25:18 +0000 Subject: [PATCH 15/39] pynorms --- ush/python/pygfs/task/marine_bmat.py | 2 +- ush/python/pygfs/task/marine_recenter.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 7975c3bea8c..66991ace662 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -219,7 +219,7 @@ def finalize(self: Task) -> None: os.rename(os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.incr.{self.task_config.MARINE_WINDOW_END_ISO}.nc"), os.path.join(self.task_config.DATAstaticb, f"ocn.bkgerr_stddev.nc")) os.rename(os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.incr.{self.task_config.MARINE_WINDOW_END_ISO}.nc"), - os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.nc")) + os.path.join(self.task_config.DATAstaticb, f"ice.bkgerr_stddev.nc")) # Save output files to COM logger.info(f"Copy files to ROTDIR") diff --git a/ush/python/pygfs/task/marine_recenter.py b/ush/python/pygfs/task/marine_recenter.py index 92686dc1fc6..78fbca00418 100644 --- a/ush/python/pygfs/task/marine_recenter.py +++ b/ush/python/pygfs/task/marine_recenter.py @@ -137,4 +137,3 @@ def finalize(self): logger.info("---------------- Save recentered increments and ensemble statistics") soca_ens_list = parse_j2yaml(self.task_config.MARINE_ECEN_FINALIZE_YAML_TMPL, self.task_config) FileHandler(soca_ens_list).sync() - \ No newline at end of file From 831b4afd7f4a3a9ab116714e61708d26d0626a76 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 14 Jul 2025 18:39:21 +0000 Subject: [PATCH 16/39] Use vtscale spython script where it lives --- ush/python/pygfs/task/marine_bmat.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 66991ace662..4812d790c7a 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -126,8 +126,6 @@ def initialize(self: Task) -> None: # initialize vtscales python script vtscales_config = self.jedi_dict['soca_parameters_diffusion_vt'].render_jcb(self.task_config, 'soca_vtscales') save_as_yaml(vtscales_config, os.path.join(self.task_config.DATA, 'soca_vtscales.yaml')) - FileHandler({'copy': [[os.path.join(self.task_config.CALC_SCALE_EXEC), - os.path.join(self.task_config.DATA, 'calc_scales.x')]]}).sync() # initialize JEDI applications self.jedi_dict['gridgen'].initialize(self.task_config) @@ -182,7 +180,7 @@ def execute(self) -> None: # vertical diffusion exec_cmd = Executable("python") - exec_name = os.path.join(self.task_config.DATA, 'calc_scales.x') + exec_name = self.task_config.CALC_SCALE_EXEC exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('soca_vtscales.yaml') mdau.run(exec_cmd) From 8746086f5c78b86c1ad39b95615d6268b158e131 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 14 Jul 2025 21:31:14 +0000 Subject: [PATCH 17/39] Debugging --- jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN | 2 +- sorc/gdas.cd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN index daf72870893..4aea2e39a2a 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN +++ b/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN @@ -63,7 +63,7 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd "${DATAROOT}" && true +cd "${DATAROOT//enkf/}" && true export err=$? if [[ ${err} -ne 0 ]]; then err_exit diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 62e3b84f093..d4792063c9d 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 62e3b84f093cc33dc0ec0aa0ca8e34d18ef4b1a8 +Subproject commit d4792063c9d4adb6eff0c59e6345ea0dabaec3b0 From 65d6f3398b616311a27f2c2ff50ecc3c040635f0 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 14 Jul 2025 21:36:44 +0000 Subject: [PATCH 18/39] pynorms --- ush/python/pygfs/task/marine_bmat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 4812d790c7a..ea3c0cb640b 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -145,7 +145,7 @@ def initialize(self: Task) -> None: FileHandler(letkf_stage_list).sync() # create the symbolic link to the static B-matrix directory - FileHandler({'link': [[self.task_config.DATAstaticb, + FileHandler({'link': [[self.task_config.DATAstaticb, os.path.join(self.task_config.DATA, 'staticb')]]}).sync() @logit(logger) From d3230504d6d61f4c5c3bf694b23d78ae0fcd705d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 17 Jul 2025 12:55:16 +0000 Subject: [PATCH 19/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index d4792063c9d..35c71701b20 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit d4792063c9d4adb6eff0c59e6345ea0dabaec3b0 +Subproject commit 35c71701b209d05cbc2e9ca9dac57a5c4fbab3b0 From 8481614790a7ecf5ef11188adfb57b6e5381354a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 17 Jul 2025 13:29:21 +0000 Subject: [PATCH 20/39] Update gdas hash --- scripts/exgdas_global_ocean_analysis_ecen.py | 25 -------------------- sorc/gdas.cd | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100755 scripts/exgdas_global_ocean_analysis_ecen.py diff --git a/scripts/exgdas_global_ocean_analysis_ecen.py b/scripts/exgdas_global_ocean_analysis_ecen.py deleted file mode 100755 index ea781ae65de..00000000000 --- a/scripts/exgdas_global_ocean_analysis_ecen.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -# exgdas_global_marine_analysis_ecen.py -# This script creates an MarineRecenter class -# and runs the initialize, execute, and finalize methods -import os - -from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs.task.marine_recenter import MarineRecenter - -# Initialize root logger -logger = Logger(level='DEBUG', colored_log=True) - - -if __name__ == '__main__': - - # Take configuration from environment and cast it as python dictionary - config = cast_strdict_as_dtypedict(os.environ) - - # Instantiate the aerosol analysis task - MarineRecen = MarineRecenter(config) - MarineRecen.initialize() - MarineRecen.execute('gridgen') - MarineRecen.execute('gdas_ens_handler') - MarineRecen.run() - MarineRecen.finalize() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 35c71701b20..24e4e315f9d 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 35c71701b209d05cbc2e9ca9dac57a5c4fbab3b0 +Subproject commit 24e4e315f9d2c7f12efa79a364f568a5e11891c8 From 75dc2355c04330f110949ce1c62a84e9b8b9f507 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 18 Jul 2025 14:09:06 +0000 Subject: [PATCH 21/39] Updates --- dev/jobs/{ocnanalecen.sh => marineanlecen.sh} | 2 +- dev/workflow/rocoto/gfs_tasks.py | 17 +++++++------- jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT | 4 ++-- ...YSIS_ECEN => JGLOBAL_MARINE_ANALYSIS_ECEN} | 17 +++++--------- jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE | 6 ++--- jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE | 4 ++-- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 23 ++++++------------- jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL | 4 ++-- jobs/JGLOBAL_MARINE_BMAT | 2 +- jobs/JGLOBAL_MARINE_BMAT_INITIALIZE | 2 +- ...en.py => exglobal_marine_analysis_ecen.py} | 0 ush/python/pygfs/task/marine_letkf.py | 1 + 12 files changed, 34 insertions(+), 48 deletions(-) rename dev/jobs/{ocnanalecen.sh => marineanlecen.sh} (91%) rename jobs/{JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN => JGLOBAL_MARINE_ANALYSIS_ECEN} (80%) rename scripts/{exgdas_global_marine_analysis_ecen.py => exglobal_marine_analysis_ecen.py} (100%) diff --git a/dev/jobs/ocnanalecen.sh b/dev/jobs/marineanlecen.sh similarity index 91% rename from dev/jobs/ocnanalecen.sh rename to dev/jobs/marineanlecen.sh index 42cd7d67b30..070afdc469d 100755 --- a/dev/jobs/ocnanalecen.sh +++ b/dev/jobs/marineanlecen.sh @@ -20,6 +20,6 @@ export PYTHONPATH ############################################################### # Execute the JJOB -"${HOMEgfs}"/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN +"${HOMEgfs}"/jobs/JGLOBAL_MARINE_ANALYSIS_ECEN status=$? exit "${status}" diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index 0008c58b2a9..8d1cebd7bcc 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -717,7 +717,7 @@ def marineanlletkf(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/marineanlletkf.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -848,7 +848,7 @@ def ocnanalecen(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/ocnanalecen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2478,14 +2478,13 @@ def atmensanlinit(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - cycledef = "gdas" resources = self.get_resource('atmensanlinit') task_name = f'{self.run}_atmensanlinit' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': cycledef, + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2511,7 +2510,7 @@ def atmensanlobs(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlobs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2537,7 +2536,7 @@ def atmensanlsol(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlsol.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2563,7 +2562,7 @@ def atmensanlletkf(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlletkf.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2592,7 +2591,7 @@ def atmensanlfv3inc(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlfv3inc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2616,7 +2615,7 @@ def atmensanlfinal(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run.replace('enkf', ''), + 'cycledef': self.run, 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlfinal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT b/jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT index 092a3aafd2e..50bb03e1258 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT @@ -1,8 +1,8 @@ #!/bin/bash export WIPE_DATA="NO" -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATAjob="${DATAROOT}/marineanalysis.${PDY:-}${cyc}" export DATAens="${DATAjob}/ensdata" -export DATA="${DATAjob}/marinevariational" +export DATA="${DATAjob}/marineanlvar" source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlchkpt" -c "base marineanl marineanlchkpt" diff --git a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN b/jobs/JGLOBAL_MARINE_ANALYSIS_ECEN similarity index 80% rename from jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN rename to jobs/JGLOBAL_MARINE_ANALYSIS_ECEN index 4aea2e39a2a..d817d0462fa 100755 --- a/jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_ECEN @@ -1,7 +1,7 @@ #!/bin/bash -export DATAjob="${DATAROOT//enkf/}/${RUN//enkf/}marineanalysis.${PDY:-}${cyc}" -export DATA="${DATAjob}/${jobid}" +export DATAjob="${DATAROOT/enkf}/marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/ocnanalecen" export DATAens="${DATAjob}/ensdata" source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalecen" -c "base marineanl ocnanalecen" @@ -15,17 +15,16 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") export gPDY=${GDATE:0:8} export gcyc=${GDATE:8:2} export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_ENS_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_ENS_PREV:COM_ICE_HISTORY_TMPL -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ +RUN=${GDUMP} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \ COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \ COMOUT_ICE_ANALYSIS:COM_ICE_ANALYSIS_TMPL @@ -42,7 +41,7 @@ fi ############################################################### # Run relevant script -EXSCRIPT=${GDASOCNCENPY:-${HOMEgfs}/scripts/exgdas_global_marine_analysis_ecen.py} +EXSCRIPT=${GDASOCNCENPY:-${HOMEgfs}/scripts/exglobal_marine_analysis_ecen.py} ${EXSCRIPT} && true export err=$? if [[ ${err} -ne 0 ]]; then @@ -63,14 +62,10 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd "${DATAROOT//enkf/}" && true +cd "${DATAROOT/enkf}" && true export err=$? if [[ ${err} -ne 0 ]]; then err_exit fi -if [[ "${KEEPDATA}" == "NO" ]]; then - rm -rf "${DATA}" -fi - exit 0 diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE b/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE index e20bcc9aa9e..d0da7f265af 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE @@ -1,8 +1,8 @@ #!/bin/bash export WIPE_DATA="NO" -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATAjob="${DATAROOT}/marineanalysis.${PDY:-}${cyc}" export DATAens="${DATAjob}/ensdata" -export DATA="${DATAjob}/marinevariational" +export DATA="${DATAjob}/marineanlvar" source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlfinal" -c "base marineanl marineanlfinal" ############################################## @@ -46,7 +46,7 @@ if [[ ${err} -ne 0 ]]; then fi if [[ "${KEEPDATA}" == "NO" ]]; then - rm -rf "${DATA}" + rm -rf "${DATAjob}" fi exit 0 diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE index 0c851459f5f..95e5ab12385 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE @@ -1,8 +1,8 @@ #!/bin/bash -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATAjob="${DATAROOT}/marineanalysis.${PDY:-}${cyc}" export DATAens="${DATAjob}/ensdata" -export DATA="${DATAjob}/marinevariational" +export DATA="${DATAjob}/marineanlvar" source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlinit" -c "base marineanl marineanlinit" diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index c347e41cd19..74d843179ba 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,9 +1,8 @@ #!/bin/bash -# TODO: use ${RUN}, ${DATAROOT} rather than ${RUN//enkf/}, ${DATAROOT//enkf/} once https://github.com/NOAA-EMC/global-workflow/issues/3767 is fixed -export DATAjob="${DATAROOT//enkf/}/${RUN//enkf/}marineanalysis.${PDY:-}${cyc}" -export DATA="${DATAjob}/${jobid}" -# Create the directory to hold ensemble perturbations +# Use ${DATAROOT/enkf} because DATAjob is created by a gdas_marineanlinit job +export DATAjob="${DATAROOT/enkf}/marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/marineanlletkf" export DATAens="${DATAjob}/ensdata" if [[ ! -d "${DATAens}" ]]; then mkdir -p "${DATAens}"; fi @@ -19,14 +18,10 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} export GDUMP="gdas" -export GDUMP_ENS="enkf${GDUMP}" - -# TODO: use ${RUN} rather than ${RUN//enkf/} once https://github.com/NOAA-EMC/global-workflow/issues/3767 is fixed -export OPREFIX="${RUN//enkf/}.t${cyc}z." ec_offset=${NMEM_ENS_GFS_OFFSET:-20} export NMEM_ENS_MAX=${NMEM_ENS} -if [[ "${RUN//enkf/}" == "gfs" && "${DOHYBVAR_OCN}" == "YES" ]]; then +if [[ "${RUN/enkf}" == "gfs" && "${DOHYBVAR_OCN}" == "YES" ]]; then NMEM_ENS=${NMEM_ENS_GFS} export mem_offset=$((ec_offset * cyc/6)) else @@ -37,7 +32,6 @@ RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -# TODO: remove explicit RUN specification once https://github.com/NOAA-EMC/global-workflow/issues/3767 is fixed RUN="${GDUMP}" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OBS:COM_OBS_TMPL \ @@ -45,8 +39,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \ COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL -# TODO: remove explicit RUN specification once https://github.com/NOAA-EMC/global-workflow/issues/3767 is fixed -RUN=${GDUMP_ENS} YMD=${PDY} HH=${cyc} MEMDIR="ensstat" declare_from_tmpl -rx \ +YMD=${PDY} HH=${cyc} MEMDIR="ensstat" declare_from_tmpl -rx \ COMOUT_CONF:COM_CONF_TMPL if [[ ! -d ${COMOUT_CONF} ]]; then mkdir -p "${COMOUT_CONF}"; fi @@ -78,13 +71,11 @@ fi ########################################## # Remove the Temporary working directory ########################################## -cd "${DATAROOT}" && true +# Use ${DATAROOT/enkf} because DATAjob is created by a gdas_marineanlinit job +cd "${DATAROOT/enkf}" && true export err=$? if [[ ${err} -ne 0 ]]; then err_exit fi -if [[ "${KEEPDATA}" == "NO" ]]; then - rm -rf "${DATA}" -fi exit 0 diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL b/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL index 18a872aa9a3..9a97107efa4 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL @@ -1,9 +1,9 @@ #! /usr/bin/env bash export WIPE_DATA="NO" -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATAjob="${DATAROOT}/marineanalysis.${PDY:-}${cyc}" export DATAens="${DATAjob}/ensdata" -export DATA="${DATAjob}/marinevariational" +export DATA="${DATAjob}/marineanlvar" source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlvar" -c "base marineanl marineanlvar" ############################################## diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 0e0c614d2d6..7d74fac16e2 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -1,7 +1,7 @@ #!/bin/bash export WIPE_DATA="NO" -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATAjob="${DATAROOT}/marineanalysis.${PDY:-}${cyc}" # DATA is pre-populated by the MARINE_BMAT_INIT job export DATA="${DATAjob}/marinebmat" # Create the directory to hold ensemble perturbations diff --git a/jobs/JGLOBAL_MARINE_BMAT_INITIALIZE b/jobs/JGLOBAL_MARINE_BMAT_INITIALIZE index 490fbf25152..043535ce044 100755 --- a/jobs/JGLOBAL_MARINE_BMAT_INITIALIZE +++ b/jobs/JGLOBAL_MARINE_BMAT_INITIALIZE @@ -1,7 +1,7 @@ #!/bin/bash -export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATAjob="${DATAROOT}/marineanalysis.${PDY:-}${cyc}" # Create the directory to hold the BMAT initialization data export DATA="${DATAjob}/marinebmat" # Create the directory to hold ensemble perturbations diff --git a/scripts/exgdas_global_marine_analysis_ecen.py b/scripts/exglobal_marine_analysis_ecen.py similarity index 100% rename from scripts/exgdas_global_marine_analysis_ecen.py rename to scripts/exglobal_marine_analysis_ecen.py diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index f09fc7f2972..88780bfc46f 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -57,6 +57,7 @@ def __init__(self, config: Dict) -> None: self.task_config.PARMsoca = os.path.join(self.task_config.PARMgfs, 'gdas', 'soca') self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS self.task_config.letkf_app = "true" + self.task_config.OPREFIX = f"{self.task_config.RUN.replace('enkf','')}.t{self.task_config.cyc:02d}z." @logit(logger) def initialize(self): From c30580e538640483f1a1c9abae85161a5fb74f3f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 18 Jul 2025 19:51:21 +0000 Subject: [PATCH 22/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 24e4e315f9d..55365d396fa 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 24e4e315f9d2c7f12efa79a364f568a5e11891c8 +Subproject commit 55365d396fa916e69a6c772e6c43390d7c4d84dc From bc4f43f89d0b5999aa15854b886190659f5a3f93 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 21 Jul 2025 12:58:23 +0000 Subject: [PATCH 23/39] change ocnanalecen to marineanlecen --- dev/jobs/marineanlecen.sh | 2 +- dev/parm/config/gcafs/config.resources | 4 ++-- dev/parm/config/gfs/config.marineanlecen | 11 +++++++++++ dev/parm/config/gfs/config.ocnanalecen | 11 ----------- dev/parm/config/gfs/config.resources | 4 ++-- dev/parm/config/gfs/config.resources.GAEAC6 | 2 +- dev/workflow/applications/gfs_cycled.py | 4 ++-- dev/workflow/rocoto/gfs_tasks.py | 14 +++++++------- env/GAEAC5.env | 4 ++-- env/GAEAC6.env | 4 ++-- env/HERA.env | 6 +++--- env/HERCULES.env | 4 ++-- env/ORION.env | 6 +++--- env/WCOSS2.env | 6 +++--- jobs/JGLOBAL_MARINE_ANALYSIS_ECEN | 4 ++-- parm/archive/gdas.yaml.j2 | 3 +++ 16 files changed, 46 insertions(+), 43 deletions(-) create mode 100644 dev/parm/config/gfs/config.marineanlecen delete mode 100644 dev/parm/config/gfs/config.ocnanalecen diff --git a/dev/jobs/marineanlecen.sh b/dev/jobs/marineanlecen.sh index 070afdc469d..596af184a2c 100755 --- a/dev/jobs/marineanlecen.sh +++ b/dev/jobs/marineanlecen.sh @@ -10,7 +10,7 @@ if [[ ${status} -ne 0 ]]; then exit "${status}" fi -export job="ocnanalecen" +export job="marineanlecen" export jobid="${job}.$$" ############################################################### diff --git a/dev/parm/config/gcafs/config.resources b/dev/parm/config/gcafs/config.resources index 674447c3986..1d32c979a4e 100644 --- a/dev/parm/config/gcafs/config.resources +++ b/dev/parm/config/gcafs/config.resources @@ -28,7 +28,7 @@ if (( $# != 1 )); then echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" echo "wavegempak waveawipsbulls waveawipsgridded" echo "postsnd awips gempak npoess" - echo "marineanlinit prepoceanobs marinebmat marineanlvar ocnanalecen marineanalletkf marineanlchkpt marineanlfinal ocnanalvrfy" + echo "marineanlinit prepoceanobs marinebmat marineanlvar marineanlecen marineanalletkf marineanlchkpt marineanlfinal ocnanalvrfy" exit 1 fi @@ -590,7 +590,7 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / 2 )) ;; - "ocnanalecen") + "marineanlecen") ntasks=16 case ${OCNRES} in "025") diff --git a/dev/parm/config/gfs/config.marineanlecen b/dev/parm/config/gfs/config.marineanlecen new file mode 100644 index 00000000000..5e59b6753c6 --- /dev/null +++ b/dev/parm/config/gfs/config.marineanlecen @@ -0,0 +1,11 @@ +#!/bin/bash + +########## config.marineanlecen ########## +# Ocn Analysis specific + +echo "BEGIN: config.marineanlecen" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlecen + +echo "END: config.marineanlecen" diff --git a/dev/parm/config/gfs/config.ocnanalecen b/dev/parm/config/gfs/config.ocnanalecen deleted file mode 100644 index b64c2bcf625..00000000000 --- a/dev/parm/config/gfs/config.ocnanalecen +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -########## config.ocnanalecen ########## -# Ocn Analysis specific - -echo "BEGIN: config.ocnanalecen" - -# Get task specific resources -. "${EXPDIR}/config.resources" ocnanalecen - -echo "END: config.ocnanalecen" diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources index 5dd708d5958..5a2714dce73 100644 --- a/dev/parm/config/gfs/config.resources +++ b/dev/parm/config/gfs/config.resources @@ -26,7 +26,7 @@ if (( $# != 1 )); then echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" echo "wavegempak waveawipsbulls waveawipsgridded" echo "postsnd awips gempak npoess" - echo "marineanlinit prepoceanobs marinebmatinit marinebmat marineanlvar ocnanalecen marineanalletkf marineanlchkpt marineanlfinal ocnanalvrfy" + echo "marineanlinit prepoceanobs marinebmatinit marinebmat marineanlvar marineanlecen marineanalletkf marineanlchkpt marineanlfinal ocnanalvrfy" exit 1 fi @@ -568,7 +568,7 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / 2 )) ;; - "ocnanalecen") + "marineanlecen") ntasks=16 walltime="00:10:00" case ${OCNRES} in diff --git a/dev/parm/config/gfs/config.resources.GAEAC6 b/dev/parm/config/gfs/config.resources.GAEAC6 index 18441a72aea..5e955796956 100644 --- a/dev/parm/config/gfs/config.resources.GAEAC6 +++ b/dev/parm/config/gfs/config.resources.GAEAC6 @@ -76,7 +76,7 @@ case ${step} in esac ;; - "ocnanalecen") + "marineanlecen") case ${OCNRES} in "025") ntasks=288 diff --git a/dev/workflow/applications/gfs_cycled.py b/dev/workflow/applications/gfs_cycled.py index b1fa137e69c..16c8b74bb94 100644 --- a/dev/workflow/applications/gfs_cycled.py +++ b/dev/workflow/applications/gfs_cycled.py @@ -119,7 +119,7 @@ def _get_app_configs(self, run): if options['do_letkf_ocn']: configs += ['marineanlletkf'] if options['do_hybvar']: - configs += ['ocnanalecen'] + configs += ['marineanlecen'] configs += ['marineanlchkpt', 'marineanlfinal'] if options['do_ocean'] or options['do_ice']: @@ -397,7 +397,7 @@ def get_task_names(self): if options['do_letkf_ocn']: task_names[run] += ['marineanlletkf'] if options['do_hybvar']: - task_names[run] += ['ocnanalecen'] + task_names[run] += ['marineanlecen'] task_names[run].append('esnowanl') if options['do_jedisnowda'] else 0 task_names[run].append('efcs') if 'gdas' in run else 0 diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index 8d1cebd7bcc..bf9d07e58f5 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -834,7 +834,7 @@ def marineanlvar(self): return task - def ocnanalecen(self): + def marineanlecen(self): # can run in parallel with marinebmat deps = [] @@ -842,14 +842,14 @@ def ocnanalecen(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('ocnanalecen') - task_name = f'{self.run}_ocnanalecen' + resources = self.get_resource('marineanlecen') + task_name = f'{self.run}_marineanlecen' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run, - 'command': f'{self.HOMEgfs}/dev/jobs/ocnanalecen.sh', + 'command': f'{self.HOMEgfs}/dev/jobs/marineanlecen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -865,7 +865,7 @@ def marineanlchkpt(self): dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlvar'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f"enkf{self.run}_ocnanalecen"} + dep_dict = {'type': 'task', 'name': f"enkf{self.run}_marineanlecen"} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_mergensst']: data = f'&ROTDIR;/{self.run}.@Y@m@d/@H/atmos/{self.run}.t@Hz.sfcanl.nc' @@ -2791,7 +2791,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.run}_esfc'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f'{self.run}_ocnanalecen'} + dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} @@ -2975,7 +2975,7 @@ def earc_tars(self): dep_dict = {'type': 'metatask', 'name': f'{self.run}_ecmn'} deps.append(rocoto.add_dependency(dep_dict)) if self._base.get('DOHYBVAR_OCN', True): - dep_dict = {'type': 'task', 'name': f'enkfgfs_ocnanalecen'} + dep_dict = {'type': 'task', 'name': f'enkfgfs_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'gfs_marineanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/env/GAEAC5.env b/env/GAEAC5.env index ab27864848a..b2db810b525 100755 --- a/env/GAEAC5.env +++ b/env/GAEAC5.env @@ -135,9 +135,9 @@ case ${step} in export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export APRUN_MARINEANLVAR="${APRUN_default}" ;; - "ocnanalecen") + "marineanlecen") - export APRUN_OCNANALECEN="${APRUN_default}" + export APRUN_MARINEANLECEN="${APRUN_default}" ;; "marineanlchkpt") diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 2a5b93d7ee9..4920f15695a 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -135,9 +135,9 @@ case ${step} in export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export APRUN_MARINEANLVAR="${APRUN_default}" ;; - "ocnanalecen") + "marineanlecen") - export APRUN_OCNANALECEN="${APRUN_default}" + export APRUN_MARINEANLECEN="${APRUN_default}" ;; "marineanlchkpt") diff --git a/env/HERA.env b/env/HERA.env index 065f5dbc469..37857b7fd53 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -154,10 +154,10 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then export APRUN_MARINEANLOBSSTATS="${launcher} -n 1" -elif [[ "${step}" = "ocnanalecen" ]]; then +elif [[ "${step}" = "marineanlecen" ]]; then - export NTHREADS_OCNANALECEN=${NTHREADSmax} - export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" + export NTHREADS_MARINEANLECEN=${NTHREADSmax} + export APRUN_MARINEANLECEN="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLECEN}" elif [[ "${step}" = "marineanlletkf" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 65d18b0cc39..bbbfb6e6a52 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -141,9 +141,9 @@ case ${step} in export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export APRUN_MARINEANLVAR="${APRUN_default}" ;; - "ocnanalecen") + "marineanlecen") - export APRUN_OCNANALECEN="${APRUN_default}" + export APRUN_MARINEANLECEN="${APRUN_default}" ;; "marineanlchkpt") diff --git a/env/ORION.env b/env/ORION.env index 8da8d43b338..18cf9f1ac81 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -150,10 +150,10 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then export APRUN_MARINEANLOBSSTATS="${launcher} -n 1" -elif [[ "${step}" = "ocnanalecen" ]]; then +elif [[ "${step}" = "marineanlecen" ]]; then - export NTHREADS_OCNANALECEN=${NTHREADSmax} - export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" + export NTHREADS_MARINEANLECEN=${NTHREADSmax} + export APRUN_MARINEANLECEN="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLECEN}" elif [[ "${step}" = "marineanlletkf" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 35d3fdfc889..4c4d1ce2975 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -126,10 +126,10 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then export APRUN_MARINEANLOBSSTATS="${launcher} -n 1" -elif [[ "${step}" = "ocnanalecen" ]]; then +elif [[ "${step}" = "marineanlecen" ]]; then - export NTHREADS_OCNANALECEN=${NTHREADSmax} - export APRUN_OCNANALECEN="${APRUN_default}" + export NTHREADS_MARINEANLECEN=${NTHREADSmax} + export APRUN_MARINEANLECEN="${APRUN_default}" elif [[ "${step}" = "marineanlletkf" ]]; then diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_ECEN b/jobs/JGLOBAL_MARINE_ANALYSIS_ECEN index d817d0462fa..9b409155000 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_ECEN +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_ECEN @@ -1,10 +1,10 @@ #!/bin/bash export DATAjob="${DATAROOT/enkf}/marineanalysis.${PDY:-}${cyc}" -export DATA="${DATAjob}/ocnanalecen" +export DATA="${DATAjob}/marineanlecen" export DATAens="${DATAjob}/ensdata" -source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalecen" -c "base marineanl ocnanalecen" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlecen" -c "base marineanl marineanlecen" ############################################## # Set variables used in the script diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index a1fac1b3718..74c8d3c7a0d 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -27,6 +27,9 @@ gdas: - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlvar.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlfinal.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlchkpt.log" + {% if DOHYBVAR_OCN %} + - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlecen.log" + {% endif %} {% endif %} {% if DO_AERO_ANL %} - "logs/{{ cycle_YMDH }}/gdas_aeroanlgenb.log" From 2b372de2da5356fed9ce355077d5928850568c98 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 21 Jul 2025 13:06:45 +0000 Subject: [PATCH 24/39] some ocnanalecen slipped through last commit --- dev/workflow/rocoto/tasks.py | 2 +- parm/archive/enkf.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/workflow/rocoto/tasks.py b/dev/workflow/rocoto/tasks.py index b9cfac6d697..4c6bae4ea29 100644 --- a/dev/workflow/rocoto/tasks.py +++ b/dev/workflow/rocoto/tasks.py @@ -19,7 +19,7 @@ class Tasks: 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prep_emissions', 'prepoceanobs', 'marineanlinit', 'marineanlletkf', 'marinebmatinit', 'marinebmat', 'marineanlvar', - 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', + 'marineanlecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', 'eobs', 'epos', 'esfc', 'eupd', 'earc_vrfy', 'earc_tars', 'ecen', 'echgres', 'ediag', 'efcs', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', 'atmos_ensstat', diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index 792f6cf83b1..4c1030da17a 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -35,7 +35,7 @@ enkf: {% endif %} {% endif %} {% if DOHYBVAR_OCN %} - {% set steps = steps + ["ocnanalecen"] %} + {% set steps = steps + ["marineanlecen"] %} {% endif %} {% for step in steps %} From 8694f3b149a85bed293d6fd5739aa3317332c4ba Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 21 Jul 2025 13:14:16 +0000 Subject: [PATCH 25/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 55365d396fa..bf3075c2b29 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 55365d396fa916e69a6c772e6c43390d7c4d84dc +Subproject commit bf3075c2b2998409fac13fa235614fd0a0be1d49 From 0510f72d3c1537e0e062e7978d62e19263d1d624 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 21 Jul 2025 14:10:10 +0000 Subject: [PATCH 26/39] Revert some things --- dev/workflow/rocoto/gfs_tasks.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index 1be032d0655..d43ff5f4231 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -705,9 +705,9 @@ def prepoceanobs(self): def marineanlletkf(self): deps = [] - dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf', '')}_prepoceanobs"} + dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf', '')}_marinebmat"} + dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmat'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -717,7 +717,7 @@ def marineanlletkf(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/dev/jobs/marineanlletkf.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -838,7 +838,7 @@ def marineanlecen(self): # can run in parallel with marinebmat deps = [] - dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf', '')}_marinebmatinit"} + dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmatinit'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -848,7 +848,7 @@ def marineanlecen(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/dev/jobs/marineanlecen.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -865,7 +865,7 @@ def marineanlchkpt(self): dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlvar'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f"enkf{self.run}_marineanlecen"} + dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_mergensst']: data = f'&ROTDIR;/{self.run}.@Y@m@d/@H/atmos/{self.run}.t@Hz.sfcanl.nc' @@ -2466,13 +2466,14 @@ def atmensanlinit(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + cycledef = "gdas" resources = self.get_resource('atmensanlinit') task_name = f'{self.run}_atmensanlinit' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': cycledef, 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlinit.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2498,7 +2499,7 @@ def atmensanlobs(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlobs.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2524,7 +2525,7 @@ def atmensanlsol(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlsol.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2550,7 +2551,7 @@ def atmensanlletkf(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlletkf.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2579,7 +2580,7 @@ def atmensanlfv3inc(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlfv3inc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2603,7 +2604,7 @@ def atmensanlfinal(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.run, + 'cycledef': self.run.replace('enkf', ''), 'command': f'{self.HOMEgfs}/dev/jobs/atmensanlfinal.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', @@ -2779,7 +2780,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.run}_esfc'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlecen'} + dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf", "")}_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} @@ -2963,7 +2964,7 @@ def earc_tars(self): dep_dict = {'type': 'metatask', 'name': f'{self.run}_ecmn'} deps.append(rocoto.add_dependency(dep_dict)) if self._base.get('DOHYBVAR_OCN', True): - dep_dict = {'type': 'task', 'name': f'enkfgfs_marineanlecen'} + dep_dict = {'type': 'task', 'name': f'gfs_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'gfs_marineanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) From 05ade1b5f480562aa9f12afa50cd63f35b18bd10 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 21 Jul 2025 15:23:04 +0000 Subject: [PATCH 27/39] Clean up --- dev/workflow/rocoto/gfs_tasks.py | 12 ++++++------ jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 1 + sorc/gdas.cd | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dev/workflow/rocoto/gfs_tasks.py b/dev/workflow/rocoto/gfs_tasks.py index d43ff5f4231..da5d238f18a 100644 --- a/dev/workflow/rocoto/gfs_tasks.py +++ b/dev/workflow/rocoto/gfs_tasks.py @@ -705,9 +705,9 @@ def prepoceanobs(self): def marineanlletkf(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} + dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf','')}_prepoceanobs"} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmat'} + dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf','')}_marinebmat"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) @@ -838,7 +838,7 @@ def marineanlecen(self): # can run in parallel with marinebmat deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}_marinebmatinit'} + dep_dict = {'type': 'task', 'name': f"{self.run.replace('enkf','')}_marinebmatinit"} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -865,7 +865,7 @@ def marineanlchkpt(self): dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlvar'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlecen'} + dep_dict = {'type': 'task', 'name': f'enkf{self.run}_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_mergensst']: data = f'&ROTDIR;/{self.run}.@Y@m@d/@H/atmos/{self.run}.t@Hz.sfcanl.nc' @@ -2780,7 +2780,7 @@ def efcs(self): dep_dict = {'type': 'task', 'name': f'{self.run}_esfc'} deps.append(rocoto.add_dependency(dep_dict)) if self.options['do_hybvar_ocn']: - dep_dict = {'type': 'task', 'name': f'{self.run.replace("enkf", "")}_marineanlecen'} + dep_dict = {'type': 'task', 'name': f'{self.run}_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) dep_dict = {'type': 'task', 'name': f'{self.run}_stage_ic'} @@ -2964,7 +2964,7 @@ def earc_tars(self): dep_dict = {'type': 'metatask', 'name': f'{self.run}_ecmn'} deps.append(rocoto.add_dependency(dep_dict)) if self._base.get('DOHYBVAR_OCN', True): - dep_dict = {'type': 'task', 'name': f'gfs_marineanlecen'} + dep_dict = {'type': 'task', 'name': f'enkfgfs_marineanlecen'} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'gfs_marineanlfinal'} deps.append(rocoto.add_dependency(dep_dict)) diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 74d843179ba..0c62809c5e0 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -18,6 +18,7 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} export GDUMP="gdas" +export GDUMP_ENS="enkf${GDUMP}" ec_offset=${NMEM_ENS_GFS_OFFSET:-20} export NMEM_ENS_MAX=${NMEM_ENS} diff --git a/sorc/gdas.cd b/sorc/gdas.cd index bf3075c2b29..1092971c357 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit bf3075c2b2998409fac13fa235614fd0a0be1d49 +Subproject commit 1092971c35717ea2bb230e8fd5ac30a2c1240071 From 1aed6ded46ac23b0023f0bec5d7ed22b27a601e1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Tue, 22 Jul 2025 09:53:43 -0400 Subject: [PATCH 28/39] Update dev/parm/config/gfs/config.marineanlecen Co-authored-by: Rahul Mahajan --- dev/parm/config/gfs/config.marineanlecen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/parm/config/gfs/config.marineanlecen b/dev/parm/config/gfs/config.marineanlecen index 5e59b6753c6..b5b53bbe4ff 100644 --- a/dev/parm/config/gfs/config.marineanlecen +++ b/dev/parm/config/gfs/config.marineanlecen @@ -6,6 +6,6 @@ echo "BEGIN: config.marineanlecen" # Get task specific resources -. "${EXPDIR}/config.resources" marineanlecen +source "${EXPDIR}/config.resources" marineanlecen echo "END: config.marineanlecen" From 7dd1745ab053df7f934af5eed9fcbc6a577fba61 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 22 Jul 2025 14:51:05 +0000 Subject: [PATCH 29/39] Address reviewer comments --- dev/parm/config/gfs/config.marineanl.j2 | 5 ----- dev/parm/config/gfs/config.marineanlecen | 4 ++++ dev/parm/config/gfs/config.marinebmat | 3 +++ dev/workflow/hosts/hera.yaml | 4 ++-- env/HERA.env | 3 +-- env/ORION.env | 3 +-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dev/parm/config/gfs/config.marineanl.j2 b/dev/parm/config/gfs/config.marineanl.j2 index d6231762aed..57c148b6402 100644 --- a/dev/parm/config/gfs/config.marineanl.j2 +++ b/dev/parm/config/gfs/config.marineanl.j2 @@ -5,9 +5,7 @@ echo "BEGIN: config.marineanl" -export JEDI_CONFIG_YAML_BMAT="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" export JEDI_CONFIG_YAML_ANALYSIS="${PARMgfs}/gdas/soca_analysis_jedi_config.yaml.j2" -export JEDI_CONFIG_YAML_ECEN="${PARMgfs}/gdas/soca/soca_ecen_jedi_config.yaml.j2" export JCB_ALGO_YAML_VAR="{{ JCB_ALGO_YAML_VAR }}" export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" @@ -20,12 +18,9 @@ export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yam export SOCA_FIX_YAML_TMPL="${PARMgfs}/gdas/soca/soca_fix_stage_${OCNRES}.yaml.j2" export MARINE_UTILITY_YAML_TMPL="${PARMgfs}/gdas/soca/soca_utils_stage.yaml.j2" export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_mem.yaml.j2" -export MARINE_BMAT_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_bmat_finalize.yaml.j2" export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export MARINE_DET_STAGE_FILES_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_files_stage.yaml.j2" export MARINE_DET_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_finalize.yaml.j2" -export MARINE_ECEN_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ecen_stage.yaml.j2" -export MARINE_ECEN_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ecen_finalize.yaml.j2" export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" export MARINE_JCB_GDAS_OBS="${PARMgfs}/gdas/jcb-gdas/observations/marine" diff --git a/dev/parm/config/gfs/config.marineanlecen b/dev/parm/config/gfs/config.marineanlecen index b5b53bbe4ff..f440bd423a9 100644 --- a/dev/parm/config/gfs/config.marineanlecen +++ b/dev/parm/config/gfs/config.marineanlecen @@ -8,4 +8,8 @@ echo "BEGIN: config.marineanlecen" # Get task specific resources source "${EXPDIR}/config.resources" marineanlecen +export JEDI_CONFIG_YAML_ECEN="${PARMgfs}/gdas/soca/soca_ecen_jedi_config.yaml.j2" +export MARINE_ECEN_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ecen_stage.yaml.j2" +export MARINE_ECEN_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ecen_finalize.yaml.j2" + echo "END: config.marineanlecen" diff --git a/dev/parm/config/gfs/config.marinebmat b/dev/parm/config/gfs/config.marinebmat index d88739dcedc..e5316c6fc2c 100644 --- a/dev/parm/config/gfs/config.marinebmat +++ b/dev/parm/config/gfs/config.marinebmat @@ -8,4 +8,7 @@ echo "BEGIN: config.marinebmat" # Get task specific resources . "${EXPDIR}/config.resources" marinebmat +export JEDI_CONFIG_YAML_BMAT="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" +export MARINE_BMAT_FINALIZE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_bmat_finalize.yaml.j2" + echo "END: config.marinebmat" diff --git a/dev/workflow/hosts/hera.yaml b/dev/workflow/hosts/hera.yaml index c6b0d15179e..05e3a0785fc 100644 --- a/dev/workflow/hosts/hera.yaml +++ b/dev/workflow/hosts/hera.yaml @@ -5,7 +5,7 @@ BASE_DATA: '/scratch3/NCEPDEV/global/role.glopara/data' BASE_IC: '/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR' AERO_INPUTS_DIR: '/scratch1/NCEPDEV/global/glopara/data/GEFS_ExtData/20250310' PACKAGEROOT: '/scratch3/NCEPDEV/global/role.glopara/nwpara' -HOMEDIR: '/scratch3/NCEPDEV/global/${USER}' +HOMEDIR: '/scratch3/NCEPDEV/da/${USER}' STMP: '/scratch3/NCEPDEV/stmp/${USER}' PTMP: '/scratch3/NCEPDEV/stmp/${USER}' NOSCRUB: '${HOMEDIR}' @@ -19,7 +19,7 @@ QUEUE: batch PARTITION_BATCH: hera PARTITION_SERVICE: service # HPSS properties -HPSS_PROJECT: emc-global +HPSS_PROJECT: emc-da ARCHCOM_TO: 'hpss' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' CHGRP_RSTPROD: 'YES' diff --git a/env/HERA.env b/env/HERA.env index 37857b7fd53..c1e3edd196a 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -156,8 +156,7 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then elif [[ "${step}" = "marineanlecen" ]]; then - export NTHREADS_MARINEANLECEN=${NTHREADSmax} - export APRUN_MARINEANLECEN="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLECEN}" + export APRUN_MARINEANLECEN=${APRUN_default} elif [[ "${step}" = "marineanlletkf" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 18cf9f1ac81..1cc6dde1777 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -152,8 +152,7 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then elif [[ "${step}" = "marineanlecen" ]]; then - export NTHREADS_MARINEANLECEN=${NTHREADSmax} - export APRUN_MARINEANLECEN="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLECEN}" + export APRUN_MARINEANLECEN=${APRUN_default} elif [[ "${step}" = "marineanlletkf" ]]; then From 15e60275af1de3868f15541267b3c2c25d3ce1b0 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 22 Jul 2025 14:52:04 +0000 Subject: [PATCH 30/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 1092971c357..a4584afdc65 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 1092971c35717ea2bb230e8fd5ac30a2c1240071 +Subproject commit a4584afdc65c098e21342b6ddb8cdffbd62532a7 From 15604cea0168e8998f7f605b06450194d6349d46 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 25 Jul 2025 13:15:08 +0000 Subject: [PATCH 31/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a4584afdc65..8a0fae56ff9 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a4584afdc65c098e21342b6ddb8cdffbd62532a7 +Subproject commit 8a0fae56ff90d64c8fa59be6162ce80ed2177ca3 From d05edc80d00c0282de66e18d895bf964d350125f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 25 Jul 2025 13:30:52 +0000 Subject: [PATCH 32/39] Update gdas hash to develop --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 8a0fae56ff9..ea5879c4ce5 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 8a0fae56ff90d64c8fa59be6162ce80ed2177ca3 +Subproject commit ea5879c4ce5a29bf73f9e48cf2588878676a108a From 567cbfee8584198ea2b073952a048ad5c97e4c85 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Fri, 25 Jul 2025 12:12:25 -0400 Subject: [PATCH 33/39] Accidentally updated hera.yaml --- dev/workflow/hosts/hera.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/workflow/hosts/hera.yaml b/dev/workflow/hosts/hera.yaml index 05e3a0785fc..c6b0d15179e 100644 --- a/dev/workflow/hosts/hera.yaml +++ b/dev/workflow/hosts/hera.yaml @@ -5,7 +5,7 @@ BASE_DATA: '/scratch3/NCEPDEV/global/role.glopara/data' BASE_IC: '/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR' AERO_INPUTS_DIR: '/scratch1/NCEPDEV/global/glopara/data/GEFS_ExtData/20250310' PACKAGEROOT: '/scratch3/NCEPDEV/global/role.glopara/nwpara' -HOMEDIR: '/scratch3/NCEPDEV/da/${USER}' +HOMEDIR: '/scratch3/NCEPDEV/global/${USER}' STMP: '/scratch3/NCEPDEV/stmp/${USER}' PTMP: '/scratch3/NCEPDEV/stmp/${USER}' NOSCRUB: '${HOMEDIR}' @@ -19,7 +19,7 @@ QUEUE: batch PARTITION_BATCH: hera PARTITION_SERVICE: service # HPSS properties -HPSS_PROJECT: emc-da +HPSS_PROJECT: emc-global ARCHCOM_TO: 'hpss' ATARDIR: '/NCEPDEV/${HPSS_PROJECT}/1year/${USER}/${machine}/scratch/${PSLOT}' CHGRP_RSTPROD: 'YES' From 18bf2a1bc6d0220ca5fa8326e904005b2e17a941 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 25 Jul 2025 16:34:49 +0000 Subject: [PATCH 34/39] Missing config var --- dev/parm/config/gfs/config.marinebmatinit | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/parm/config/gfs/config.marinebmatinit b/dev/parm/config/gfs/config.marinebmatinit index e42d10c7587..7e82ed33c53 100644 --- a/dev/parm/config/gfs/config.marinebmatinit +++ b/dev/parm/config/gfs/config.marinebmatinit @@ -8,4 +8,6 @@ echo "BEGIN: config.marinebmatinit" # Get task specific resources source "${EXPDIR}/config.resources" marinebmatinit +export JEDI_CONFIG_YAML_BMAT="${PARMgfs}/gdas/soca_bmat_jedi_config.yaml.j2" + echo "END: config.marinebmatinit" From 096d89902878c85fd9e225330317a49879046695 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 28 Jul 2025 13:27:26 +0000 Subject: [PATCH 35/39] Fix archiving issue --- parm/archive/gdas.yaml.j2 | 3 --- 1 file changed, 3 deletions(-) diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 74c8d3c7a0d..a1fac1b3718 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -27,9 +27,6 @@ gdas: - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlvar.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlfinal.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlchkpt.log" - {% if DOHYBVAR_OCN %} - - "logs/{{ cycle_YMDH }}/{{ RUN }}_marineanlecen.log" - {% endif %} {% endif %} {% if DO_AERO_ANL %} - "logs/{{ cycle_YMDH }}/gdas_aeroanlgenb.log" From 758969edc22e9ba26bfd915cd1b967923d97e285 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 28 Jul 2025 18:55:33 +0000 Subject: [PATCH 36/39] Update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index ea5879c4ce5..41a5edc4a13 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit ea5879c4ce5a29bf73f9e48cf2588878676a108a +Subproject commit 41a5edc4a135f5b25e469e0650a4e857fa00b219 From 01ecb58afb9bf0c017b7255a334837f2a61bdc59 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 29 Jul 2025 14:22:29 +0000 Subject: [PATCH 37/39] Try something --- dev/jobs/{marineanlecen.sh => ocnanalecen.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dev/jobs/{marineanlecen.sh => ocnanalecen.sh} (100%) diff --git a/dev/jobs/marineanlecen.sh b/dev/jobs/ocnanalecen.sh similarity index 100% rename from dev/jobs/marineanlecen.sh rename to dev/jobs/ocnanalecen.sh From ec7c3bd2987ed3c9ef520dab761e4cc9c451e077 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 29 Jul 2025 14:24:17 +0000 Subject: [PATCH 38/39] Try something --- dev/jobs/{ocnanalecen.sh => marineanlecen.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dev/jobs/{ocnanalecen.sh => marineanlecen.sh} (100%) diff --git a/dev/jobs/ocnanalecen.sh b/dev/jobs/marineanlecen.sh similarity index 100% rename from dev/jobs/ocnanalecen.sh rename to dev/jobs/marineanlecen.sh From 1fcd43725094b1cbb77fbeae8878c1b683183508 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 29 Jul 2025 17:39:30 +0000 Subject: [PATCH 39/39] apply patch --- dev/workflow/build_opts.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/workflow/build_opts.yaml b/dev/workflow/build_opts.yaml index 2d983d08698..d90fd8662d3 100644 --- a/dev/workflow/build_opts.yaml +++ b/dev/workflow/build_opts.yaml @@ -36,7 +36,7 @@ build: gfs_model: command: "./build_ufs.sh -e gfs_model.x" cores: 12 - walltime: "00:30:00" + walltime: "00:40:00" gfs_ww3prepost: command: "./build_ww3prepost.sh" @@ -46,7 +46,7 @@ build: gefs_model: command: "./build_ufs.sh -w -e gefs_model.x" cores: 12 - walltime: "00:30:00" + walltime: "00:40:00" gefs_ww3_prepost: command: "./build_ww3prepost.sh -w" @@ -56,12 +56,12 @@ build: sfs_model: command: "./build_ufs.sh -y -e sfs_model.x" cores: 12 - walltime: "00:30:00" + walltime: "00:40:00" gcafs_model: command: "./build_ufs.sh -a ATMAERO -e gcafs_model.x" cores: 12 - walltime: "00:30:00" + walltime: "00:40:00" upp: command: "./build_upp.sh"