From b52379f44fd981382225d8de8de0e14c626aee08 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 16 Jan 2025 19:21:31 +0000 Subject: [PATCH 1/7] old work --- parm/config/gfs/config.base | 3 +- ush/python/pygfs/task/marine_analysis.py | 1 + ush/python/pygfs/task/marine_letkf.py | 72 +++++++++++++++++++++--- workflow/hosts/hera.yaml | 6 +- 4 files changed, 71 insertions(+), 11 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 4f702f96683..6e99512ba04 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -105,7 +105,8 @@ export NCP="/bin/cp -p" export NMV="/bin/mv" export NLN="/bin/ln -sf" export VERBOSE="YES" -export KEEPDATA="NO" +#export KEEPDATA="NO" +export KEEPDATA="YES" export DEBUG_POSTSCRIPT="NO" # PBS only; sets debug=true export CHGRP_RSTPROD="@CHGRP_RSTPROD@" export CHGRP_CMD="@CHGRP_CMD@" diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 75cc28c7b30..d6770779bdf 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -211,6 +211,7 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER envconfig_jcb['obs_list'] = ['adt_rads_all'] envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES)) + print("print envconfig_jcb: ", envconfig_jcb) # Write obs_list_short save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 54d40f8d665..233338bfcd9 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -1,18 +1,24 @@ #!/usr/bin/env python3 +import copy import f90nml import pygfs.utils.marine_da_utils as mdau from logging import getLogger import os from pygfs.task.analysis import Analysis from typing import Dict -from wxflow import (AttrDict, +from wxflow import (add_to_datetime, + AttrDict, Executable, FileHandler, logit, parse_j2yaml, + save_as_yaml, + Template, + TemplateConstants, to_timedelta, - to_YMDH) + to_YMDH, + YAMLFile) logger = getLogger(__name__.split('.')[-1]) @@ -37,6 +43,7 @@ def __init__(self, config: Dict) -> None: logger.info("init") super().__init__(config) + _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) _half_assim_freq = to_timedelta(f"{self.task_config.assim_freq}H") / 2 _letkf_yaml_file = 'letkf.yaml' _letkf_exec_args = [self.task_config.MARINE_LETKF_EXEC, @@ -44,16 +51,23 @@ def __init__(self, config: Dict) -> None: 'localensembleda', _letkf_yaml_file] # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert - _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) + self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle - self.task_config.WINDOW_BEGIN = self.task_config.current_cycle - _half_assim_freq + self.task_config.MARINE_WINDOW_MIDDLE = self.task_config.current_cycle + self.task_config.WINDOW_BEGIN = _window_begin + self.task_config.WINDOW_END = _window_end + self.task_config.MARINE_WINDOW_BEGIN = _window_begin + self.task_config.MARINE_WINDOW_END = _window_end self.task_config.letkf_exec_args = _letkf_exec_args self.task_config.letkf_yaml_file = _letkf_yaml_file self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') self.task_config.ENSPERT_RELPATH = _enspert_relpath + self.task_config.PARMsoca = os.path.join(self.task_config.PARMgfs, 'gdas', 'soca') @logit(logger) def initialize(self): @@ -131,9 +145,53 @@ def initialize(self): FileHandler({'copy': obs_files_to_copy}).sync() # make the letkf.yaml - letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) - letkf_yaml.observations.observers = obs_to_use - letkf_yaml.save(self.task_config.letkf_yaml_file) + # letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) + # letkf_yaml.observations.observers = obs_to_use + # letkf_yaml.save(self.task_config.letkf_yaml_file) + +#################################################################################################### + # Write obs_list_short +# save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') + save_as_yaml(os.path.join(self.task_config.PARMsoca, 'letkf','letkf_obs_list.yaml.j2'), 'obs_list_short.yaml') + os.environ['OBS_LIST_SHORT'] = 'obs_list_short.yaml' + + print("self.task_config: ", self.task_config) + envconfig_jcb = copy.deepcopy(self.task_config) +# envconfig_jcb['cyc'] = int(self.task_config.current_cycle.strftime('%H')) + envconfig_jcb['cyc'] = int(os.getenv('cyc')) + envconfig_jcb['PDY'] = self.task_config.current_cycle.strftime('%Y%m%d') + print("PDY: ", envconfig_jcb['PDY']) + print("type of cyc: ", type(envconfig_jcb['cyc'])) + #print("envconfig_jcb: ", envconfig_jcb) + # Render the JCB configuration files + jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') + jcb_algo_yaml = os.path.join(self.task_config.PARMsoca, 'letkf/marine-jcb-lektf.yaml.j2') + + jcb_base_config = YAMLFile(path=jcb_base_yaml) + jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) + jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) + jcb_algo_config = YAMLFile(path=jcb_algo_yaml) + jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) + jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) + + # Override base with the application specific config + jcb_config = {**jcb_base_config, **jcb_algo_config} + + # convert datetime to string + jcb_config['window_begin'] = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') + jcb_config['window_middle'] = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') + + # Current hack so that this is not done directly in the JCB base yaml +# jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') + + # Render the full JEDI configuration file using JCB + jedi_config = render(jcb_config) + + # Save the JEDI configuration file + letkf_yaml_jcb = 'letkf.yaml' + mdau.clean_empty_obsspaces(jedi_config, target=letkf_yaml_jcb, app='var') + + ###################################### # swap date and stack size in mom_input.nml domain_stack_size = self.task_config.DOMAIN_STACK_SIZE diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index e9e749ad3c4..f87ca0c40ad 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -8,10 +8,10 @@ HOMEDIR: '/scratch1/NCEPDEV/global/${USER}' STMP: '/scratch1/NCEPDEV/stmp2/${USER}' PTMP: '/scratch1/NCEPDEV/stmp4/${USER}' NOSCRUB: $HOMEDIR -ACCOUNT: fv3-cpu +ACCOUNT: da-cpu SCHEDULER: slurm -QUEUE: batch -QUEUE_SERVICE: batch +QUEUE: debug +QUEUE_SERVICE: debug PARTITION_BATCH: hera PARTITION_SERVICE: service RESERVATION: '' From a03d8e4af48033c3288c91496ad9da86a47844c3 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 30 Jan 2025 20:06:44 +0000 Subject: [PATCH 2/7] made jcb letkf config work --- ush/python/pygfs/task/marine_letkf.py | 36 +++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 999f391b22b..a5104bcb131 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -2,10 +2,11 @@ import copy import f90nml -import pygfs.utils.marine_da_utils as mdau +from jcb import render from logging import getLogger import os from pygfs.task.analysis import Analysis +import pygfs.utils.marine_da_utils as mdau from typing import Dict from wxflow import (add_to_datetime, AttrDict, @@ -141,36 +142,34 @@ def initialize(self): else: logger.warning(f"{obs_file} is not available in {self.task_config.COMIN_OBS}") + print("obs_to_use: ", obs_to_use) + observers = {'observers': obs_to_use} + print("observers: ", observers) + # stage the desired obs files FileHandler({'copy': obs_files_to_copy}).sync() - # make the letkf.yaml - # letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) - # letkf_yaml.observations.observers = obs_to_use - # letkf_yaml.save(self.task_config.letkf_yaml_file) - #################################################################################################### # Write obs_list_short -# save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') save_as_yaml(os.path.join(self.task_config.PARMsoca, 'letkf','letkf_obs_list.yaml.j2'), 'obs_list_short.yaml') os.environ['OBS_LIST_SHORT'] = 'obs_list_short.yaml' - print("self.task_config: ", self.task_config) + # print("self.task_config: ", self.task_config) envconfig_jcb = copy.deepcopy(self.task_config) # envconfig_jcb['cyc'] = int(self.task_config.current_cycle.strftime('%H')) envconfig_jcb['cyc'] = int(os.getenv('cyc')) envconfig_jcb['PDY'] = self.task_config.current_cycle.strftime('%Y%m%d') - print("PDY: ", envconfig_jcb['PDY']) - print("type of cyc: ", type(envconfig_jcb['cyc'])) - #print("envconfig_jcb: ", envconfig_jcb) + envconfig_jcb['window_length'] = f"PT{self.task_config['assim_freq']}H" + # Render the JCB configuration files jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') jcb_algo_yaml = os.path.join(self.task_config.PARMsoca, 'letkf/marine-jcb-lektf.yaml.j2') - jcb_base_config = YAMLFile(path=jcb_base_yaml) + jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=envconfig_jcb) jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) - jcb_algo_config = YAMLFile(path=jcb_algo_yaml) +# jcb_algo_config = YAMLFile(path=jcb_algo_yaml) + jcb_algo_config = parse_j2yaml(path=jcb_algo_yaml, data=envconfig_jcb) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) @@ -180,16 +179,17 @@ def initialize(self): # convert datetime to string jcb_config['window_begin'] = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') jcb_config['window_middle'] = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') - - # Current hack so that this is not done directly in the JCB base yaml -# jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') - + jcb_config['window_length'] = f"PT{self.task_config['assim_freq']}H" + # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) + jedi_config['observations'] = observers # Save the JEDI configuration file letkf_yaml_jcb = 'letkf.yaml' - mdau.clean_empty_obsspaces(jedi_config, target=letkf_yaml_jcb, app='var') + # TODO (AFE) - is this needed? will require addition of letkf case +# mdau.clean_empty_obsspaces(jedi_config, target=letkf_yaml_jcb, app='var') + save_as_yaml(jedi_config, letkf_yaml_jcb) ###################################### From eef23c6131973081b85555f6fc44b805c68bda0b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 6 Feb 2025 20:29:46 +0000 Subject: [PATCH 3/7] wrap up --- ush/python/pygfs/task/marine_analysis.py | 1 - ush/python/pygfs/task/marine_letkf.py | 72 +++++++++++------------- 2 files changed, 32 insertions(+), 41 deletions(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 77e8e3f33da..d4ee08ba329 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -219,7 +219,6 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['RUN'] = self.task_config.RUN envconfig_jcb['current_cycle'] = self.task_config.current_cycle envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES).zfill(3)) - print("print envconfig_jcb: ", envconfig_jcb) # Write obs_list_short save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index a5104bcb131..8da0d032ea5 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -81,43 +81,43 @@ def initialize(self): None """ + # NOTE: This task requires the ensemble background to be staged, which is done in the + # BMAT task (which is a dependency of this task) + logger.info("initialize") # make directories and stage ensemble background files soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - stageconf = AttrDict() - keys = ['current_cycle', - 'previous_cycle', - 'COM_ICE_LETKF_TMPL', - 'COM_OCEAN_LETKF_TMPL', - 'COM_ICE_HISTORY_TMPL', - 'COM_OCEAN_HISTORY_TMPL', - 'COMIN_OCEAN_HISTORY_PREV', - 'COMIN_ICE_HISTORY_PREV', - 'COMOUT_ICE_LETKF', - 'COMOUT_OCEAN_LETKF', - 'DATA', - 'ENSPERT_RELPATH', - 'GDUMP_ENS', - 'NMEM_ENS', - 'OPREFIX', - 'PARMgfs', - 'ROTDIR', - 'RUN', - 'WINDOW_BEGIN', - 'WINDOW_MIDDLE'] - for key in keys: - stageconf[key] = self.task_config[key] - - # stage ensemble background files - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconf) - FileHandler(soca_ens_bkg_stage_list).sync() - - # stage letkf-specific files - letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) + # stageconf = AttrDict() + # keys = ['current_cycle', + # 'previous_cycle', + # 'COM_ICE_LETKF_TMPL', + # 'COM_OCEAN_LETKF_TMPL', + # 'COM_ICE_HISTORY_TMPL', + # 'COM_OCEAN_HISTORY_TMPL', + # 'COMIN_OCEAN_HISTORY_PREV', + # 'COMIN_ICE_HISTORY_PREV', + # 'COMOUT_ICE_LETKF', + # 'COMOUT_OCEAN_LETKF', + # 'DATA', + # 'ENSPERT_RELPATH', + # 'GDUMP_ENS', + # 'NMEM_ENS', + # 'OPREFIX', + # 'PARMgfs', + # 'ROTDIR', + # 'RUN', + # 'WINDOW_BEGIN', + # 'WINDOW_MIDDLE'] + # for key in keys: + # stageconf[key] = self.task_config[key] + + # stage letkf-specific files and directories +# letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) FileHandler(letkf_stage_list).sync() - + print("MARINE_OBS_LIST_YAML:", self.task_config.MARINE_OBS_LIST_YAML) obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) # get the list of observations @@ -142,21 +142,15 @@ def initialize(self): else: logger.warning(f"{obs_file} is not available in {self.task_config.COMIN_OBS}") - print("obs_to_use: ", obs_to_use) + # set aside for LETKF configuration observers = {'observers': obs_to_use} - print("observers: ", observers) # stage the desired obs files FileHandler({'copy': obs_files_to_copy}).sync() #################################################################################################### - # Write obs_list_short - save_as_yaml(os.path.join(self.task_config.PARMsoca, 'letkf','letkf_obs_list.yaml.j2'), 'obs_list_short.yaml') - os.environ['OBS_LIST_SHORT'] = 'obs_list_short.yaml' - # print("self.task_config: ", self.task_config) envconfig_jcb = copy.deepcopy(self.task_config) -# envconfig_jcb['cyc'] = int(self.task_config.current_cycle.strftime('%H')) envconfig_jcb['cyc'] = int(os.getenv('cyc')) envconfig_jcb['PDY'] = self.task_config.current_cycle.strftime('%Y%m%d') envconfig_jcb['window_length'] = f"PT{self.task_config['assim_freq']}H" @@ -187,8 +181,6 @@ def initialize(self): # Save the JEDI configuration file letkf_yaml_jcb = 'letkf.yaml' - # TODO (AFE) - is this needed? will require addition of letkf case -# mdau.clean_empty_obsspaces(jedi_config, target=letkf_yaml_jcb, app='var') save_as_yaml(jedi_config, letkf_yaml_jcb) ###################################### From 42e2d31f2d3061ad0d310c8b411e7d036112d2a3 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 24 Feb 2025 20:48:56 +0000 Subject: [PATCH 4/7] cleanup --- parm/config/gfs/config.base | 3 +-- ush/python/pygfs/task/marine_letkf.py | 27 --------------------------- workflow/hosts/hera.yaml | 6 +++--- 3 files changed, 4 insertions(+), 32 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 309d4820948..586029f785a 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -106,8 +106,7 @@ export NCP="/bin/cp -p" export NMV="/bin/mv" export NLN="/bin/ln -sf" export VERBOSE="YES" -#export KEEPDATA="NO" -export KEEPDATA="YES" +export KEEPDATA="NO" export DEBUG_POSTSCRIPT="NO" # PBS only; sets debug=true export CHGRP_RSTPROD="@CHGRP_RSTPROD@" export CHGRP_CMD="@CHGRP_CMD@" diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 8da0d032ea5..69812beda3f 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -89,35 +89,9 @@ def initialize(self): # make directories and stage ensemble background files soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - # stageconf = AttrDict() - # keys = ['current_cycle', - # 'previous_cycle', - # 'COM_ICE_LETKF_TMPL', - # 'COM_OCEAN_LETKF_TMPL', - # 'COM_ICE_HISTORY_TMPL', - # 'COM_OCEAN_HISTORY_TMPL', - # 'COMIN_OCEAN_HISTORY_PREV', - # 'COMIN_ICE_HISTORY_PREV', - # 'COMOUT_ICE_LETKF', - # 'COMOUT_OCEAN_LETKF', - # 'DATA', - # 'ENSPERT_RELPATH', - # 'GDUMP_ENS', - # 'NMEM_ENS', - # 'OPREFIX', - # 'PARMgfs', - # 'ROTDIR', - # 'RUN', - # 'WINDOW_BEGIN', - # 'WINDOW_MIDDLE'] - # for key in keys: - # stageconf[key] = self.task_config[key] - # stage letkf-specific files and directories -# letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) FileHandler(letkf_stage_list).sync() - print("MARINE_OBS_LIST_YAML:", self.task_config.MARINE_OBS_LIST_YAML) obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) # get the list of observations @@ -162,7 +136,6 @@ def initialize(self): jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=envconfig_jcb) jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) -# jcb_algo_config = YAMLFile(path=jcb_algo_yaml) jcb_algo_config = parse_j2yaml(path=jcb_algo_yaml, data=envconfig_jcb) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get) jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get) diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 028406c5b2f..b95d0abb8d1 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -8,10 +8,10 @@ HOMEDIR: '/scratch1/NCEPDEV/global/${USER}' STMP: '/scratch1/NCEPDEV/stmp2/${USER}' PTMP: '/scratch1/NCEPDEV/stmp4/${USER}' NOSCRUB: $HOMEDIR -ACCOUNT: da-cpu +ACCOUNT: fv3-cpu SCHEDULER: slurm -QUEUE: debug -QUEUE_SERVICE: debug +QUEUE: batch +QUEUE_SERVICE: batch PARTITION_BATCH: hera PARTITION_SERVICE: service RESERVATION: '' From 0ad7a2d17b78a922f6c6067632145bcd58b6cafb Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 25 Feb 2025 16:04:56 +0000 Subject: [PATCH 5/7] style --- ush/python/pygfs/task/marine_letkf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 69812beda3f..5d29200567d 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -55,7 +55,6 @@ def __init__(self, config: Dict) -> None: _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2) - self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle self.task_config.MARINE_WINDOW_MIDDLE = self.task_config.current_cycle self.task_config.WINDOW_BEGIN = _window_begin @@ -147,7 +146,7 @@ def initialize(self): jcb_config['window_begin'] = self.task_config.MARINE_WINDOW_BEGIN.strftime('%Y-%m-%dT%H:%M:%SZ') jcb_config['window_middle'] = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') jcb_config['window_length'] = f"PT{self.task_config['assim_freq']}H" - + # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) jedi_config['observations'] = observers From 087c8629cded8e07d5341ea1f992c51e517de332 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 17 Mar 2025 15:43:47 +0000 Subject: [PATCH 6/7] jcb wrangling --- ush/python/pygfs/task/marine_letkf.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 89c0bb27e4d..91c3f4fe920 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -5,6 +5,7 @@ from jcb import render from logging import getLogger import os +from pygfs.jedi import Jedi from pygfs.task.analysis import Analysis import pygfs.utils.marine_da_utils as mdau from typing import Dict @@ -69,6 +70,16 @@ def __init__(self, config: Dict) -> None: self.task_config.ENSPERT_RELPATH = _enspert_relpath self.task_config.PARMsoca = os.path.join(self.task_config.PARMgfs, 'gdas', 'soca') + # Create dictionary of JEDI objects + expected_keys = ['marineanlletkf'] + self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) + self.jedi_dict.PARMgfs = self.task_config.PARMgfs + #self.jedi_dict.MARINE_WINDOW_BEGIN = _window_begin.strftime('%Y-%m-%dT%H:%M:%SZ') + self.jedi_dict.MARINE_WINDOW_BEGIN = _window_begin + self.jedi_dict.MARINE_WINDOW_END = _window_end + print("jedi_dict: ",self.jedi_dict) + + @logit(logger) def initialize(self): """Method initialize for ocean and sea ice LETKF task @@ -123,6 +134,13 @@ def initialize(self): #################################################################################################### + + # initialize JEDI LETKF observer application + #self.task_config['jcb_algo'] = 'marine_letkf' + logger.info(f"Initializing JEDI LETKF observer application") + #self.jedi_dict['marineanlletkf'].initialize(self.task_config) + self.jedi_dict['marineanlletkf'].initialize(self.jedi_dict) + envconfig_jcb = copy.deepcopy(self.task_config) envconfig_jcb['cyc'] = int(os.getenv('cyc')) envconfig_jcb['PDY'] = self.task_config.current_cycle.strftime('%Y%m%d') From bf9cc9db6237526f0c144d052b3c58314de30819 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 25 Mar 2025 15:43:10 +0000 Subject: [PATCH 7/7] change window length --- ush/python/pygfs/task/marine_letkf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 91c3f4fe920..965bc18f241 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -77,6 +77,8 @@ def __init__(self, config: Dict) -> None: #self.jedi_dict.MARINE_WINDOW_BEGIN = _window_begin.strftime('%Y-%m-%dT%H:%M:%SZ') self.jedi_dict.MARINE_WINDOW_BEGIN = _window_begin self.jedi_dict.MARINE_WINDOW_END = _window_end + #self.jedi_dict.MARINE_WINDOW_LENGTH = _window_end - _window_begin + self.jedi_dict.MARINE_WINDOW_LENGTH = f"PT{self.task_config['assim_freq']}H" print("jedi_dict: ",self.jedi_dict)