From 8a8634b803305d2207ea85af9b1f24d2a0ab3c18 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Sat, 29 Mar 2025 11:41:15 +0000 Subject: [PATCH 1/9] jcbify the obs list --- ci/cases/gfsv17/marine3dvar.yaml | 2 +- parm/config/gfs/config.marineanl | 1 + parm/config/gfs/yaml/defaults.yaml | 4 +- ush/python/pygfs/task/marine_analysis.py | 51 ++++++++++----------- ush/python/pygfs/task/marine_letkf.py | 57 +++++++++++++++++------- 5 files changed, 69 insertions(+), 46 deletions(-) diff --git a/ci/cases/gfsv17/marine3dvar.yaml b/ci/cases/gfsv17/marine3dvar.yaml index c2884992b63..8cd94928b62 100644 --- a/ci/cases/gfsv17/marine3dvar.yaml +++ b/ci/cases/gfsv17/marine3dvar.yaml @@ -23,5 +23,5 @@ prepoceanobs: marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca - SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml + SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml.j2 SOCA_NINNER: 100 diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index e3dcc4529ea..26378ba427b 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -20,5 +20,6 @@ export MARINE_UTILITY_YAML_TMPL="${PARMgfs}/gdas/soca/soca_utils_stage.yaml.j2" export MARINE_ENSDA_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/ensda/stage_ens_mem.yaml.j2" export MARINE_DET_STAGE_BKG_YAML_TMPL="${PARMgfs}/gdas/soca/soca_det_bkg_stage.yaml.j2" export MARINE_JCB_GDAS_ALGO="${PARMgfs}/gdas/jcb-gdas/algorithm/marine" +export MARINE_JCB_GDAS_OBS="${PARMgfs}/gdas/jcb-gdas/observations/marine" echo "END: config.marineanl" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 272be765a6c..5e31bffdc96 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -55,13 +55,13 @@ snowanl: marineanl: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_ANL_GEOM: "${FIXgfs}/gdas/soca/72x35x25/soca" - SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs + SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml.j2" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/soca/marine-jcb-3dfgat.yaml.j2" prepoceanobs: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" - SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in ocnanal + SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml.j2" # TODO: This is also repeated in ocnanal OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml" use_exp_obs: "YES" dmpdir_exp: "${BASE_DATA}/experimental_obs" diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index bbce30c6369..fa6d09a6929 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -11,6 +11,7 @@ import subprocess import yaml from jcb import render +from pprint import pprint from wxflow import (AttrDict, FileHandler, @@ -25,21 +26,6 @@ logger = getLogger(__name__.split('.')[-1]) -def parse_obs_list_file(obs_list_yaml_path): - # Get the list of observation types from the obs_list.yaml - obs_types = [] - with open(obs_list_yaml_path, 'r') as file: - for line in file: - # Remove leading/trailing whitespace and check if the line is uncommented - line = line.strip() - if line.startswith('- !INC') and not line.startswith('#'): - # Extract the type using regex - match = re.search(r'\$\{MARINE_OBS_YAML_DIR\}/(.+)\.yaml', line) - if match: - obs_types.append(str(match.group(1))) - return obs_types - - class MarineAnalysis(Task): """ Class for global marine analysis tasks @@ -134,16 +120,25 @@ def _fetch_observations(self: Task) -> None: """ # get the list of observations - obs_list_config = YAMLFile(self.task_config.MARINE_OBS_LIST_YAML) - obs_list_config = Template.substitute_structure(obs_list_config, TemplateConstants.DOLLAR_PARENTHESES, self.task_config) - obs_list_config = {'observations': obs_list_config} - logger.info(f"{obs_list_config}") + + # "observations" is expected by later JCB code to populate it with config info, + # but the obs_list as such is needed later + self.task_config.observations = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config)['observations'] + self.task_config.obs_list = self.task_config.observations + + # also expected by JCB + self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS + + obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') + self.task_config.observations = parse_j2yaml(obsconfigfile, self.task_config) obs_files = [] - for ob in obs_list_config['observations']['observers']: - logger.info(f"******** {self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") - obs_files.append(f"{self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") - obs_list = [] + + for observer in self.task_config['observations']['observers']: + logger.info(f"******** {self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") + obs_files.append(f"{self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") + + obs_files_to_copy = [] # copy obs from COM_OBS to DATA/obs for obs_file in obs_files: @@ -153,11 +148,11 @@ def _fetch_observations(self: Task) -> None: logger.info(f"******* {obs_src}") if os.path.exists(obs_src): logger.info(f"******* fetching {obs_file}") - obs_list.append([obs_src, obs_dst]) + obs_files_to_copy.append([obs_src, obs_dst]) else: logger.info(f"******* {obs_file} is not in the database") - FileHandler({'copy': obs_list}).sync() + FileHandler({'copy': obs_files_to_copy}).sync() @logit(logger) def _prep_scratch_dir(self: Task) -> None: @@ -217,15 +212,15 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['PDY'] = os.getenv('PDY') envconfig_jcb['cyc'] = os.getenv('cyc') envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER - envconfig_jcb['obs_list'] = ['adt_rads_all'] envconfig_jcb['HOMEgfs'] = self.task_config.HOMEgfs envconfig_jcb['DO_TEST_MODE'] = self.task_config.DO_TEST_MODE envconfig_jcb['RUN'] = self.task_config.RUN envconfig_jcb['current_cycle'] = self.task_config.current_cycle envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES).zfill(3)) + envconfig_jcb['observations'] = self.task_config.observations # Write obs_list_short - save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') + save_as_yaml(self.task_config['obs_list'], 'obs_list_short.yaml') os.environ['OBS_LIST_SHORT'] = 'obs_list_short.yaml' # Render the JCB configuration files @@ -245,6 +240,8 @@ def _prep_variational_yaml(self: Task) -> None: # Current hack so that this is not done directly in the JCB base yaml jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') + print("barbar") + pprint(jcb_config) # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 02f4e2a7a45..3a80750cc24 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -6,6 +6,7 @@ import os from pygfs.task.analysis import Analysis from typing import Dict +from pprint import pprint from wxflow import (AttrDict, Executable, FileHandler, @@ -54,6 +55,9 @@ def __init__(self, config: Dict) -> None: self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') self.task_config.ENSPERT_RELPATH = _enspert_relpath + self.task_config.PARMsoca = os.path.join(self.task_config.PARMgfs, 'gdas', 'soca') + self.task_config.cyc = os.getenv('cyc') + self.task_config.PDY = os.getenv('PDY') @logit(logger) def initialize(self): @@ -71,8 +75,9 @@ def initialize(self): # make directories and stage ensemble background files soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - stageconf = AttrDict() - keys = ['current_cycle', + stageconfig = AttrDict() + keys = ['cyc' + 'current_cycle', 'previous_cycle', 'COM_ICE_LETKF_TMPL', 'COM_OCEAN_LETKF_TMPL', @@ -88,32 +93,49 @@ def initialize(self): 'NMEM_ENS', 'OPREFIX', 'PARMgfs', + 'PDY' 'ROTDIR', 'RUN', 'WINDOW_BEGIN', 'WINDOW_MIDDLE'] for key in keys: - stageconf[key] = self.task_config[key] + stageconfig[key] = self.task_config[key] + #stageconfig['PDY'] = os.getenv('PDY') + #stageconfig['cyc'] = os.getenv('cyc') # stage ensemble background files - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconf) + soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconfig) FileHandler(soca_ens_bkg_stage_list).sync() # stage letkf-specific files - letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconfig) FileHandler(letkf_stage_list).sync() - obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) + jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') + jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=stageconfig) + print("jcb_base_config") + pprint(jcb_base_config) - # get the list of observations + # "observations" is expected by later JCB code to populate it with config info, + self.task_config.observations = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config)['observations'] + + # also expected by JCB + self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS + + jcb_config ={**jcb_base_config, **self.task_config } + + obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') + self.task_config.observations = parse_j2yaml(obsconfigfile, jcb_config) + + # get the list of expected observation files:w obs_files = [] - for ob in obs_list['observers']: - obs_name = ob['obs space']['name'].lower() + for observer in self.task_config['observations']['observers']: + obs_name = observer['obs space']['name'].lower() # TODO(AFE) - this should be removed when the obs config yamls are jinjafied - if 'distribution' not in ob['obs space']: - ob['obs space']['distribution'] = {'name': 'Halo', 'halo size': self.task_config['DIST_HALO_SIZE']} + if 'distribution' not in observer['obs space']: + observer['obs space']['distribution'] = {'name': 'Halo', 'halo size': self.task_config['DIST_HALO_SIZE']} obs_filename = f"{self.task_config.OPREFIX}{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc4" - obs_files.append((obs_filename, ob)) + obs_files.append((obs_filename, observer)) obs_files_to_copy = [] obs_to_use = [] @@ -131,7 +153,9 @@ def initialize(self): FileHandler({'copy': obs_files_to_copy}).sync() # make the letkf.yaml - letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) + # TODO (AFE) switch to fully JCB version + jcb_config = {**jcb_base_config, **stageconfig} + letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, jcb_config) letkf_yaml.observations.observers = obs_to_use letkf_yaml.save(self.task_config.letkf_yaml_file) @@ -191,10 +215,11 @@ def finalize(self): letkfsaveconf[key] = self.task_config[key] # get the list of obs output files - obs_list = parse_j2yaml(letkfsaveconf.MARINE_OBS_LIST_YAML, self.task_config) + #obs_list = parse_j2yaml(letkfsaveconf.MARINE_OBS_LIST_YAML, self.task_config) obs_files = [] - for ob in obs_list['observers']: - obs_files.append(ob['obs space']['obsdataout']['engine']['obsfile']) + #for ob in obs_list['observers']: + for observer in self.task_config['observations']['observers']: + obs_files.append(observer['obs space']['obsdataout']['engine']['obsfile']) obs_files_to_copy = [] # copy obs from diags to COMOUT for obs_src in obs_files: From b9c5a6abae1dc303fbaa9635dc559db2cc433bf9 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Sun, 30 Mar 2025 12:45:37 +0000 Subject: [PATCH 2/9] cleanup --- ush/python/pygfs/task/marine_letkf.py | 50 ++++++++++++--------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 3a80750cc24..2f8a0dfb5e7 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -58,6 +58,7 @@ def __init__(self, config: Dict) -> None: self.task_config.PARMsoca = os.path.join(self.task_config.PARMgfs, 'gdas', 'soca') self.task_config.cyc = os.getenv('cyc') self.task_config.PDY = os.getenv('PDY') + self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS @logit(logger) def initialize(self): @@ -76,7 +77,8 @@ def initialize(self): soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() stageconfig = AttrDict() - keys = ['cyc' + keys = ['app_path_observations', + 'cyc', 'current_cycle', 'previous_cycle', 'COM_ICE_LETKF_TMPL', @@ -88,48 +90,42 @@ def initialize(self): 'COMOUT_ICE_LETKF', 'COMOUT_OCEAN_LETKF', 'DATA', + 'DIST_HALO_SIZE', 'ENSPERT_RELPATH', 'GDUMP_ENS', 'NMEM_ENS', 'OPREFIX', 'PARMgfs', - 'PDY' + 'PDY', 'ROTDIR', 'RUN', 'WINDOW_BEGIN', 'WINDOW_MIDDLE'] for key in keys: stageconfig[key] = self.task_config[key] - #stageconfig['PDY'] = os.getenv('PDY') - #stageconfig['cyc'] = os.getenv('cyc') - # stage ensemble background files - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconfig) - FileHandler(soca_ens_bkg_stage_list).sync() + jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') + jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=stageconfig) + + jcb_config = {**jcb_base_config, **stageconfig} # stage letkf-specific files - letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconfig) + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, jcb_config) FileHandler(letkf_stage_list).sync() - jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') - jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=stageconfig) - print("jcb_base_config") - pprint(jcb_base_config) + # stage ensemble background files + soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconfig) + FileHandler(soca_ens_bkg_stage_list).sync() # "observations" is expected by later JCB code to populate it with config info, - self.task_config.observations = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config)['observations'] - - # also expected by JCB - self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS - - jcb_config ={**jcb_base_config, **self.task_config } + jcb_config['observations'] = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, jcb_config)['observations'] obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') - self.task_config.observations = parse_j2yaml(obsconfigfile, jcb_config) + jcb_config['observations'] = parse_j2yaml(obsconfigfile, jcb_config) - # get the list of expected observation files:w + # get the list of expected observation files obs_files = [] - for observer in self.task_config['observations']['observers']: + for observer in jcb_config['observations']['observers']: obs_name = observer['obs space']['name'].lower() # TODO(AFE) - this should be removed when the obs config yamls are jinjafied if 'distribution' not in observer['obs space']: @@ -154,11 +150,11 @@ def initialize(self): # make the letkf.yaml # TODO (AFE) switch to fully JCB version - jcb_config = {**jcb_base_config, **stageconfig} letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, jcb_config) letkf_yaml.observations.observers = obs_to_use letkf_yaml.save(self.task_config.letkf_yaml_file) + # TODO(AFE) get rid of this, I think # swap date and stack size in mom_input.nml domain_stack_size = self.task_config.DOMAIN_STACK_SIZE ymdhms = [int(s) for s in self.task_config.WINDOW_BEGIN.strftime('%Y,%m,%d,%H,%M,%S').split(',')] @@ -214,20 +210,18 @@ def finalize(self): for key in keys: letkfsaveconf[key] = self.task_config[key] - # get the list of obs output files - #obs_list = parse_j2yaml(letkfsaveconf.MARINE_OBS_LIST_YAML, self.task_config) + # get the list of obs output file - letkf yaml is already complete + letkf_config = parse_j2yaml(self.task_config.letkf_yaml_file, AttrDict()) obs_files = [] - #for ob in obs_list['observers']: - for observer in self.task_config['observations']['observers']: + for observer in letkf_config['observations']['observers']: obs_files.append(observer['obs space']['obsdataout']['engine']['obsfile']) obs_files_to_copy = [] - # copy obs from diags to COMOUT + # copy files from diags to COMOUT for obs_src in obs_files: obs_dst = os.path.join(letkfsaveconf.COMOUT_OCEAN_LETKF, 'diags', os.path.basename(obs_src)) if os.path.exists(obs_src): obs_files_to_copy.append([obs_src, obs_dst]) - # stage the desired diag files FileHandler({'mkdir': [os.path.join(letkfsaveconf.COMOUT_OCEAN_LETKF, 'diags')]}).sync() FileHandler({'copy': obs_files_to_copy}).sync() letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf) From 78ec4db3ee7c04ba1ae067c2cb9a70e6b43da727 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Sun, 30 Mar 2025 14:30:51 +0000 Subject: [PATCH 3/9] clean up marine analysis --- ush/python/pygfs/task/marine_analysis.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index fa6d09a6929..85b820c9b7a 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -11,7 +11,6 @@ import subprocess import yaml from jcb import render -from pprint import pprint from wxflow import (AttrDict, FileHandler, @@ -240,8 +239,6 @@ def _prep_variational_yaml(self: Task) -> None: # Current hack so that this is not done directly in the JCB base yaml jcb_config['marine_pseudo_model_states'] = parse_yaml('bkg_list.yaml') - print("barbar") - pprint(jcb_config) # Render the full JEDI configuration file using JCB jedi_config = render(jcb_config) From b1a9056253e45531b35c4451c40f3c394fa874fb Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 31 Mar 2025 00:26:38 +0000 Subject: [PATCH 4/9] cleanup --- ush/python/pygfs/task/marine_letkf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 2f8a0dfb5e7..4559774b5e8 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -6,7 +6,6 @@ import os from pygfs.task.analysis import Analysis from typing import Dict -from pprint import pprint from wxflow import (AttrDict, Executable, FileHandler, From e92cfe03150770c582eab5da49e52e5c4e650ad8 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 31 Mar 2025 00:56:50 +0000 Subject: [PATCH 5/9] relocated env var --- ush/python/pygfs/task/marine_analysis.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 85b820c9b7a..ba2023bac80 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -55,7 +55,8 @@ def __init__(self, config): 'ENSPERT_RELPATH': _enspert_relpath, 'CALC_SCALE_EXEC': _calc_scale_exec, 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'app_path_observations': self.task_config.MARINE_JCB_GDAS_OBS } ) @@ -125,9 +126,6 @@ def _fetch_observations(self: Task) -> None: self.task_config.observations = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config)['observations'] self.task_config.obs_list = self.task_config.observations - # also expected by JCB - self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS - obsconfigfile = os.path.join(self.task_config['PARMgfs'], 'gdas/soca/obs/obs_list_base_yaml.j2') self.task_config.observations = parse_j2yaml(obsconfigfile, self.task_config) From 3add8421bb8ca23d00d3d4a466f9c4b14148d416 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 1 Apr 2025 18:28:45 +0000 Subject: [PATCH 6/9] removing redundant refs to obs list --- jobs/JGLOBAL_PREP_OCEAN_OBS | 2 +- parm/config/gfs/config.prepoceanobs.j2 | 8 +------- parm/config/gfs/yaml/defaults.yaml | 1 - ush/python/pygfs/task/marine_letkf.py | 2 ++ workflow/hosts/hera.yaml | 4 ++-- 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/jobs/JGLOBAL_PREP_OCEAN_OBS b/jobs/JGLOBAL_PREP_OCEAN_OBS index 3f8f347f31e..092944cd70f 100755 --- a/jobs/JGLOBAL_PREP_OCEAN_OBS +++ b/jobs/JGLOBAL_PREP_OCEAN_OBS @@ -1,6 +1,6 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "prepoceanobs" -c "base prepoceanobs" +source "${HOMEgfs}/ush/jjob_header.sh" -e "prepoceanobs" -c "base marineanl prepoceanobs" ############################################## diff --git a/parm/config/gfs/config.prepoceanobs.j2 b/parm/config/gfs/config.prepoceanobs.j2 index cda489bc26f..24a5982ed80 100644 --- a/parm/config/gfs/config.prepoceanobs.j2 +++ b/parm/config/gfs/config.prepoceanobs.j2 @@ -8,10 +8,8 @@ export OCNOBS2IODAEXEC="${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda. export SOCA_INPUT_FIX_DIR="{{ SOCA_INPUT_FIX_DIR }}" -export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" export OBSPREP_YAML="{{ OBSPREP_YAML }}" -export OBS_LIST="{{ SOCA_OBS_LIST }}" -export OBS_YAML=${OBS_LIST} +#export OBS_LIST="{{ SOCA_OBS_LIST }}" # ocean analysis needs own dmpdir until standard dmpdir has full ocean obs use_exp_obs="{{ use_exp_obs }}" @@ -21,10 +19,6 @@ fi export DMPDIR="${dmpdir_exp:-${DMPDIR}}" -# For BUFR2IODA json and python scripts -export JSON_TMPL_DIR="${PARMgfs}/gdas/ioda/bufr2ioda" -export BUFR2IODA_PY_DIR="${USHgfs}" - # Get task specific resources . "${EXPDIR}/config.resources" prepoceanobs echo "END: config.prepoceanobs" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 50d60ceee12..5923b322716 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -63,7 +63,6 @@ marineanl: prepoceanobs: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" - SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml.j2" # TODO: This is also repeated in ocnanal OBSPREP_YAML: "${PARMgfs}/gdas/soca/obsprep/obsprep_config.yaml" use_exp_obs: "YES" dmpdir_exp: "${BASE_DATA}/experimental_obs" diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 4559774b5e8..00ac6df483d 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -58,6 +58,7 @@ def __init__(self, config: Dict) -> None: self.task_config.cyc = os.getenv('cyc') self.task_config.PDY = os.getenv('PDY') self.task_config.app_path_observations = self.task_config.MARINE_JCB_GDAS_OBS + self.task_config.letkf_app = "true" @logit(logger) def initialize(self): @@ -79,6 +80,7 @@ def initialize(self): keys = ['app_path_observations', 'cyc', 'current_cycle', + 'letkf_app', 'previous_cycle', 'COM_ICE_LETKF_TMPL', 'COM_OCEAN_LETKF_TMPL', diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 449cee84a7c..b6f7cc501f9 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -5,7 +5,7 @@ BASE_DATA: '/scratch1/NCEPDEV/global/glopara/data' BASE_IC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR' AERO_INPUTS_DIR: /scratch1/NCEPDEV/global/glopara/data/GEFS_ExtData/20250310 PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' -HOMEDIR: '/scratch1/NCEPDEV/global/${USER}' +HOMEDIR: '/scratch1/NCEPDEV/da/${USER}' STMP: '/scratch1/NCEPDEV/stmp2/${USER}' PTMP: '/scratch1/NCEPDEV/stmp4/${USER}' NOSCRUB: '${HOMEDIR}' @@ -15,7 +15,7 @@ COMINnam: /scratch1/NCEPDEV/global/glopara/data/external_gempak/nam COMINukmet: /scratch1/NCEPDEV/global/glopara/data/external_gempak/ukmet # BQS properties SCHEDULER: slurm -QUEUE: batch +QUEUE: debug PARTITION_BATCH: hera PARTITION_SERVICE: service # HPSS properties From 29709e5a84ddec5e6e1bc275033822037912f12a Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 1 Apr 2025 20:50:04 +0000 Subject: [PATCH 7/9] style --- parm/config/gfs/config.prepoceanobs.j2 | 1 - ush/python/pygfs/task/marine_analysis.py | 7 +++++-- ush/python/pygfs/task/marine_letkf.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/parm/config/gfs/config.prepoceanobs.j2 b/parm/config/gfs/config.prepoceanobs.j2 index 24a5982ed80..f0fc5721ee1 100644 --- a/parm/config/gfs/config.prepoceanobs.j2 +++ b/parm/config/gfs/config.prepoceanobs.j2 @@ -9,7 +9,6 @@ export OCNOBS2IODAEXEC="${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda. export SOCA_INPUT_FIX_DIR="{{ SOCA_INPUT_FIX_DIR }}" export OBSPREP_YAML="{{ OBSPREP_YAML }}" -#export OBS_LIST="{{ SOCA_OBS_LIST }}" # ocean analysis needs own dmpdir until standard dmpdir has full ocean obs use_exp_obs="{{ use_exp_obs }}" diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index ba2023bac80..6654050dfd5 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -132,8 +132,11 @@ def _fetch_observations(self: Task) -> None: obs_files = [] for observer in self.task_config['observations']['observers']: - logger.info(f"******** {self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") - obs_files.append(f"{self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") + #logger.info(f"******** {self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") + #obs_files.append(f"{self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") + filename = f"{self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4" + logger.info(f"******** {filename}") + obs_files.append(filename) obs_files_to_copy = [] diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 00ac6df483d..b802d964f8e 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -108,7 +108,7 @@ def initialize(self): jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml') jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=stageconfig) - jcb_config = {**jcb_base_config, **stageconfig} + jcb_config = {**jcb_base_config, **stageconfig} # stage letkf-specific files letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, jcb_config) From 793617644a7c69673661e23f017a027b6dd8a4be Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 1 Apr 2025 21:09:21 +0000 Subject: [PATCH 8/9] style --- ush/python/pygfs/task/marine_analysis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 6654050dfd5..d61b56fc377 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -132,8 +132,6 @@ def _fetch_observations(self: Task) -> None: obs_files = [] for observer in self.task_config['observations']['observers']: - #logger.info(f"******** {self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") - #obs_files.append(f"{self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4") filename = f"{self.task_config.OPREFIX}{observer['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4" logger.info(f"******** {filename}") obs_files.append(filename) From dbcab846779650823dbf96cd2515c7b3c636ac34 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 1 Apr 2025 21:11:27 +0000 Subject: [PATCH 9/9] renormalized hera settings --- workflow/hosts/hera.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index b6f7cc501f9..449cee84a7c 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -5,7 +5,7 @@ BASE_DATA: '/scratch1/NCEPDEV/global/glopara/data' BASE_IC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR' AERO_INPUTS_DIR: /scratch1/NCEPDEV/global/glopara/data/GEFS_ExtData/20250310 PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara' -HOMEDIR: '/scratch1/NCEPDEV/da/${USER}' +HOMEDIR: '/scratch1/NCEPDEV/global/${USER}' STMP: '/scratch1/NCEPDEV/stmp2/${USER}' PTMP: '/scratch1/NCEPDEV/stmp4/${USER}' NOSCRUB: '${HOMEDIR}' @@ -15,7 +15,7 @@ COMINnam: /scratch1/NCEPDEV/global/glopara/data/external_gempak/nam COMINukmet: /scratch1/NCEPDEV/global/glopara/data/external_gempak/ukmet # BQS properties SCHEDULER: slurm -QUEUE: debug +QUEUE: batch PARTITION_BATCH: hera PARTITION_SERVICE: service # HPSS properties