From 20e48a1aa06901c1036566e3582bf2a64595903a Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 21 Aug 2024 18:01:20 +0000 Subject: [PATCH 01/33] debugging for ocean ensemble run --- ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml | 32 +++++++++++++++++++++++ jobs/JGLOBAL_MARINE_BMAT | 14 ++++++---- ush/forecast_postdet.sh | 9 ++++--- ush/python/pygfs/task/marine_bmat.py | 4 ++- ush/python/pygfs/utils/marine_da_utils.py | 8 +++++- workflow/setup_expt.py | 6 +++++ 6 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml b/ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml new file mode 100644 index 00000000000..df3965a94d6 --- /dev/null +++ b/ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml @@ -0,0 +1,32 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: S2S + resdetatmos: 48 + resdetocean: 5.0 + resensatmos: 48 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + #icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 + idate: 2021032412 + edate: 2021032418 + #nens: 0 + #icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48mx500 + #idate: 2021032312 + #edate: 2021032318 + #icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48O500 + #idate: 2021032412 + #edate: 2021032418 + nens: 3 + icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48mx500ens + gfs_cyc: 0 + start: warm + yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml + +skip_ci_on_hosts: + - wcoss2 + - orion + - hercules diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index 3dacec92783..ff3a7ffa8ce 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -2,17 +2,21 @@ source "${HOMEgfs}/ush/preamble.sh" -if (( 10#${ENSMEM:-0} > 0 )); then +# source config.base, config.ocnanal and config.marinebmat +# and pass marinebmat to ${machine}.env +source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" + + +#if (( 10#${ENSMEM:-0} > 0 )); then +if (( 10#${NMEM_ENS:-0} > 0 )); then export DATAjob="${DATAROOT}/${RUN}marinebmat.${PDY:-}${cyc}" - export DATA="${DATAjob}/${jobid}" + #export DATA="${DATAjob}/${jobid}" + export DATA=${DATA:-${DATAROOT}/${jobid:?}} # Create the directory to hold ensemble perturbations export DATAenspert="${DATAjob}/enspert" if [[ ! -d "${DATAenspert}" ]]; then mkdir -p "${DATAenspert}"; fi fi -# source config.base, config.ocnanal and config.marinebmat -# and pass marinebmat to ${machine}.env -source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" ############################################## # Set variables used in the script diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7de31d6235b..433806dea94 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -428,11 +428,12 @@ MOM6_postdet() { # GEFS perturbations # TODO if [[ $RUN} == "gefs" ]] block maybe be needed + # yup # to ensure it does not interfere with the GFS when ensemble is updated in the GFS - if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then - ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ - || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) - fi + #if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then + # ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ + # || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) + #fi fi # if [[ "${RERUN}" == "NO" ]]; then # Link output files diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 9d64e621c9c..ac228d1cfac 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -81,6 +81,7 @@ def initialize(self: Task) -> None: # stage backgrounds # TODO(G): Check ocean backgrounds dates for consistency bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) + print("bkg_list: ",bkg_list) FileHandler(bkg_list).sync() for cice_fname in ['./INPUT/cice.res.nc', './bkg/ice.bkg.f006.nc', './bkg/ice.bkg.f009.nc']: mdau.cice_hist2fms(cice_fname, cice_fname) @@ -214,6 +215,7 @@ def vertical_diffusion(self: Task) -> None: # compute the coefficients of the diffusion operator mdau.run(exec_cmd) + @logit(logger) def ensemble_perturbations(self: Task) -> None: """Generate the 3D ensemble of perturbation for the 3DEnVAR @@ -323,7 +325,7 @@ def finalize(self: Task) -> None: FileHandler({'copy': diagb_list}).sync() # Copy the ensemble perturbation diagnostics to the ROTDIR - if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: window_middle_iso = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') weight_list = [] src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc") diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 016551878b3..12c9e74cf8b 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -1,6 +1,7 @@ import f90nml -import os from logging import getLogger +import os +import re import xarray as xr from wxflow import (FileHandler, @@ -97,3 +98,8 @@ def stage_ens_mem(task_config: AttrDict) -> None: letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) logger.info(f"{letkf_stage_list}") FileHandler(letkf_stage_list).sync() + + for _, ens_mem_file in letkf_stage_list['copy']: + print("ens_mem_file: ", ens_mem_file) + if re.fullmatch(r'ice\.\d+\.nc', os.path.basename(ens_mem_file)): + cice_hist2fms(ens_mem_file, ens_mem_file) \ No newline at end of file diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 3e70df0f021..3213a8e7edf 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -168,6 +168,12 @@ def link_files_from_src_to_dst(src_dir, dst_dir): makedirs_if_missing(dst_dir) link_files_from_src_to_dst(src_dir, dst_dir) + dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_ice_anl_dir) + src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ice_anl_dir) + makedirs_if_missing(dst_dir) + link_files_from_src_to_dst(src_dir, dst_dir) + + # Link mediator files if do_med: dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_med_dir) From f749b3fba153afbf720a127e99d90ed6c1cbc610 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 29 Aug 2024 14:19:46 +0000 Subject: [PATCH 02/33] fix ci stager for ensemble + DO_IAU --- parm/stage/analysis.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/stage/analysis.yaml.j2 b/parm/stage/analysis.yaml.j2 index e014313b6d0..81a50ba0a62 100644 --- a/parm/stage/analysis.yaml.j2 +++ b/parm/stage/analysis.yaml.j2 @@ -10,7 +10,7 @@ analysis: {% for mem in range(first_mem, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_ATMOS_ANALYSIS_MEM = COMOUT_ATMOS_ANALYSIS_MEM_list[imem] %} - {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "radstat", "ratminc.nc"] %} + {% for ftype in ["abias", "abias_air", "abias_int", "abias_pc", "atminc.nc", "atmi009.nc", "atmi003.nc", "radstat", "ratminc.nc", "ratmi009.nc", "ratmi003.nc"] %} {% if path_exists(ICSDIR ~ "/" ~ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) ~ "/" ~ RUN ~ ".t" ~ current_cycle_HH ~ "z." ~ ftype) %} - ["{{ ICSDIR }}/{{ COMOUT_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.{{ ftype }}", "{{ COMOUT_ATMOS_ANALYSIS_MEM }}"] {% endif %} From 6b1832acf52a2cf6bfa2db487472e8457a60c01d Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 5 Sep 2024 13:33:48 +0000 Subject: [PATCH 03/33] removing ice var changer --- ush/python/pygfs/utils/marine_da_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 645f82af72f..117537e1766 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -76,7 +76,7 @@ def stage_ens_mem(task_config: AttrDict) -> None: logger.info(f"{letkf_stage_list}") FileHandler(letkf_stage_list).sync() - for _, ens_mem_file in letkf_stage_list['copy']: - print("ens_mem_file: ", ens_mem_file) - if re.fullmatch(r'ice\.\d+\.nc', os.path.basename(ens_mem_file)): - cice_hist2fms(ens_mem_file, ens_mem_file) \ No newline at end of file + # for _, ens_mem_file in letkf_stage_list['copy']: + # print("ens_mem_file: ", ens_mem_file) + # if re.fullmatch(r'ice\.\d+\.nc', os.path.basename(ens_mem_file)): + # cice_hist2fms(ens_mem_file, ens_mem_file) \ No newline at end of file From 581f1c3c3df18e332aed2bc3d3f56c044a029208 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Fri, 13 Sep 2024 13:43:28 +0000 Subject: [PATCH 04/33] the latests --- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 12 ++- parm/config/gfs/config.com | 2 + ush/python/pygfs/task/marine_letkf.py | 89 ++++++++++++++++++----- ush/python/pygfs/utils/marine_da_utils.py | 7 +- workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 27 +++++++ workflow/rocoto/tasks.py | 2 +- 7 files changed, 114 insertions(+), 29 deletions(-) diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 38dc3049f90..4de12471a6d 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -11,12 +11,18 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} +# needed for obspace yamls +OPREFIX="${RUN}.t${cyc}z." +export OPREFIX YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ - COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL + COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ + COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL \ + COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \ + COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL ############################################## # Begin JOB SPECIFIC work diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 61d592561d7..d949edb33a6 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -82,12 +82,14 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf' declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2' declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' +declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf' declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 36c26d594b3..566e1a3f2fc 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -6,11 +6,13 @@ from pygfs.task.analysis import Analysis from typing import Dict from wxflow import (AttrDict, + Executable, FileHandler, logit, parse_j2yaml, to_timedelta, - to_YMDH) + to_YMDH, + WorkflowException) logger = getLogger(__name__.split('.')[-1]) @@ -49,6 +51,7 @@ def __init__(self, config: Dict) -> None: self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') + self.task_config.OPREFIX = f"{self.task_config.RUN}.t{self.task_config.cyc}z." @logit(logger) def initialize(self): @@ -64,17 +67,37 @@ def initialize(self): logger.info("initialize") # make directories and stage ensemble background files - ensbkgconf = AttrDict() - keys = ['previous_cycle', 'current_cycle', 'DATA', 'NMEM_ENS', - 'PARMgfs', 'ROTDIR', 'COM_OCEAN_HISTORY_TMPL', 'COM_ICE_HISTORY_TMPL'] - for key in keys: - ensbkgconf[key] = self.task_config[key] - ensbkgconf.RUN = 'enkfgdas' - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, ensbkgconf) - FileHandler(soca_ens_bkg_stage_list).sync() soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) + self.task_config['NMEM_ENS']=3 + stageconf = AttrDict() + keys = ['current_cycle', + 'previous_cycle', + 'COM_ICE_LETKF_TMPL', + 'COM_OCEAN_LETKF_TMPL', + 'COM_ICE_HISTORY_TMPL', + 'COM_OCEAN_HISTORY_TMPL', + 'COMIN_OCEAN_HISTORY_PREV', + 'COMIN_ICE_HISTORY_PREV', + 'COMOUT_ICE_LETKF', + 'COMOUT_OCEAN_LETKF', + 'DATA', + 'NMEM_ENS', + 'PARMgfs', + 'ROTDIR', + 'WINDOW_BEGIN', + 'WINDOW_MIDDLE'] + for key in keys: + stageconf[key] = self.task_config[key] + stageconf.RUN = 'enkfgdas' + + # stage ensemble background files + soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, stageconf) + print("soca_ens_bkg_stage_list: ",soca_ens_bkg_stage_list) + FileHandler(soca_ens_bkg_stage_list).sync() + + # stage letkf-specific files + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) FileHandler(letkf_stage_list).sync() obs_list = parse_j2yaml(self.task_config.OBS_YAML, self.task_config) @@ -83,7 +106,7 @@ def initialize(self): obs_files = [] for ob in obs_list['observers']: obs_name = ob['obs space']['name'].lower() - obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc" + obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc4" obs_files.append((obs_filename, ob)) obs_files_to_copy = [] @@ -102,12 +125,7 @@ def initialize(self): FileHandler({'copy': obs_files_to_copy}).sync() # make the letkf.yaml - letkfconf = AttrDict() - keys = ['WINDOW_BEGIN', 'WINDOW_MIDDLE', 'RUN', 'gcyc', 'NMEM_ENS'] - for key in keys: - letkfconf[key] = self.task_config[key] - letkfconf.RUN = 'enkfgdas' - letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, letkfconf) + letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) letkf_yaml.observations.observers = obs_to_use letkf_yaml.save(self.task_config.letkf_yaml_file) @@ -120,6 +138,7 @@ def initialize(self): nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) nml.write(self.task_config.mom_input_nml, force=True) # force to overwrite if necessary + @logit(logger) def run(self): """Method run for ocean and sea ice LETKF task @@ -133,6 +152,33 @@ def run(self): logger.info("run") + exec_cmd_gridgen = Executable(self.task_config.APRUN_MARINEANALLETKF) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_EXEC) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_YAML) + + try: + logger.debug(f"Executing {exec_cmd_gridgen}") + exec_cmd_gridgen() + except OSError: + raise OSError(f"Failed to execute {exec_cmd_gridgen}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd_gridgen}") + pass + + exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANALLETKF) + for letkf_exec_arg in self.task_config.letkf_exec_args: + exec_cmd_letkf.add_default_arg(letkf_exec_arg) + + try: + logger.debug(f"Executing {exec_cmd_letkf}") + exec_cmd_letkf() + except OSError: + raise OSError(f"Failed to execute {exec_cmd_letkf}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd_letkf}") + pass + + @logit(logger) def finalize(self): """Method finalize for ocean and sea ice LETKF task @@ -145,3 +191,12 @@ def finalize(self): """ logger.info("finalize") + + letkfsaveconf = AttrDict() + keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', + 'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL'] + for key in keys: + letkfsaveconf[key] = self.task_config[key] + letkfsaveconf.RUN = 'enkfgdas' + letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf) + FileHandler(letkf_save_list).sync() diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 117537e1766..0fde56544d4 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -74,9 +74,4 @@ def stage_ens_mem(task_config: AttrDict) -> None: logger.debug(f"{jinja.Jinja(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf).render}") letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) logger.info(f"{letkf_stage_list}") - FileHandler(letkf_stage_list).sync() - - # for _, ens_mem_file in letkf_stage_list['copy']: - # print("ens_mem_file: ", ens_mem_file) - # if re.fullmatch(r'ice\.\d+\.nc', os.path.basename(ens_mem_file)): - # cice_hist2fms(ens_mem_file, ens_mem_file) \ No newline at end of file + FileHandler(letkf_stage_list).sync() \ No newline at end of file diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index b8aa2dba3a6..62d6d024d74 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -45,7 +45,7 @@ def _get_app_configs(self): if self.do_jediocnvar: configs += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] if self.do_hybvar: - configs += ['ocnanalecen'] + configs += ['marineanalletkf', 'ocnanalecen'] configs += ['ocnanalchkpt', 'ocnanalpost'] if self.do_vrfy_oceanda: configs += ['ocnanalvrfy'] @@ -147,7 +147,7 @@ def get_task_names(self): if self.do_jediocnvar: gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] if self.do_hybvar: - gdas_gfs_common_tasks_before_fcst += ['ocnanalecen'] + gdas_gfs_common_tasks_before_fcst += ['marineanalletkf', 'ocnanalecen'] gdas_gfs_common_tasks_before_fcst += ['ocnanalchkpt', 'ocnanalpost'] if self.do_vrfy_oceanda: gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 76db3db88ea..ee5831209e2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -665,6 +665,33 @@ def prepoceanobs(self): return task + def marineanalletkf(self): + + ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) + + deps = [] + data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' + dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('marineanalletkf') + task_name = f'{self.run}marineanalletkf' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanalletkf.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def marinebmat(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index d8d5edb5e60..e0204689244 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -16,7 +16,7 @@ class Tasks: 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', - 'ocnanalprep', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', + 'ocnanalprep', 'marineanalletkf', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', From 6a2cd3d1906849a6213cbae2742af613343c50fb Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 19 Sep 2024 13:00:27 +0000 Subject: [PATCH 05/33] removal of cruft --- ...0_3DVarAOWCDAens.yaml => C48mx500_hybAOWCDA.yaml} | 7 ------- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 3 --- jobs/JGLOBAL_MARINE_BMAT | 2 -- parm/config/gfs/config.marineanalletkf | 1 + ush/forecast_postdet.sh | 12 ++++++------ ush/python/pygfs/task/marine_bmat.py | 1 - ush/python/pygfs/task/marine_letkf.py | 1 - 7 files changed, 7 insertions(+), 20 deletions(-) rename ci/cases/pr/{C48mx500_3DVarAOWCDAens.yaml => C48mx500_hybAOWCDA.yaml} (71%) diff --git a/ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml similarity index 71% rename from ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml rename to ci/cases/pr/C48mx500_hybAOWCDA.yaml index df3965a94d6..b64e7579867 100644 --- a/ci/cases/pr/C48mx500_3DVarAOWCDAens.yaml +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -13,13 +13,6 @@ arguments: #icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 idate: 2021032412 edate: 2021032418 - #nens: 0 - #icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48mx500 - #idate: 2021032312 - #edate: 2021032318 - #icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48O500 - #idate: 2021032412 - #edate: 2021032418 nens: 3 icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48mx500ens gfs_cyc: 0 diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 4de12471a6d..07e53093e91 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -11,9 +11,6 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} -# needed for obspace yamls -OPREFIX="${RUN}.t${cyc}z." -export OPREFIX YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ diff --git a/jobs/JGLOBAL_MARINE_BMAT b/jobs/JGLOBAL_MARINE_BMAT index ff3a7ffa8ce..82f7620d0da 100755 --- a/jobs/JGLOBAL_MARINE_BMAT +++ b/jobs/JGLOBAL_MARINE_BMAT @@ -7,10 +7,8 @@ source "${HOMEgfs}/ush/preamble.sh" source "${HOMEgfs}/ush/jjob_header.sh" -e "marinebmat" -c "base ocnanal marinebmat" -#if (( 10#${ENSMEM:-0} > 0 )); then if (( 10#${NMEM_ENS:-0} > 0 )); then export DATAjob="${DATAROOT}/${RUN}marinebmat.${PDY:-}${cyc}" - #export DATA="${DATAjob}/${jobid}" export DATA=${DATA:-${DATAROOT}/${jobid:?}} # Create the directory to hold ensemble perturbations export DATAenspert="${DATAjob}/enspert" diff --git a/parm/config/gfs/config.marineanalletkf b/parm/config/gfs/config.marineanalletkf index fde3433a13d..a53a16688a9 100644 --- a/parm/config/gfs/config.marineanalletkf +++ b/parm/config/gfs/config.marineanalletkf @@ -11,6 +11,7 @@ echo "BEGIN: config.marineanalletkf" export MARINE_LETKF_EXEC="${JEDI_BIN}/gdas.x" export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" +export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j2" export GRIDGEN_EXEC="${JEDI_BIN}/gdas_soca_gridgen.x" export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 70bba1392df..f01bf28e161 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -460,13 +460,13 @@ MOM6_postdet() { fi # GEFS perturbations - # TODO if [[ $RUN} == "gefs" ]] block maybe be needed - # yup + if [[ "${RUN}" == "gefs" ]]; then # to ensure it does not interfere with the GFS when ensemble is updated in the GFS - #if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then - # ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ - # || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) - #fi + if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then + ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ + || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) + fi + fi # if [[ "${RUN}" == "gefs" ]]; then fi # if [[ "${RERUN}" == "NO" ]]; then # Link output files diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 97a7e0e7fe1..067f52f73e9 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -81,7 +81,6 @@ def initialize(self: Task) -> None: # stage backgrounds # TODO(G): Check ocean backgrounds dates for consistency bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) - print("bkg_list: ",bkg_list) FileHandler(bkg_list).sync() # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 566e1a3f2fc..6f38232c6b3 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -51,7 +51,6 @@ def __init__(self, config: Dict) -> None: self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') - self.task_config.OPREFIX = f"{self.task_config.RUN}.t{self.task_config.cyc}z." @logit(logger) def initialize(self): From fb55a4ec3ce3a7015e32e376fcfa92cc8bc951da Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 19 Sep 2024 13:53:47 +0000 Subject: [PATCH 06/33] undo odd thing --- workflow/setup_expt.py | 235 ----------------------------------------- 1 file changed, 235 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index dbb9b9d0093..e213394e20f 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -29,241 +29,6 @@ def makedirs_if_missing(dirname): os.makedirs(dirname) -def fill_ROTDIR(host, inputs): - """ - Method to populate the ROTDIR for supported modes. - INPUTS: - host: host object from class Host - inputs: user inputs to setup_expt.py - """ - - fill_modes = { - 'cycled': fill_ROTDIR_cycled, - 'forecast-only': fill_ROTDIR_forecasts - } - - try: - fill_modes[inputs.mode](host, inputs) - except KeyError: - raise NotImplementedError(f'{inputs.mode} is not a supported mode.\n' + - 'Currently supported modes are:\n' + - f'{" | ".join(fill_modes.keys())}') - - return - - -def fill_ROTDIR_cycled(host, inputs): - """ - Implementation of 'fill_ROTDIR' for cycled mode - """ - - rotdir = os.path.join(inputs.comroot, inputs.pslot) - - do_ocean = do_ice = do_med = False - - if 'S2S' in inputs.app: - do_ocean = do_ice = do_med = True - - if inputs.icsdir is None: - warnings.warn("User did not provide '--icsdir' to stage initial conditions") - return - - rdatestr = datetime_to_YMDH(inputs.idate - to_timedelta('T06H')) - idatestr = datetime_to_YMDH(inputs.idate) - - # Test if we are using the new COM structure or the old flat one for ICs - if inputs.start in ['warm']: - pathstr = os.path.join(inputs.icsdir, f'{inputs.run}.{rdatestr[:8]}', - rdatestr[8:], 'model_data', 'atmos') - else: - pathstr = os.path.join(inputs.icsdir, f'{inputs.run}.{idatestr[:8]}', - idatestr[8:], 'model_data', 'atmos') - - if os.path.isdir(pathstr): - flat_structure = False - else: - flat_structure = True - - # Destination always uses the new COM structure - # These should match the templates defined in config.com - if inputs.start in ['warm']: - dst_atm_dir = os.path.join('model_data', 'atmos', 'restart') - dst_med_dir = os.path.join('model_data', 'med', 'restart') - else: - dst_atm_dir = os.path.join('model_data', 'atmos', 'input') - dst_med_dir = '' # no mediator files for a "cold start" - do_med = False - dst_ocn_rst_dir = os.path.join('model_data', 'ocean', 'restart') - dst_ocn_anl_dir = os.path.join('analysis', 'ocean') - dst_ice_rst_dir = os.path.join('model_data', 'ice', 'restart') - dst_ice_anl_dir = os.path.join('analysis', 'ice') - dst_atm_anl_dir = os.path.join('analysis', 'atmos') - - if flat_structure: - # ICs are in the old flat COM structure - if inputs.start in ['warm']: # This is warm start experiment - src_atm_dir = os.path.join('atmos', 'RESTART') - src_med_dir = os.path.join('med', 'RESTART') - elif inputs.start in ['cold']: # This is a cold start experiment - src_atm_dir = os.path.join('atmos', 'INPUT') - src_med_dir = '' # no mediator files for a "cold start" - do_med = False - # ocean and ice have the same filenames for warm and cold - src_ocn_rst_dir = os.path.join('ocean', 'RESTART') - src_ocn_anl_dir = 'ocean' - src_ice_rst_dir = os.path.join('ice', 'RESTART') - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = 'atmos' - else: - src_atm_dir = dst_atm_dir - src_med_dir = dst_med_dir - src_ocn_rst_dir = dst_ocn_rst_dir - src_ocn_anl_dir = dst_ocn_anl_dir - src_ice_rst_dir = dst_ice_rst_dir - src_ice_anl_dir = dst_ice_anl_dir - src_atm_anl_dir = dst_atm_anl_dir - - def link_files_from_src_to_dst(src_dir, dst_dir): - files = os.listdir(src_dir) - for fname in files: - os.symlink(os.path.join(src_dir, fname), - os.path.join(dst_dir, fname)) - return - - # Link ensemble member initial conditions - if inputs.nens > 0: - previous_cycle_dir = f'enkf{inputs.run}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'enkf{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'enkf{inputs.run}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, memdir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_ice_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_ice_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_ice_anl_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_ice_anl_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, memdir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, memdir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link deterministic initial conditions - previous_cycle_dir = f'{inputs.run}.{rdatestr[:8]}/{rdatestr[8:]}' - current_cycle_dir = f'{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - - # Link atmospheric files - if inputs.start in ['warm']: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_atm_dir) - elif inputs.start in ['cold']: - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_dir) - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_dir) - - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link ocean files - if do_ocean: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_ocn_rst_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_ocn_rst_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # First 1/2 cycle needs a MOM6 increment - incfile = f'{inputs.run}.t{idatestr[8:]}z.ocninc.nc' - src_file = os.path.join(inputs.icsdir, current_cycle_dir, src_ocn_anl_dir, incfile) - dst_file = os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir, incfile) - makedirs_if_missing(os.path.join(rotdir, current_cycle_dir, dst_ocn_anl_dir)) - os.symlink(src_file, dst_file) - - # Link ice files - if do_ice: - # First 1/2 cycle needs a CICE6 analysis restart - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_ice_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, src_ice_anl_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link mediator files - if do_med: - dst_dir = os.path.join(rotdir, previous_cycle_dir, dst_med_dir) - src_dir = os.path.join(inputs.icsdir, previous_cycle_dir, src_med_dir) - makedirs_if_missing(dst_dir) - link_files_from_src_to_dst(src_dir, dst_dir) - - # Link bias correction and radiance diagnostics files - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['abias', 'abias_pc', 'abias_air', 'radstat']: - fname = f'{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - # First 1/2 cycle also needs a atmos increment if doing warm start - if inputs.start in ['warm']: - for ftype in ['atmi003.nc', 'atminc.nc', 'atmi009.nc']: - fname = f'{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - if inputs.nens > 0: - current_cycle_dir = f'enkf{inputs.run}.{idatestr[:8]}/{idatestr[8:]}' - for ii in range(1, inputs.nens + 1): - memdir = f'mem{ii:03d}' - src_dir = os.path.join(inputs.icsdir, current_cycle_dir, memdir, src_atm_anl_dir) - dst_dir = os.path.join(rotdir, current_cycle_dir, memdir, dst_atm_anl_dir) - makedirs_if_missing(dst_dir) - for ftype in ['ratmi003.nc', 'ratminc.nc', 'ratmi009.nc']: - fname = f'enkf{inputs.run}.t{idatestr[8:]}z.{ftype}' - src_file = os.path.join(src_dir, fname) - if os.path.exists(src_file): - os.symlink(src_file, os.path.join(dst_dir, fname)) - - return - - -def fill_ROTDIR_forecasts(host, inputs): - """ - Implementation of 'fill_ROTDIR' for forecast-only mode - """ - print('forecast-only mode treats ICs differently and cannot be staged here') - - def fill_EXPDIR(inputs): """ Method to copy config files from workflow to experiment directory From 398ec2299f89ba7df17fb64c0186b5210155fc6d Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 19 Sep 2024 17:14:36 +0000 Subject: [PATCH 07/33] style --- ush/python/pygfs/task/marine_bmat.py | 1 - ush/python/pygfs/task/marine_letkf.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 067f52f73e9..0e9ac169833 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -213,7 +213,6 @@ def vertical_diffusion(self: Task) -> None: # compute the coefficients of the diffusion operator mdau.run(exec_cmd) - @logit(logger) def ensemble_perturbations(self: Task) -> None: """Generate the 3D ensemble of perturbation for the 3DEnVAR diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 6f38232c6b3..198bbcd3efc 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -68,7 +68,6 @@ def initialize(self): # make directories and stage ensemble background files soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - self.task_config['NMEM_ENS']=3 stageconf = AttrDict() keys = ['current_cycle', 'previous_cycle', @@ -92,7 +91,6 @@ def initialize(self): # stage ensemble background files soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, stageconf) - print("soca_ens_bkg_stage_list: ",soca_ens_bkg_stage_list) FileHandler(soca_ens_bkg_stage_list).sync() # stage letkf-specific files @@ -137,7 +135,6 @@ def initialize(self): nml['fms_nml']['domains_stack_size'] = int(domain_stack_size) nml.write(self.task_config.mom_input_nml, force=True) # force to overwrite if necessary - @logit(logger) def run(self): """Method run for ocean and sea ice LETKF task @@ -177,7 +174,6 @@ def run(self): raise WorkflowException(f"An error occured during execution of {exec_cmd_letkf}") pass - @logit(logger) def finalize(self): """Method finalize for ocean and sea ice LETKF task From f314c2ab992dc424095dd432878811b1e20c189f Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 19 Sep 2024 17:16:48 +0000 Subject: [PATCH 08/33] style --- ush/python/pygfs/utils/marine_da_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 0fde56544d4..7ffe44fb432 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -74,4 +74,5 @@ def stage_ens_mem(task_config: AttrDict) -> None: logger.debug(f"{jinja.Jinja(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf).render}") letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) logger.info(f"{letkf_stage_list}") - FileHandler(letkf_stage_list).sync() \ No newline at end of file + FileHandler(letkf_stage_list).sync() + From c7cb09e724057ecc2f8fe1e29ddf43c956e4c24b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 19 Sep 2024 17:19:03 +0000 Subject: [PATCH 09/33] style --- ush/python/pygfs/utils/marine_da_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 7ffe44fb432..54cefd6805d 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -75,4 +75,3 @@ def stage_ens_mem(task_config: AttrDict) -> None: letkf_stage_list = parse_j2yaml(task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, ensbkgconf) logger.info(f"{letkf_stage_list}") FileHandler(letkf_stage_list).sync() - From 1e53cb5ef47804788f16c9e4e925af065259f878 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 19 Sep 2024 17:19:50 +0000 Subject: [PATCH 10/33] removed personal bits --- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml index b64e7579867..6e96b294844 100644 --- a/ci/cases/pr/C48mx500_hybAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -10,11 +10,10 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - #icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 idate: 2021032412 edate: 2021032418 nens: 3 - icsdir: /scratch1/NCEPDEV/da/Andrew.Eichmann/ICSDIR/C48mx500ens gfs_cyc: 0 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml From c66994e72d9ef79ab566cf53cef685ae36e51d7b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Fri, 20 Sep 2024 13:56:34 +0000 Subject: [PATCH 11/33] the great deawkwarding --- env/HERA.env | 6 +++--- env/ORION.env | 6 +++--- env/WCOSS2.env | 6 +++--- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 2 +- jobs/rocoto/{marineanalletkf.sh => marineanlletkf.sh} | 2 +- .../gfs/{config.marineanalletkf => config.marineanlletkf} | 8 ++++---- parm/config/gfs/config.resources | 4 ++-- ush/python/pygfs/task/marine_letkf.py | 4 ++-- workflow/applications/gfs_cycled.py | 4 ++-- workflow/rocoto/gfs_tasks.py | 8 ++++---- workflow/rocoto/tasks.py | 2 +- 11 files changed, 26 insertions(+), 26 deletions(-) rename jobs/rocoto/{marineanalletkf.sh => marineanlletkf.sh} (95%) rename parm/config/gfs/{config.marineanalletkf => config.marineanlletkf} (74%) diff --git a/env/HERA.env b/env/HERA.env index 0d77547b5b9..f17236ff0cd 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -155,10 +155,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLLETKF}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/ORION.env b/env/ORION.env index e8c1bcbf58e..a1d7dcc03f7 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -148,10 +148,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLLETKF}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index cea24fb26bd..80a81e3ddc2 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -126,10 +126,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLLETKF}" elif [[ "${step}" = "atmanlfv3inc" ]]; then diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 07e53093e91..80975fac767 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,6 +1,6 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanalletkf" -c "base ocnanal marineanalletkf" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base ocnanal marineanlletkf" ############################################## # Set variables used in the script diff --git a/jobs/rocoto/marineanalletkf.sh b/jobs/rocoto/marineanlletkf.sh similarity index 95% rename from jobs/rocoto/marineanalletkf.sh rename to jobs/rocoto/marineanlletkf.sh index f2bfb9f70c3..d4333461f3c 100755 --- a/jobs/rocoto/marineanalletkf.sh +++ b/jobs/rocoto/marineanlletkf.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="marineanalletkf" +export job="marineanlletkf" export jobid="${job}.$$" ############################################################### diff --git a/parm/config/gfs/config.marineanalletkf b/parm/config/gfs/config.marineanlletkf similarity index 74% rename from parm/config/gfs/config.marineanalletkf rename to parm/config/gfs/config.marineanlletkf index a53a16688a9..75a8a947b34 100644 --- a/parm/config/gfs/config.marineanalletkf +++ b/parm/config/gfs/config.marineanlletkf @@ -1,12 +1,12 @@ #!/bin/bash -########## config.marineanalletkf ########## +########## config.marineanlletkf ########## # Ocn Analysis specific -echo "BEGIN: config.marineanalletkf" +echo "BEGIN: config.marineanlletkf" # Get task specific resources -. "${EXPDIR}/config.resources" marineanalletkf +. "${EXPDIR}/config.resources" marineanlletkf export MARINE_LETKF_EXEC="${JEDI_BIN}/gdas.x" export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" @@ -16,4 +16,4 @@ export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j export GRIDGEN_EXEC="${JEDI_BIN}/gdas_soca_gridgen.x" export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" -echo "END: config.marineanalletkf" +echo "END: config.marineanlletkf" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index afc5939fcdd..18485936514 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -26,7 +26,7 @@ if (( $# != 1 )); then echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostbndpntbll wavepostpnt" echo "wavegempak waveawipsbulls waveawipsgridded" echo "postsnd awips gempak npoess" - echo "ocnanalprep prepoceanobs marinebmat ocnanalrun ocnanalecen marineanalletkf ocnanalchkpt ocnanalpost ocnanalvrfy" + echo "ocnanalprep prepoceanobs marinebmat ocnanalrun ocnanalecen marineanlletkf ocnanalchkpt ocnanalpost ocnanalvrfy" exit 1 fi @@ -600,7 +600,7 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; - "marineanalletkf") + "marineanlletkf") ntasks=16 case ${OCNRES} in "025") diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 198bbcd3efc..ca6fe2cd163 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -148,7 +148,7 @@ def run(self): logger.info("run") - exec_cmd_gridgen = Executable(self.task_config.APRUN_MARINEANALLETKF) + exec_cmd_gridgen = Executable(self.task_config.APRUN_MARINEANLLETKF) exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_EXEC) exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_YAML) @@ -161,7 +161,7 @@ def run(self): raise WorkflowException(f"An error occured during execution of {exec_cmd_gridgen}") pass - exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANALLETKF) + exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANLLETKF) for letkf_exec_arg in self.task_config.letkf_exec_args: exec_cmd_letkf.add_default_arg(letkf_exec_arg) diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index c05139729cb..0db08cff1ff 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -46,7 +46,7 @@ def _get_app_configs(self): if self.do_jediocnvar: configs += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] if self.do_hybvar: - configs += ['marineanalletkf', 'ocnanalecen'] + configs += ['marineanlletkf', 'ocnanalecen'] configs += ['ocnanalchkpt', 'ocnanalpost'] if self.do_vrfy_oceanda: configs += ['ocnanalvrfy'] @@ -148,7 +148,7 @@ def get_task_names(self): if self.do_jediocnvar: gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'ocnanalprep', 'marinebmat', 'ocnanalrun'] if self.do_hybvar: - gdas_gfs_common_tasks_before_fcst += ['marineanalletkf', 'ocnanalecen'] + gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] gdas_gfs_common_tasks_before_fcst += ['ocnanalchkpt', 'ocnanalpost'] if self.do_vrfy_oceanda: gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 0b963676b84..e9b79f730e7 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -665,7 +665,7 @@ def prepoceanobs(self): return task - def marineanalletkf(self): + def marineanlletkf(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) @@ -675,14 +675,14 @@ def marineanalletkf(self): deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('marineanalletkf') - task_name = f'{self.run}marineanalletkf' + resources = self.get_resource('marineanlletkf') + task_name = f'{self.run}marineanlletkf' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanalletkf.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlletkf.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 3ff8f2a6fac..a20e6a06e3a 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -15,7 +15,7 @@ class Tasks: 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', - 'ocnanalprep', 'marineanalletkf', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', + 'ocnanalprep', 'marineanlletkf', 'marinebmat', 'ocnanalrun', 'ocnanalecen', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', From 1069b583593efd6f1bdf986d171ea58c705ace8d Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 24 Sep 2024 15:34:34 +0000 Subject: [PATCH 12/33] using communal ICs --- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml index 6e96b294844..3c9038b7046 100644 --- a/ci/cases/pr/C48mx500_hybAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -10,7 +10,7 @@ arguments: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500 + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610 idate: 2021032412 edate: 2021032418 nens: 3 From 5c2140217433d00fe9f249becbffbea8b95fb98b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 24 Sep 2024 20:21:22 +0000 Subject: [PATCH 13/33] renaming --- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 2 +- ...rine_analysis_letkf.py => exglobal_marine_analysis_letkf.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename scripts/{exgdas_global_marine_analysis_letkf.py => exglobal_marine_analysis_letkf.py} (100%) diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 80975fac767..0d992191f3a 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -28,7 +28,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ ############################################################### # Run relevant script -EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exgdas_global_marine_analysis_letkf.py} +EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exglobal_marine_analysis_letkf.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/scripts/exgdas_global_marine_analysis_letkf.py b/scripts/exglobal_marine_analysis_letkf.py similarity index 100% rename from scripts/exgdas_global_marine_analysis_letkf.py rename to scripts/exglobal_marine_analysis_letkf.py From e6bc17b146648cecb5e8bbcdf44ef59d7db823e5 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 26 Sep 2024 13:53:01 +0000 Subject: [PATCH 14/33] rearrange task deps --- workflow/rocoto/gfs_tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index e9b79f730e7..b436c77566d 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -670,10 +670,11 @@ def marineanlletkf(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) deps = [] - data = f'{ocean_hist_path}/gdas.ocean.t@Hz.inst.f009.nc' - dep_dict = {'type': 'data', 'data': data, 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': 'enkfgdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = {'type': 'task', 'name': f'{self.run}prepoceanobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('marineanlletkf') task_name = f'{self.run}marineanlletkf' From b19d000f4501d077ca13cf38e6bb00421db7f679 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 30 Sep 2024 13:42:23 +0000 Subject: [PATCH 15/33] merge cleanup --- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 2 +- ush/python/pygfs/task/marine_letkf.py | 2 +- workflow/applications/gfs_cycled.py | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 0d992191f3a..da812455542 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,6 +1,6 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base ocnanal marineanlletkf" +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base marineanl marineanlletkf" ############################################## # Set variables used in the script diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index ca6fe2cd163..8d86d4ad1b2 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -97,7 +97,7 @@ def initialize(self): letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) FileHandler(letkf_stage_list).sync() - obs_list = parse_j2yaml(self.task_config.OBS_YAML, self.task_config) + obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) # get the list of observations obs_files = [] diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index a7021e2529e..cf83177828c 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -148,13 +148,8 @@ def get_task_names(self): if self.do_jediocnvar: gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: -<<<<<<< HEAD gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] - gdas_gfs_common_tasks_before_fcst += ['ocnanalchkpt', 'ocnanalpost'] -======= - gdas_gfs_common_tasks_before_fcst += ['ocnanalecen'] gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] ->>>>>>> develop if self.do_vrfy_oceanda: gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] From 926bbc35a84b26c9762406e0d0838f620637216f Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 3 Oct 2024 15:55:37 +0000 Subject: [PATCH 16/33] debugging after merge --- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 9 +++++++++ parm/config/gfs/config.marineanlletkf | 4 ++-- ush/python/pygfs/task/marine_letkf.py | 11 ++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index da812455542..83d1b98eaf3 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,5 +1,12 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" + +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/${jobid}" +# Create the directory to hold ensemble perturbations +export DATAens="${DATAjob}/ensdata" +if [[ ! -d "${DATAens}" ]]; then mkdir -p "${DATAens}"; fi + source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base marineanl marineanlletkf" ############################################## @@ -11,6 +18,8 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} +export GDUMP="gdas" +export GDUMP_ENS="enkf${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ diff --git a/parm/config/gfs/config.marineanlletkf b/parm/config/gfs/config.marineanlletkf index 75a8a947b34..e598cfba20c 100644 --- a/parm/config/gfs/config.marineanlletkf +++ b/parm/config/gfs/config.marineanlletkf @@ -8,12 +8,12 @@ echo "BEGIN: config.marineanlletkf" # Get task specific resources . "${EXPDIR}/config.resources" marineanlletkf -export MARINE_LETKF_EXEC="${JEDI_BIN}/gdas.x" +export MARINE_LETKF_EXEC="${EXECgfs}/gdas.x" export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j2" -export GRIDGEN_EXEC="${JEDI_BIN}/gdas_soca_gridgen.x" +export GRIDGEN_EXEC="${EXECgfs}/gdas_soca_gridgen.x" export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" echo "END: config.marineanlletkf" diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 8d86d4ad1b2..fd3f4421e2a 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -43,6 +43,8 @@ def __init__(self, config: Dict) -> None: 'soca', 'localensembleda', _letkf_yaml_file] + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert + _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle self.task_config.WINDOW_BEGIN = self.task_config.current_cycle - _half_assim_freq @@ -50,8 +52,9 @@ def __init__(self, config: Dict) -> None: self.task_config.letkf_yaml_file = _letkf_yaml_file self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') - self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') - + self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') + self.task_config.ENSPERT_RELPATH = _enspert_relpath + @logit(logger) def initialize(self): """Method initialize for ocean and sea ice LETKF task @@ -80,6 +83,8 @@ def initialize(self): 'COMOUT_ICE_LETKF', 'COMOUT_OCEAN_LETKF', 'DATA', + 'ENSPERT_RELPATH', + 'GDUMP_ENS', 'NMEM_ENS', 'PARMgfs', 'ROTDIR', @@ -90,7 +95,7 @@ def initialize(self): stageconf.RUN = 'enkfgdas' # stage ensemble background files - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, stageconf) + soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconf) FileHandler(soca_ens_bkg_stage_list).sync() # stage letkf-specific files From b4bafa31a0d14ef379958fa3fe51c7a2b55d9268 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 9 Oct 2024 18:09:45 +0000 Subject: [PATCH 17/33] fixing halo size key --- parm/config/gfs/config.marineanlletkf | 1 + ush/python/pygfs/task/marine_letkf.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/parm/config/gfs/config.marineanlletkf b/parm/config/gfs/config.marineanlletkf index e598cfba20c..8b84af4eaa5 100644 --- a/parm/config/gfs/config.marineanlletkf +++ b/parm/config/gfs/config.marineanlletkf @@ -15,5 +15,6 @@ export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j export GRIDGEN_EXEC="${EXECgfs}/gdas_soca_gridgen.x" export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" +export DIST_HALO_SIZE=500000 echo "END: config.marineanlletkf" diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index fd3f4421e2a..d1187f2e621 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -108,6 +108,9 @@ def initialize(self): obs_files = [] for ob in obs_list['observers']: obs_name = ob['obs space']['name'].lower() + # TODO(AFE) - this should be removed when the obs config yamls are jinjafied + if 'distribution' not in ob['obs space']: + ob['obs space']['distribution'] = {'name': 'Halo', 'halo size': self.task_config['DIST_HALO_SIZE']} obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc4" obs_files.append((obs_filename, ob)) From 40fea74ef222b73934d642c31485b9b8bad4bdbb Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 9 Oct 2024 19:35:34 +0000 Subject: [PATCH 18/33] thou shall not pass --- ush/python/pygfs/task/marine_letkf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index d1187f2e621..4273a5b45fa 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -167,7 +167,6 @@ def run(self): raise OSError(f"Failed to execute {exec_cmd_gridgen}") except Exception: raise WorkflowException(f"An error occured during execution of {exec_cmd_gridgen}") - pass exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANLLETKF) for letkf_exec_arg in self.task_config.letkf_exec_args: @@ -180,7 +179,6 @@ def run(self): raise OSError(f"Failed to execute {exec_cmd_letkf}") except Exception: raise WorkflowException(f"An error occured during execution of {exec_cmd_letkf}") - pass @logit(logger) def finalize(self): From 938fa53300cfd2b1479227d05df58e3591424cbf Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 9 Oct 2024 20:26:46 +0000 Subject: [PATCH 19/33] fix workflow generator --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index f54d51e73ae..5b412400cf9 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -670,7 +670,7 @@ def marineanlletkf(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) deps = [] - dep_dict = {'type': 'metatask', 'name': 'enkfgdasfcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}prepoceanobs'} deps.append(rocoto.add_dependency(dep_dict)) From 77f44e1056b1ecc52d6fedf4714bf9d6cfe8a89a Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 10 Oct 2024 14:32:33 +0000 Subject: [PATCH 20/33] ocn inc corrected for gefs --- ush/forecast_postdet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index f01bf28e161..8d6ed38d0f3 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -463,7 +463,7 @@ MOM6_postdet() { if [[ "${RUN}" == "gefs" ]]; then # to ensure it does not interfere with the GFS when ensemble is updated in the GFS if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then - ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ + ${NCP} "${COMIN_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" \ || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) fi fi # if [[ "${RUN}" == "gefs" ]]; then From dcd4322741a80aae171f1ad98ba8c42cfd57e3d6 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Fri, 11 Oct 2024 13:46:35 +0000 Subject: [PATCH 21/33] using GDUMP_ENS --- ush/python/pygfs/task/marine_letkf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 4273a5b45fa..f4373fda4f7 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -92,7 +92,6 @@ def initialize(self): 'WINDOW_MIDDLE'] for key in keys: stageconf[key] = self.task_config[key] - stageconf.RUN = 'enkfgdas' # stage ensemble background files soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconf) @@ -194,10 +193,9 @@ def finalize(self): logger.info("finalize") letkfsaveconf = AttrDict() - keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', + keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', 'GDUMP_ENS', 'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL'] for key in keys: letkfsaveconf[key] = self.task_config[key] - letkfsaveconf.RUN = 'enkfgdas' letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf) FileHandler(letkf_save_list).sync() From 8024e5fe373cafddbc5e54ade76241cc1c7f5f7d Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 16 Oct 2024 18:20:32 +0000 Subject: [PATCH 22/33] add underscore in task names --- workflow/rocoto/gfs_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 139c15e853f..1575892a75f 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -670,9 +670,9 @@ def marineanlletkf(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = {'type': 'task', 'name': f'{self.run}prepoceanobs'} + dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From c4f289b3dcbf4235c1e035f78a456d0c55655925 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Wed, 16 Oct 2024 18:23:58 +0000 Subject: [PATCH 23/33] comment pruning --- ush/forecast_postdet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 8d6ed38d0f3..e562b8cd753 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -461,7 +461,7 @@ MOM6_postdet() { # GEFS perturbations if [[ "${RUN}" == "gefs" ]]; then - # to ensure it does not interfere with the GFS when ensemble is updated in the GFS + # to ensure it does not interfere with the GFS if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then ${NCP} "${COMIN_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" \ || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) From afc8257d4d0b8df2ba0c60fed53afd9252883a32 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:28:34 -0400 Subject: [PATCH 24/33] Update workflow/rocoto/gfs_tasks.py Co-authored-by: Walter Kolczynski - NOAA --- workflow/rocoto/gfs_tasks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 1575892a75f..ca4614a6ca2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -667,8 +667,6 @@ def prepoceanobs(self): def marineanlletkf(self): - ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) - deps = [] dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) From 7716bdddc6879cfa8f30a2a2daaa57495a6b156b Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 17 Oct 2024 01:47:38 +0000 Subject: [PATCH 25/33] review stuff --- env/WCOSS2.env | 2 +- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 2 ++ ush/python/pygfs/task/marine_letkf.py | 4 +++- workflow/rocoto/gfs_tasks.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 80a81e3ddc2..c67c16f929d 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -129,7 +129,7 @@ elif [[ "${step}" = "ocnanalecen" ]]; then elif [[ "${step}" = "marineanlletkf" ]]; then export NTHREADS_MARINEANLLETKF=${NTHREADSmax} - export APRUN_MARINEANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLLETKF}" + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "atmanlfv3inc" ]]; then diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 83d1b98eaf3..d88e85bddae 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -20,6 +20,8 @@ gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} export GDUMP="gdas" export GDUMP_ENS="enkf${GDUMP}" +export RUN="${GDUMP}" +export OPREFIX="${RUN}.t${cyc}z." YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index f4373fda4f7..ad9bbac80fa 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -54,7 +54,7 @@ def __init__(self, config: Dict) -> None: self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') self.task_config.ENSPERT_RELPATH = _enspert_relpath - + @logit(logger) def initialize(self): """Method initialize for ocean and sea ice LETKF task @@ -86,8 +86,10 @@ def initialize(self): 'ENSPERT_RELPATH', 'GDUMP_ENS', 'NMEM_ENS', + 'OPREFIX', 'PARMgfs', 'ROTDIR', + 'RUN', 'WINDOW_BEGIN', 'WINDOW_MIDDLE'] for key in keys: diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ca4614a6ca2..3063cc95206 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -675,7 +675,7 @@ def marineanlletkf(self): dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('marineanlletkf') - task_name = f'{self.run}marineanlletkf' + task_name = f'{self.run}_marineanlletkf' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, From ee50ae6e6543e9427d859d92bbb46b2d9dbc6070 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Thu, 17 Oct 2024 12:56:44 +0000 Subject: [PATCH 26/33] style points --- ush/python/pygfs/task/marine_letkf.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index ad9bbac80fa..7d05c25cdfd 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -43,7 +43,7 @@ def __init__(self, config: Dict) -> None: 'soca', 'localensembleda', _letkf_yaml_file] - # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle @@ -52,9 +52,9 @@ def __init__(self, config: Dict) -> None: self.task_config.letkf_yaml_file = _letkf_yaml_file self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') - self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') + self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') self.task_config.ENSPERT_RELPATH = _enspert_relpath - + @logit(logger) def initialize(self): """Method initialize for ocean and sea ice LETKF task From f200ec05ab63bb77764bb3b5d71567f38127d503 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Fri, 18 Oct 2024 13:29:45 -0400 Subject: [PATCH 27/33] Update jobs/JGLOBAL_MARINE_ANALYSIS_LETKF Co-authored-by: Walter Kolczynski - NOAA --- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index d88e85bddae..2a88f89eab7 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -20,10 +20,9 @@ gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} export GDUMP="gdas" export GDUMP_ENS="enkf${GDUMP}" -export RUN="${GDUMP}" export OPREFIX="${RUN}.t${cyc}z." -YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ +RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL From d48236f9bd68e65f50727e6e85af0f7f91e6f726 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 28 Oct 2024 16:27:25 +0000 Subject: [PATCH 28/33] review changes --- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 2 ++ env/HERA.env | 2 +- env/ORION.env | 2 +- ush/python/pygfs/task/marine_letkf.py | 17 +++-------------- workflow/rocoto/gfs_tasks.py | 2 +- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml index 3c9038b7046..d7f3e8e0683 100644 --- a/ci/cases/pr/C48mx500_hybAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -22,3 +22,5 @@ skip_ci_on_hosts: - wcoss2 - orion - hercules + - hera + - gaea diff --git a/env/HERA.env b/env/HERA.env index acc74d6864e..e8323c16fd1 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -156,7 +156,7 @@ elif [[ "${step}" = "ocnanalecen" ]]; then elif [[ "${step}" = "marineanlletkf" ]]; then export NTHREADS_MARINEANLLETKF=${NTHREADSmax} - export APRUN_MARINEANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLLETKF}" + export APRUN_MARINEANLLETKF=${APRUN_default} elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 295c5652409..c0c2212a365 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -152,7 +152,7 @@ elif [[ "${step}" = "ocnanalecen" ]]; then elif [[ "${step}" = "marineanlletkf" ]]; then export NTHREADS_MARINEANLLETKF=${NTHREADSmax} - export APRUN_MARINEANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANLLETKF}" + export APRUN_MARINEANLLETKF="${APRUN_default} elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 7d05c25cdfd..bcbaa809b0a 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import f90nml +import pygfs.utils.marine_da_utils as mdau from logging import getLogger import os from pygfs.task.analysis import Analysis @@ -161,25 +162,13 @@ def run(self): exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_EXEC) exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_YAML) - try: - logger.debug(f"Executing {exec_cmd_gridgen}") - exec_cmd_gridgen() - except OSError: - raise OSError(f"Failed to execute {exec_cmd_gridgen}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd_gridgen}") + mdau.run(exec_cmd_gridgen) exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANLLETKF) for letkf_exec_arg in self.task_config.letkf_exec_args: exec_cmd_letkf.add_default_arg(letkf_exec_arg) - try: - logger.debug(f"Executing {exec_cmd_letkf}") - exec_cmd_letkf() - except OSError: - raise OSError(f"Failed to execute {exec_cmd_letkf}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd_letkf}") + mdau.run(exec_cmd_letkf) @logit(logger) def finalize(self): diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 3063cc95206..e81502f702c 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -668,7 +668,7 @@ def prepoceanobs(self): def marineanlletkf(self): deps = [] - dep_dict = {'type': 'metatask', 'name': f'{self.run}_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'enkfgdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} deps.append(rocoto.add_dependency(dep_dict)) From 522d3c57ab6eb4f1e2307593fa1e988a275819c9 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 28 Oct 2024 18:34:33 +0000 Subject: [PATCH 29/33] merged and tested --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 9478dbf198b..d3bb68a6b88 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -667,7 +667,7 @@ def prepoceanobs(self): def marineanlletkf(self): deps = [] - dep_dict = {'type': 'metatask', 'name': f'enkfgdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + dep_dict = {'type': 'metatask', 'name': f'enkfgdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} deps.append(rocoto.add_dependency(dep_dict)) From 3da66c3eee5568d0efe7475827b2f33401b0b802 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Mon, 28 Oct 2024 18:36:51 +0000 Subject: [PATCH 30/33] orion env fix --- env/ORION.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/ORION.env b/env/ORION.env index c0c2212a365..40434f407bf 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -152,7 +152,7 @@ elif [[ "${step}" = "ocnanalecen" ]]; then elif [[ "${step}" = "marineanlletkf" ]]; then export NTHREADS_MARINEANLLETKF=${NTHREADSmax} - export APRUN_MARINEANLLETKF="${APRUN_default} + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then From eee78a140a8d27af08ec08413467715dc16a703e Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 29 Oct 2024 14:41:03 +0000 Subject: [PATCH 31/33] review and merge tweaks --- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 2 +- parm/stage/ocean_ens_perturbations.yaml.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml index d7f3e8e0683..036aa8ca603 100644 --- a/ci/cases/pr/C48mx500_hybAOWCDA.yaml +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -14,7 +14,7 @@ arguments: idate: 2021032412 edate: 2021032418 nens: 3 - gfs_cyc: 0 + interval: 0 start: warm yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml diff --git a/parm/stage/ocean_ens_perturbations.yaml.j2 b/parm/stage/ocean_ens_perturbations.yaml.j2 index fede3816a70..5fc58cb349f 100644 --- a/parm/stage/ocean_ens_perturbations.yaml.j2 +++ b/parm/stage/ocean_ens_perturbations.yaml.j2 @@ -9,5 +9,5 @@ ocean_ens_perturbation: {% for mem in range(first_mem + 1, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] {% endfor %} # mem loop From 02388e637a3fa0086c64b45122cc9d947bc460ee Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA Date: Tue, 29 Oct 2024 15:17:07 +0000 Subject: [PATCH 32/33] review tweak --- parm/stage/ocean_ens_perturbations.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/stage/ocean_ens_perturbations.yaml.j2 b/parm/stage/ocean_ens_perturbations.yaml.j2 index 5fc58cb349f..586b9f66cb2 100644 --- a/parm/stage/ocean_ens_perturbations.yaml.j2 +++ b/parm/stage/ocean_ens_perturbations.yaml.j2 @@ -9,5 +9,5 @@ ocean_ens_perturbation: {% for mem in range(first_mem + 1, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc"] {% endfor %} # mem loop From 9fa78132b0f25f80f645b952ed6ca839dd8ccd68 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:30:55 -0400 Subject: [PATCH 33/33] Update ush/python/pygfs/task/marine_letkf.py Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ush/python/pygfs/task/marine_letkf.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index bcbaa809b0a..54d40f8d665 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -12,8 +12,7 @@ logit, parse_j2yaml, to_timedelta, - to_YMDH, - WorkflowException) + to_YMDH) logger = getLogger(__name__.split('.')[-1])