From b5eb3454e020a2e908ba2a639c125e87012bcba2 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 29 May 2024 14:35:29 +0000 Subject: [PATCH 01/74] Use rename_atm of jcb parm erpos --- sorc/gdas.cd | 2 +- sorc/jcb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6742ec62a12..822a50258b8 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6742ec62a12d7d6f8129057bcf77cee0e2175022 +Subproject commit 822a50258b869a8e855bd867f3194eae6c5b45ef diff --git a/sorc/jcb b/sorc/jcb index de75655d81e..4e55941a2d7 160000 --- a/sorc/jcb +++ b/sorc/jcb @@ -1 +1 @@ -Subproject commit de75655d81ec2ee668d8d47bf4a43625c81dde7c +Subproject commit 4e55941a2d7c42904d1aa2acd2a2ed1323d747a5 From 3a8f527e9f5cb7deec201f16da6d42347fce66dc Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 31 May 2024 12:32:32 -0500 Subject: [PATCH 02/74] Initial commit --- parm/gdas/atm_berror_bump_fix.yaml.j2 | 23 +++++++++++++++++++ parm/gdas/atm_berror_gsibec_fix.yaml.j2 | 10 +++++++++ parm/gdas/atm_berror_identity_fix.yaml.j2 | 2 ++ parm/gdas/atm_bkg_fix.yaml.j2 | 16 ++++++++++++++ parm/gdas/atm_fv3ens_fix.yaml.j2 | 27 +++++++++++++++++++++++ 5 files changed, 78 insertions(+) create mode 100644 parm/gdas/atm_berror_bump_fix.yaml.j2 create mode 100644 parm/gdas/atm_berror_gsibec_fix.yaml.j2 create mode 100644 parm/gdas/atm_berror_identity_fix.yaml.j2 create mode 100644 parm/gdas/atm_bkg_fix.yaml.j2 create mode 100644 parm/gdas/atm_fv3ens_fix.yaml.j2 diff --git a/parm/gdas/atm_berror_bump_fix.yaml.j2 b/parm/gdas/atm_berror_bump_fix.yaml.j2 new file mode 100644 index 00000000000..817f1f3a238 --- /dev/null +++ b/parm/gdas/atm_berror_bump_fix.yaml.j2 @@ -0,0 +1,23 @@ +{% set src_dir = BERROR_DATA_DIR %} +{% set dest_dir = DATA ~ '/berror' %} +{% set ftype_list = ['cor_rh', 'cor_rv', 'stddev'] %} +{% set nprocs = ntiles * layout_x * layout_y %} + +mkdir: +- '{{ dest_dir }}' + +copy: +{% for ftype in ftype_list %} +{% set prefix = BERROR_DATE | to_fv3time ~ '.' ~ ftype %} +{% set fname = prefix ~ '.coupler.res' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% for itile in range(1,7) %} +{% set fname = prefix ~ '.fv_tracer.res.tile' ~ itile ~ '.nc' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% endfor %} +{% endfor %} + +{% for nn in range(1,nprocs+1) %} +{% set fname = 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% endfor %} \ No newline at end of file diff --git a/parm/gdas/atm_berror_gsibec_fix.yaml.j2 b/parm/gdas/atm_berror_gsibec_fix.yaml.j2 new file mode 100644 index 00000000000..a640e2167de --- /dev/null +++ b/parm/gdas/atm_berror_gsibec_fix.yaml.j2 @@ -0,0 +1,10 @@ +{% set src_dir = HOMEgfs ~ '/fix/gdas/gsibec/' ~ CASE_ANL %} +{% set dest_dir = DATA ~ '/berror' %} +{% set fname_list = ['gfs_gsi_global.nml', 'gsi-coeffs-gfs-global.nc4'] %} + +mkdir: +- '{{ dest_dir }}' +copy: +{% for fname in fname_list %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% endfor %} \ No newline at end of file diff --git a/parm/gdas/atm_berror_identity_fix.yaml.j2 b/parm/gdas/atm_berror_identity_fix.yaml.j2 new file mode 100644 index 00000000000..0a45bc9a4f6 --- /dev/null +++ b/parm/gdas/atm_berror_identity_fix.yaml.j2 @@ -0,0 +1,2 @@ +mkdir: +copy: \ No newline at end of file diff --git a/parm/gdas/atm_bkg_fix.yaml.j2 b/parm/gdas/atm_bkg_fix.yaml.j2 new file mode 100644 index 00000000000..32c95fd04da --- /dev/null +++ b/parm/gdas/atm_bkg_fix.yaml.j2 @@ -0,0 +1,16 @@ +{% set src_dir = COM_ATMOS_RESTART_PREV %} +{% set dest_dir = DATA ~ '/bkg' %} +{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} +{% set prefix = current_cycle | to_fv3time %} + +mkdir: +- '{{ dest_dir }}' +copy: +{% set fname = prefix ~ '.coupler.res' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% for ftype in ftype_list %} +{% for itile in range(1,7) %} +{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% endfor %} +{% endfor %} \ No newline at end of file diff --git a/parm/gdas/atm_fv3ens_fix.yaml.j2 b/parm/gdas/atm_fv3ens_fix.yaml.j2 new file mode 100644 index 00000000000..a6f8c3847a1 --- /dev/null +++ b/parm/gdas/atm_fv3ens_fix.yaml.j2 @@ -0,0 +1,27 @@ +{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} +{% set prefix = current_cycle | to_fv3time %} + +mkdir: +{% for imem in range(1,NMEM_ENS+1) %} +{% set memchar = 'mem%03d' | format(imem) %} +{% set dest_dir = DATA ~ '/ens/' ~ memchar %} +- '{{ dest_dir }}' +{% endfor %} +copy: +{% for imem in range(1,NMEM_ENS+1) %} +{% set memchar = 'mem%03d' | format(imem) %} +{% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) + | replace('${RUN}', 'enkfgdas') + | replace('${YMD}', previous_cycle | to_YMD) + | replace('${HH}', previous_cycle | strftime('%H')) + | replace('${MEMDIR}', memchar) %} +{% set dest_dir = DATA ~ '/ens/' ~ memchar %} +{% set fname = prefix ~ '.coupler.res' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% for ftype in ftype_list %} +{% for itile in range(1,7) %} +{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% endfor %} +{% endfor %} +{% endfor %} From 690498229b9ff2df9952c3d50389244f593ef322 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:28:10 -0400 Subject: [PATCH 03/74] Update atm_berror_bump_fix.yaml.j2 --- parm/gdas/atm_berror_bump_fix.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/atm_berror_bump_fix.yaml.j2 b/parm/gdas/atm_berror_bump_fix.yaml.j2 index 817f1f3a238..123d15e3591 100644 --- a/parm/gdas/atm_berror_bump_fix.yaml.j2 +++ b/parm/gdas/atm_berror_bump_fix.yaml.j2 @@ -20,4 +20,4 @@ copy: {% for nn in range(1,nprocs+1) %} {% set fname = 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% endfor %} \ No newline at end of file +{% endfor %} From 0b37fe579f7fea6e221ab324c4620502240766ad Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:28:36 -0400 Subject: [PATCH 04/74] Update atm_berror_gsibec_fix.yaml.j2 --- parm/gdas/atm_berror_gsibec_fix.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/atm_berror_gsibec_fix.yaml.j2 b/parm/gdas/atm_berror_gsibec_fix.yaml.j2 index a640e2167de..0affe68030c 100644 --- a/parm/gdas/atm_berror_gsibec_fix.yaml.j2 +++ b/parm/gdas/atm_berror_gsibec_fix.yaml.j2 @@ -7,4 +7,4 @@ mkdir: copy: {% for fname in fname_list %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% endfor %} \ No newline at end of file +{% endfor %} From d770b63a051ba0b1437f2d3db5cd3a7796b34dcf Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:29:02 -0400 Subject: [PATCH 05/74] Update atm_berror_identity_fix.yaml.j2 --- parm/gdas/atm_berror_identity_fix.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/atm_berror_identity_fix.yaml.j2 b/parm/gdas/atm_berror_identity_fix.yaml.j2 index 0a45bc9a4f6..eec8c038c52 100644 --- a/parm/gdas/atm_berror_identity_fix.yaml.j2 +++ b/parm/gdas/atm_berror_identity_fix.yaml.j2 @@ -1,2 +1,2 @@ mkdir: -copy: \ No newline at end of file +copy: From 0187bbab807cf9ebb3e0a48a72e6a7153e7e6136 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:29:40 -0400 Subject: [PATCH 06/74] Update atm_bkg_fix.yaml.j2 --- parm/gdas/atm_bkg_fix.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/atm_bkg_fix.yaml.j2 b/parm/gdas/atm_bkg_fix.yaml.j2 index 32c95fd04da..8e88ef3c568 100644 --- a/parm/gdas/atm_bkg_fix.yaml.j2 +++ b/parm/gdas/atm_bkg_fix.yaml.j2 @@ -13,4 +13,4 @@ copy: {% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] {% endfor %} -{% endfor %} \ No newline at end of file +{% endfor %} From 4dcd47add81758128a1763cba662f9cf4e3e75a5 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 3 Jun 2024 13:03:52 -0500 Subject: [PATCH 07/74] Rename some things --- ..._berror_bump_fix.yaml.j2 => atm_berror_bump_staging.yaml.j2} | 0 ...ror_gsibec_fix.yaml.j2 => atm_berror_gsibec_staging.yaml.j2} | 0 parm/gdas/atm_berror_identity_fix.yaml.j2 | 2 -- parm/gdas/{atm_bkg_fix.yaml.j2 => atm_var_bkg_staging.yaml.j2} | 0 4 files changed, 2 deletions(-) rename parm/gdas/{atm_berror_bump_fix.yaml.j2 => atm_berror_bump_staging.yaml.j2} (100%) rename parm/gdas/{atm_berror_gsibec_fix.yaml.j2 => atm_berror_gsibec_staging.yaml.j2} (100%) delete mode 100644 parm/gdas/atm_berror_identity_fix.yaml.j2 rename parm/gdas/{atm_bkg_fix.yaml.j2 => atm_var_bkg_staging.yaml.j2} (100%) diff --git a/parm/gdas/atm_berror_bump_fix.yaml.j2 b/parm/gdas/atm_berror_bump_staging.yaml.j2 similarity index 100% rename from parm/gdas/atm_berror_bump_fix.yaml.j2 rename to parm/gdas/atm_berror_bump_staging.yaml.j2 diff --git a/parm/gdas/atm_berror_gsibec_fix.yaml.j2 b/parm/gdas/atm_berror_gsibec_staging.yaml.j2 similarity index 100% rename from parm/gdas/atm_berror_gsibec_fix.yaml.j2 rename to parm/gdas/atm_berror_gsibec_staging.yaml.j2 diff --git a/parm/gdas/atm_berror_identity_fix.yaml.j2 b/parm/gdas/atm_berror_identity_fix.yaml.j2 deleted file mode 100644 index 0a45bc9a4f6..00000000000 --- a/parm/gdas/atm_berror_identity_fix.yaml.j2 +++ /dev/null @@ -1,2 +0,0 @@ -mkdir: -copy: \ No newline at end of file diff --git a/parm/gdas/atm_bkg_fix.yaml.j2 b/parm/gdas/atm_var_bkg_staging.yaml.j2 similarity index 100% rename from parm/gdas/atm_bkg_fix.yaml.j2 rename to parm/gdas/atm_var_bkg_staging.yaml.j2 From f91297dd7b71d5cee77b03e2d6af249529dccbbe Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 3 Jun 2024 13:08:13 -0500 Subject: [PATCH 08/74] Rename some things --- parm/gdas/atm_lgetkf_bkg_staging.yaml.j2 | 27 +++++++++++++++++++ ...yaml.j2 => atm_var_fv3ens_staging.yaml.j2} | 0 2 files changed, 27 insertions(+) create mode 100644 parm/gdas/atm_lgetkf_bkg_staging.yaml.j2 rename parm/gdas/{atm_fv3ens_fix.yaml.j2 => atm_var_fv3ens_staging.yaml.j2} (100%) diff --git a/parm/gdas/atm_lgetkf_bkg_staging.yaml.j2 b/parm/gdas/atm_lgetkf_bkg_staging.yaml.j2 new file mode 100644 index 00000000000..f62fdd78414 --- /dev/null +++ b/parm/gdas/atm_lgetkf_bkg_staging.yaml.j2 @@ -0,0 +1,27 @@ +{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} +{% set prefix = current_cycle | to_fv3time %} + +mkdir: +{% for imem in range(1,NMEM_ENS+1) %} +{% set memchar = 'mem%03d' | format(imem) %} +- '{{ DATA }}/bkg/{{ memchar }}' +- '{{ DATA }}/anl/{{ memchar }}' +{% endfor %} +copy: +{% for imem in range(1,NMEM_ENS+1) %} +{% set memchar = 'mem%03d' | format(imem) %} +{% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) + | replace('${RUN}', 'enkfgdas') + | replace('${YMD}', previous_cycle | to_YMD) + | replace('${HH}', previous_cycle | strftime('%H')) + | replace('${MEMDIR}', memchar) %} +{% set dest_dir = DATA ~ '/bkg/' ~ memchar %} +{% set fname = prefix ~ '.coupler.res' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% for ftype in ftype_list %} +{% for itile in range(1,7) %} +{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} +- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +{% endfor %} +{% endfor %} +{% endfor %} diff --git a/parm/gdas/atm_fv3ens_fix.yaml.j2 b/parm/gdas/atm_var_fv3ens_staging.yaml.j2 similarity index 100% rename from parm/gdas/atm_fv3ens_fix.yaml.j2 rename to parm/gdas/atm_var_fv3ens_staging.yaml.j2 From 983d720cdf364f82b515c71c9f827b1721f98e86 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 3 Jun 2024 13:19:59 -0500 Subject: [PATCH 09/74] Add changes to analysis python scripts --- parm/config/gfs/config.atmanl | 3 + parm/config/gfs/config.atmensanl | 1 + ush/python/pygfs/task/analysis.py | 100 ----------- ush/python/pygfs/task/atm_analysis.py | 206 ++--------------------- ush/python/pygfs/task/atmens_analysis.py | 30 +--- 5 files changed, 16 insertions(+), 324 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index dd8ca80b11c..a9aa8201289 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -23,6 +23,9 @@ fi export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" +export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_var_bkg_staging.yaml.j2" +export BERROR_STAGING_YAML="${PARMgfs}/gdas/atm_berror_"${STATICB_TYPE}"_staging.yaml.j2" +export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/atm_fv3ens_staging.yaml.j2" export layout_x_atmanl=@LAYOUT_X_ATMANL@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index 3484cb670dd..c13ce7a0b4c 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -12,6 +12,7 @@ export INTERP_METHOD='barycentric' export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" +export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_ens_bkg_fix.yaml.j2" export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index b668ac39804..4b9d6d8bbdc 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -194,44 +194,6 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li except (AttributeError, RuntimeError): pass # checksum is missing, move on - @logit(logger) - def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: - """Compile a dictionary of model background files to copy - - This method is a placeholder for now... will be possibly made generic at a later date - - Parameters - ---------- - task_config: Dict - a dictionary containing all of the configuration needed for the task - - Returns - ---------- - bkg_dict: Dict - a dictionary containing the list of model background files to copy for FileHandler - """ - bkg_dict = {'foo': 'bar'} - return bkg_dict - - @logit(logger) - def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: - """Compile a dictionary of background error files to copy - - This method is a placeholder for now... will be possibly made generic at a later date - - Parameters - ---------- - config: Dict - a dictionary containing all of the configuration needed - - Returns - ---------- - berror_dict: Dict - a dictionary containing the list of background error files to copy for FileHandler - """ - berror_dict = {'foo': 'bar'} - return berror_dict - @logit(logger) def link_jediexe(self) -> None: """Compile a dictionary of background error files to copy @@ -258,68 +220,6 @@ def link_jediexe(self) -> None: return exe_dest - @staticmethod - @logit(logger) - def get_fv3ens_dict(config: Dict[str, Any]) -> Dict[str, Any]: - """Compile a dictionary of ensemble member restarts to copy - - This method constructs a dictionary of ensemble FV3 restart files (coupler, core, tracer) - that are needed for global atmens DA and returns said dictionary for use by the FileHandler class. - - Parameters - ---------- - config: Dict - a dictionary containing all of the configuration needed - - Returns - ---------- - ens_dict: Dict - a dictionary containing the list of ensemble member restart files to copy for FileHandler - """ - # NOTE for now this is FV3 restart files and just assumed to be fh006 - - # define template - template_res = config.COM_ATMOS_RESTART_TMPL - prev_cycle = config.previous_cycle - tmpl_res_dict = { - 'ROTDIR': config.ROTDIR, - 'RUN': config.RUN, - 'YMD': to_YMD(prev_cycle), - 'HH': prev_cycle.strftime('%H'), - 'MEMDIR': None - } - - # construct ensemble member file list - dirlist = [] - enslist = [] - for imem in range(1, config.NMEM_ENS + 1): - memchar = f"mem{imem:03d}" - - # create directory path for ensemble member restart - dirlist.append(os.path.join(config.DATA, config.dirname, f'mem{imem:03d}')) - - # get FV3 restart files, this will be a lot simpler when using history files - tmpl_res_dict['MEMDIR'] = memchar - rst_dir = Template.substitute_structure(template_res, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_res_dict.get) - run_dir = os.path.join(config.DATA, config.dirname, memchar) - - # atmens DA needs coupler - basename = f'{to_fv3time(config.current_cycle)}.coupler.res' - enslist.append([os.path.join(rst_dir, basename), os.path.join(config.DATA, config.dirname, memchar, basename)]) - - # atmens DA needs core, srf_wnd, tracer, phy_data, sfc_data - for ftype in ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data']: - template = f'{to_fv3time(config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' - for itile in range(1, config.ntiles + 1): - basename = template.format(tilenum=itile) - enslist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - ens_dict = { - 'mkdir': dirlist, - 'copy': enslist, - } - return ens_dict - @staticmethod @logit(logger) def tgz_diags(statfile: str, diagdir: str) -> None: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 95545c57a44..5f61d924156 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -85,22 +85,22 @@ def initialize(self: Analysis) -> None: # stage static background error files, otherwise it will assume ID matrix logger.info(f"Stage files for STATICB_TYPE {self.task_config.STATICB_TYPE}") - FileHandler(self.get_berror_dict(self.task_config)).sync() + if self.task_config.STATICB_TYPE != 'identity': + berror_fix_list = parse_j2yaml(self.task_config.BERROR_STAGING_YAML, self.task_config) + else: + berror_fix_list = {} + FileHandler(berror_fix_list).sync() # stage ensemble files for use in hybrid background error if self.task_config.DOHYBVAR: logger.debug(f"Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") - localconf = AttrDict() - keys = ['COM_ATMOS_RESTART_TMPL', 'previous_cycle', 'ROTDIR', 'RUN', - 'NMEM_ENS', 'DATA', 'current_cycle', 'ntiles'] - for key in keys: - localconf[key] = self.task_config[key] - localconf.RUN = 'enkfgdas' - localconf.dirname = 'ens' - FileHandler(self.get_fv3ens_dict(localconf)).sync() + fv3ens_fix_list = parse_j2yaml(self.task_config.FV3ENS_STAGING_YAML, self.task_config) + FileHandler(fv3ens_fix_list).sync() # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config))).sync() + logger.info(f"Staging background files from {self.task_config.BKG_FIX_YAML}") + bkg_fix_list = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) + FileHandler(bkg_fix_list).sync() # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") @@ -253,189 +253,3 @@ def finalize(self: Analysis) -> None: def clean(self): super().clean() - - @logit(logger) - def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: - """Compile a dictionary of model background files to copy - - This method constructs a dictionary of FV3 restart files (coupler, core, tracer) - that are needed for global atm DA and returns said dictionary for use by the FileHandler class. - - Parameters - ---------- - task_config: Dict - a dictionary containing all of the configuration needed for the task - - Returns - ---------- - bkg_dict: Dict - a dictionary containing the list of model background files to copy for FileHandler - """ - # NOTE for now this is FV3 restart files and just assumed to be fh006 - - # get FV3 restart files, this will be a lot simpler when using history files - rst_dir = os.path.join(task_config.COM_ATMOS_RESTART_PREV) # for now, option later? - run_dir = os.path.join(task_config.DATA, 'bkg') - - # Start accumulating list of background files to copy - bkglist = [] - - # atm DA needs coupler - basename = f'{to_fv3time(task_config.current_cycle)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - # atm DA needs core, srf_wnd, tracer, phy_data, sfc_data - for ftype in ['core', 'srf_wnd', 'tracer']: - template = f'{to_fv3time(self.task_config.current_cycle)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - for ftype in ['phy_data', 'sfc_data']: - template = f'{to_fv3time(self.task_config.current_cycle)}.{ftype}.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - bkg_dict = { - 'mkdir': [run_dir], - 'copy': bkglist, - } - return bkg_dict - - @logit(logger) - def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: - """Compile a dictionary of background error files to copy - - This method will construct a dictionary of either bump of gsibec background - error files for global atm DA and return said dictionary for use by the - FileHandler class. - - Parameters - ---------- - config: Dict - a dictionary containing all of the configuration needed - - Returns - ---------- - berror_dict: Dict - a dictionary containing the list of atm background error files to copy for FileHandler - """ - SUPPORTED_BERROR_STATIC_MAP = {'identity': self._get_berror_dict_identity, - 'bump': self._get_berror_dict_bump, - 'gsibec': self._get_berror_dict_gsibec} - - try: - berror_dict = SUPPORTED_BERROR_STATIC_MAP[config.STATICB_TYPE](config) - except KeyError: - raise KeyError(f"{config.STATICB_TYPE} is not a supported background error type.\n" + - f"Currently supported background error types are:\n" + - f'{" | ".join(SUPPORTED_BERROR_STATIC_MAP.keys())}') - - return berror_dict - - @staticmethod - @logit(logger) - def _get_berror_dict_identity(config: Dict[str, Any]) -> Dict[str, List[str]]: - """Identity BE does not need any files for staging. - - This is a private method and should not be accessed directly. - - Parameters - ---------- - config: Dict - a dictionary containing all of the configuration needed - Returns - ---------- - berror_dict: Dict - Empty dictionary [identity BE needs not files to stage] - """ - logger.info(f"Identity background error does not use staged files. Return empty dictionary") - return {} - - @staticmethod - @logit(logger) - def _get_berror_dict_bump(config: Dict[str, Any]) -> Dict[str, List[str]]: - """Compile a dictionary of atm bump background error files to copy - - This method will construct a dictionary of atm bump background error - files for global atm DA and return said dictionary to the parent - - This is a private method and should not be accessed directly. - - Parameters - ---------- - config: Dict - a dictionary containing all of the configuration needed - - Returns - ---------- - berror_dict: Dict - a dictionary of atm bump background error files to copy for FileHandler - """ - # BUMP atm static-B needs nicas, cor_rh, cor_rv and stddev files. - b_dir = config.BERROR_DATA_DIR - b_datestr = to_fv3time(config.BERROR_DATE) - berror_list = [] - for ftype in ['cor_rh', 'cor_rv', 'stddev']: - coupler = f'{b_datestr}.{ftype}.coupler.res' - berror_list.append([ - os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) - ]) - - template = '{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' - for itile in range(1, config.ntiles + 1): - tracer = template.format(tilenum=itile) - berror_list.append([ - os.path.join(b_dir, tracer), os.path.join(config.DATA, 'berror', tracer) - ]) - - nproc = config.ntiles * config.layout_x * config.layout_y - for nn in range(1, nproc + 1): - berror_list.append([ - os.path.join(b_dir, f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc'), - os.path.join(config.DATA, 'berror', f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc') - ]) - - # create dictionary of background error files to stage - berror_dict = { - 'mkdir': [os.path.join(config.DATA, 'berror')], - 'copy': berror_list, - } - return berror_dict - - @staticmethod - @logit(logger) - def _get_berror_dict_gsibec(config: Dict[str, Any]) -> Dict[str, List[str]]: - """Compile a dictionary of atm gsibec background error files to copy - - This method will construct a dictionary of atm gsibec background error - files for global atm DA and return said dictionary to the parent - - This is a private method and should not be accessed directly. - - Parameters - ---------- - config: Dict - a dictionary containing all of the configuration needed - - Returns - ---------- - berror_dict: Dict - a dictionary of atm gsibec background error files to copy for FileHandler - """ - # GSI atm static-B needs namelist and coefficient files. - b_dir = os.path.join(config.HOMEgfs, 'fix', 'gdas', 'gsibec', config.CASE_ANL) - berror_list = [] - for ftype in ['gfs_gsi_global.nml', 'gsi-coeffs-gfs-global.nc4']: - berror_list.append([ - os.path.join(b_dir, ftype), - os.path.join(config.DATA, 'berror', ftype) - ]) - - # create dictionary of background error files to stage - berror_dict = { - 'mkdir': [os.path.join(config.DATA, 'berror')], - 'copy': berror_list, - } - return berror_dict diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 37ac6137364..9ee77462798 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -77,27 +77,6 @@ def initialize(self: Analysis) -> None: """ super().initialize() - # Make member directories in DATA for background and in DATA and ROTDIR for analysis files - # create template dictionary for output member analysis directories - template_inc = self.task_config.COM_ATMOS_ANALYSIS_TMPL - tmpl_inc_dict = { - 'ROTDIR': self.task_config.ROTDIR, - 'RUN': self.task_config.RUN, - 'YMD': to_YMD(self.task_config.current_cycle), - 'HH': self.task_config.current_cycle.strftime('%H') - } - dirlist = [] - for imem in range(1, self.task_config.NMEM_ENS + 1): - dirlist.append(os.path.join(self.task_config.DATA, 'bkg', f'mem{imem:03d}')) - dirlist.append(os.path.join(self.task_config.DATA, 'anl', f'mem{imem:03d}')) - - # create output directory path for member analysis - tmpl_inc_dict['MEMDIR'] = f"mem{imem:03d}" - incdir = Template.substitute_structure(template_inc, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_inc_dict.get) - dirlist.append(incdir) - - FileHandler({'mkdir': dirlist}).sync() - # stage CRTM fix files logger.info(f"Staging CRTM fix files from {self.task_config.CRTM_FIX_YAML}") crtm_fix_list = parse_j2yaml(self.task_config.CRTM_FIX_YAML, self.task_config) @@ -110,13 +89,8 @@ def initialize(self: Analysis) -> None: # stage backgrounds logger.info(f"Stage ensemble member background files") - localconf = AttrDict() - keys = ['COM_ATMOS_RESTART_TMPL', 'previous_cycle', 'ROTDIR', 'RUN', - 'NMEM_ENS', 'DATA', 'current_cycle', 'ntiles'] - for key in keys: - localconf[key] = self.task_config[key] - localconf.dirname = 'bkg' - FileHandler(self.get_fv3ens_dict(localconf)).sync() + bkg_fix_list = parse_j2yaml(self.task_config.LGETKF_BKG_STAGING_YAML, self.task_config) + FileHandler(bkg_fix_list).sync() # generate ensemble da YAML file logger.debug(f"Generate ensemble da YAML file: {self.task_config.jedi_yaml}") From 9436f4b900be309b8c4a5c754fdd0dbe99d61aee Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 4 Jun 2024 02:34:11 +0000 Subject: [PATCH 10/74] update hashes and config --- parm/config/gfs/config.atmanl | 4 ++-- sorc/gdas.cd | 2 +- sorc/jcb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index dd8ca80b11c..a655066550d 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -15,10 +15,10 @@ export INTERP_METHOD='barycentric' if [[ ${DOHYBVAR} = "YES" ]]; then # shellcheck disable=SC2153 export CASE_ANL=${CASE_ENS} - export BERROR_YAML="background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}" + export BERROR_YAML="atmosphere_background_error_hybrid_${STATICB_TYPE}_${LOCALIZATION_TYPE}" else export CASE_ANL=${CASE} - export BERROR_YAML="background_error_static_${STATICB_TYPE}" + export BERROR_YAML="atmosphere_background_error_static_${STATICB_TYPE}" fi export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 822a50258b8..cf4f3b5d04b 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 822a50258b869a8e855bd867f3194eae6c5b45ef +Subproject commit cf4f3b5d04ba0d852bf98434022cde6b2682fa42 diff --git a/sorc/jcb b/sorc/jcb index 4e55941a2d7..d167c6c2731 160000 --- a/sorc/jcb +++ b/sorc/jcb @@ -1 +1 @@ -Subproject commit 4e55941a2d7c42904d1aa2acd2a2ed1323d747a5 +Subproject commit d167c6c27312ae6fbd9d82c2747fecc6687eb0f9 From 3022393658369533a2f2cae753c9e8f0a9c7c9da Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 4 Jun 2024 09:37:44 -0500 Subject: [PATCH 11/74] Minor fix --- parm/config/gfs/config.atmanl | 2 +- parm/config/gfs/config.atmensanl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index a9aa8201289..b3e9ca5b7f9 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -25,7 +25,7 @@ export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_var_bkg_staging.yaml.j2" export BERROR_STAGING_YAML="${PARMgfs}/gdas/atm_berror_"${STATICB_TYPE}"_staging.yaml.j2" -export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/atm_fv3ens_staging.yaml.j2" +export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/atm_var_fv3ens_staging.yaml.j2" export layout_x_atmanl=@LAYOUT_X_ATMANL@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index c13ce7a0b4c..c8fe76d40a0 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -12,7 +12,7 @@ export INTERP_METHOD='barycentric' export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" -export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_ens_bkg_fix.yaml.j2" +export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_lgetkf_bkg_fix.yaml.j2" export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ From cc1a3b60fdeff7f3b7ff7b74086f5a27827709e0 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 4 Jun 2024 09:39:00 -0500 Subject: [PATCH 12/74] Minor update --- parm/config/gfs/config.atmensanl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index c8fe76d40a0..4a80f969ad1 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -12,7 +12,7 @@ export INTERP_METHOD='barycentric' export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" -export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_lgetkf_bkg_fix.yaml.j2" +export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_lgetkf_bkg_staging.yaml.j2" export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ From 20971169d4701345205a06baa66ba0e93657b071 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 4 Jun 2024 09:54:50 -0500 Subject: [PATCH 13/74] Minor update --- parm/config/gfs/config.atmanl | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index b3e9ca5b7f9..90f4c398f83 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -24,7 +24,7 @@ fi export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_var_bkg_staging.yaml.j2" -export BERROR_STAGING_YAML="${PARMgfs}/gdas/atm_berror_"${STATICB_TYPE}"_staging.yaml.j2" +export BERROR_STAGING_YAML="${PARMgfs}/gdas/atm_berror_${STATICB_TYPE}_staging.yaml.j2" export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/atm_var_fv3ens_staging.yaml.j2" export layout_x_atmanl=@LAYOUT_X_ATMANL@ diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 5f61d924156..826240b8b57 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -98,7 +98,7 @@ def initialize(self: Analysis) -> None: FileHandler(fv3ens_fix_list).sync() # stage backgrounds - logger.info(f"Staging background files from {self.task_config.BKG_FIX_YAML}") + logger.info(f"Staging background files from {self.task_config.VAR_BKG_STAGING_YAML}") bkg_fix_list = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_fix_list).sync() From 53277113348b6e414828f20ed846da08e3cdbc9b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 4 Jun 2024 09:56:20 -0500 Subject: [PATCH 14/74] Coding norms --- ush/python/pygfs/task/atm_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 826240b8b57..911b83478e5 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -98,7 +98,7 @@ def initialize(self: Analysis) -> None: FileHandler(fv3ens_fix_list).sync() # stage backgrounds - logger.info(f"Staging background files from {self.task_config.VAR_BKG_STAGING_YAML}") + logger.info(f"Staging background files from {self.task_config.VAR_BKG_STAGING_YAML}") bkg_fix_list = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) FileHandler(bkg_fix_list).sync() From 2702ef092f41b15de94a623e59f1cf0035607b15 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 4 Jun 2024 15:35:03 -0400 Subject: [PATCH 15/74] Remove jcb from GW submodules --- .gitmodules | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index ea1b5c06afb..b68dfa07614 100644 --- a/.gitmodules +++ b/.gitmodules @@ -30,7 +30,4 @@ path = sorc/upp.fd url = https://github.com/NOAA-EMC/UPP.git ignore = dirty -[submodule "sorc/jcb"] - path = sorc/jcb - url = https://github.com/noaa-emc/jcb - fetchRecurseSubmodules = false + From b6456be6577f1891e0afc25799df8520ff9d9d05 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 4 Jun 2024 15:35:27 -0400 Subject: [PATCH 16/74] Remove JCB --- sorc/jcb | 1 - 1 file changed, 1 deletion(-) delete mode 160000 sorc/jcb diff --git a/sorc/jcb b/sorc/jcb deleted file mode 160000 index de75655d81e..00000000000 --- a/sorc/jcb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit de75655d81ec2ee668d8d47bf4a43625c81dde7c From 2a5f301226d13f4e8207d37ed2e312c9bd17cd05 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 4 Jun 2024 15:41:25 -0400 Subject: [PATCH 17/74] adjust link to point to new jcb location in gdasapp --- sorc/link_workflow.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 68873d0f1a7..9ac628eba45 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -91,13 +91,21 @@ fi # Link python pacakges in ush/python # TODO: This will be unnecessary when these are part of the virtualenv -packages=("wxflow" "jcb") +packages=("wxflow") for package in "${packages[@]}"; do cd "${HOMEgfs}/ush/python" || exit 1 [[ -s "${package}" ]] && rm -f "${package}" ${LINK} "${HOMEgfs}/sorc/${package}/src/${package}" . done +# Link GDASapp python packages in ush/python +packages=("jcb") +for package in "${packages[@]}"; do + cd "${HOMEgfs}/ush/python" || exit 1 + [[ -s "${package}" ]] && rm -f "${package}" + ${LINK} "${HOMEgfs}/sorc/gdas.cd/sorc/${package}/src/${package}" . +done + # Link wxflow in workflow and ci/scripts # TODO: This will be unnecessary when wxflow is part of the virtualenv cd "${HOMEgfs}/workflow" || exit 1 From 9f425a6670c1756655d468b9f4de911eb61f175e Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 4 Jun 2024 15:49:45 -0400 Subject: [PATCH 18/74] Update GDASapp Hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6742ec62a12..3b1a5a6f4e6 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6742ec62a12d7d6f8129057bcf77cee0e2175022 +Subproject commit 3b1a5a6f4e62734beb71bbd5ce1d9f00a6862483 From 4d36caf4a2ea5429d81c650f0e852fc9b03b0ac3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 4 Jun 2024 15:07:17 -0500 Subject: [PATCH 19/74] Move some stuff around --- parm/config/gfs/config.atmanl | 6 +++--- parm/config/gfs/config.atmensanl | 2 +- .../atm_berror_bump.yaml.j2} | 0 .../atm_berror_gsibec.yaml.j2} | 0 .../atm_lgetkf_bkg.yaml.j2} | 0 .../atm_var_bkg.yaml.j2} | 0 .../atm_var_fv3ens.yaml.j2} | 0 ush/python/pygfs/task/atm_analysis.py | 14 +++++++------- ush/python/pygfs/task/atmens_analysis.py | 4 ++-- 9 files changed, 13 insertions(+), 13 deletions(-) rename parm/gdas/{atm_berror_bump_staging.yaml.j2 => staging/atm_berror_bump.yaml.j2} (100%) rename parm/gdas/{atm_berror_gsibec_staging.yaml.j2 => staging/atm_berror_gsibec.yaml.j2} (100%) rename parm/gdas/{atm_lgetkf_bkg_staging.yaml.j2 => staging/atm_lgetkf_bkg.yaml.j2} (100%) rename parm/gdas/{atm_var_bkg_staging.yaml.j2 => staging/atm_var_bkg.yaml.j2} (100%) rename parm/gdas/{atm_var_fv3ens_staging.yaml.j2 => staging/atm_var_fv3ens.yaml.j2} (100%) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 90f4c398f83..13c9ac5e903 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -23,9 +23,9 @@ fi export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" -export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_var_bkg_staging.yaml.j2" -export BERROR_STAGING_YAML="${PARMgfs}/gdas/atm_berror_${STATICB_TYPE}_staging.yaml.j2" -export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/atm_var_fv3ens_staging.yaml.j2" +export VAR_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_bkg.yaml.j2" +export BERROR_STAGING_YAML="${PARMgfs}/gdas/staging/atm_berror_${STATICB_TYPE}.yaml.j2" +export FV3ENS_STAGING_YAML="${PARMgfs}/gdas/staging/atm_var_fv3ens.yaml.j2" export layout_x_atmanl=@LAYOUT_X_ATMANL@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ diff --git a/parm/config/gfs/config.atmensanl b/parm/config/gfs/config.atmensanl index 4a80f969ad1..3f7b016d97c 100644 --- a/parm/config/gfs/config.atmensanl +++ b/parm/config/gfs/config.atmensanl @@ -12,7 +12,7 @@ export INTERP_METHOD='barycentric' export CRTM_FIX_YAML="${PARMgfs}/gdas/atm_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2" -export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/atm_lgetkf_bkg_staging.yaml.j2" +export LGETKF_BKG_STAGING_YAML="${PARMgfs}/gdas/staging/atm_lgetkf_bkg.yaml.j2" export layout_x_atmensanl=@LAYOUT_X_ATMENSANL@ export layout_y_atmensanl=@LAYOUT_Y_ATMENSANL@ diff --git a/parm/gdas/atm_berror_bump_staging.yaml.j2 b/parm/gdas/staging/atm_berror_bump.yaml.j2 similarity index 100% rename from parm/gdas/atm_berror_bump_staging.yaml.j2 rename to parm/gdas/staging/atm_berror_bump.yaml.j2 diff --git a/parm/gdas/atm_berror_gsibec_staging.yaml.j2 b/parm/gdas/staging/atm_berror_gsibec.yaml.j2 similarity index 100% rename from parm/gdas/atm_berror_gsibec_staging.yaml.j2 rename to parm/gdas/staging/atm_berror_gsibec.yaml.j2 diff --git a/parm/gdas/atm_lgetkf_bkg_staging.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 similarity index 100% rename from parm/gdas/atm_lgetkf_bkg_staging.yaml.j2 rename to parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 diff --git a/parm/gdas/atm_var_bkg_staging.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2 similarity index 100% rename from parm/gdas/atm_var_bkg_staging.yaml.j2 rename to parm/gdas/staging/atm_var_bkg.yaml.j2 diff --git a/parm/gdas/atm_var_fv3ens_staging.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 similarity index 100% rename from parm/gdas/atm_var_fv3ens_staging.yaml.j2 rename to parm/gdas/staging/atm_var_fv3ens.yaml.j2 diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 911b83478e5..c57f122808b 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -86,21 +86,21 @@ def initialize(self: Analysis) -> None: # stage static background error files, otherwise it will assume ID matrix logger.info(f"Stage files for STATICB_TYPE {self.task_config.STATICB_TYPE}") if self.task_config.STATICB_TYPE != 'identity': - berror_fix_list = parse_j2yaml(self.task_config.BERROR_STAGING_YAML, self.task_config) + berror_staging_dict = parse_j2yaml(self.task_config.BERROR_STAGING_YAML, self.task_config) else: - berror_fix_list = {} - FileHandler(berror_fix_list).sync() + berror_staging_dict = {} + FileHandler(berror_staging_dict).sync() # stage ensemble files for use in hybrid background error if self.task_config.DOHYBVAR: logger.debug(f"Stage ensemble files for DOHYBVAR {self.task_config.DOHYBVAR}") - fv3ens_fix_list = parse_j2yaml(self.task_config.FV3ENS_STAGING_YAML, self.task_config) - FileHandler(fv3ens_fix_list).sync() + fv3ens_staging_dict = parse_j2yaml(self.task_config.FV3ENS_STAGING_YAML, self.task_config) + FileHandler(fv3ens_staging_dict).sync() # stage backgrounds logger.info(f"Staging background files from {self.task_config.VAR_BKG_STAGING_YAML}") - bkg_fix_list = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) - FileHandler(bkg_fix_list).sync() + bkg_staging_dict = parse_j2yaml(self.task_config.VAR_BKG_STAGING_YAML, self.task_config) + FileHandler(bkg_staging_dict).sync() # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 9ee77462798..e6cafae63b7 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -89,8 +89,8 @@ def initialize(self: Analysis) -> None: # stage backgrounds logger.info(f"Stage ensemble member background files") - bkg_fix_list = parse_j2yaml(self.task_config.LGETKF_BKG_STAGING_YAML, self.task_config) - FileHandler(bkg_fix_list).sync() + bkg_staging_dict = parse_j2yaml(self.task_config.LGETKF_BKG_STAGING_YAML, self.task_config) + FileHandler(bkg_staging_dict).sync() # generate ensemble da YAML file logger.debug(f"Generate ensemble da YAML file: {self.task_config.jedi_yaml}") From 9e357f965a3e9f449999f0b34f900b0261bdfb7b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Tue, 4 Jun 2024 20:40:58 +0000 Subject: [PATCH 20/74] gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index cf4f3b5d04b..bbc4be3dedc 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit cf4f3b5d04ba0d852bf98434022cde6b2682fa42 +Subproject commit bbc4be3dedc158f4f4809c85daf850a5d4f630f6 From ba2d12d9e93ceb0155e729c49993edb3014e9ef9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 5 Jun 2024 09:11:10 -0500 Subject: [PATCH 21/74] Add time loop for some staging --- parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 5 ++++- parm/gdas/staging/atm_var_bkg.yaml.j2 | 5 ++++- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 5 ++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 index f62fdd78414..427401ff249 100644 --- a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 @@ -1,5 +1,5 @@ {% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} -{% set prefix = current_cycle | to_fv3time %} +{% set time_list = [current_cycle] %} mkdir: {% for imem in range(1,NMEM_ENS+1) %} @@ -8,6 +8,8 @@ mkdir: - '{{ DATA }}/anl/{{ memchar }}' {% endfor %} copy: +{% for time in time_list %} +{% set prefix = time | to_fv3time %} {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} {% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) @@ -25,3 +27,4 @@ copy: {% endfor %} {% endfor %} {% endfor %} +{% endfor %} diff --git a/parm/gdas/staging/atm_var_bkg.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2 index 8e88ef3c568..31c83d6ae2c 100644 --- a/parm/gdas/staging/atm_var_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_var_bkg.yaml.j2 @@ -1,11 +1,13 @@ {% set src_dir = COM_ATMOS_RESTART_PREV %} {% set dest_dir = DATA ~ '/bkg' %} {% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} -{% set prefix = current_cycle | to_fv3time %} +{% set time_list = [current_cycle] %} mkdir: - '{{ dest_dir }}' copy: +{% for time in time_list %} +{% set prefix = time | to_fv3time %} {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] {% for ftype in ftype_list %} @@ -14,3 +16,4 @@ copy: - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] {% endfor %} {% endfor %} +{% endfor %} diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 index a6f8c3847a1..e62217e818d 100644 --- a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 +++ b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 @@ -1,5 +1,5 @@ {% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} -{% set prefix = current_cycle | to_fv3time %} +{% set time_list = [current_cycle] %} mkdir: {% for imem in range(1,NMEM_ENS+1) %} @@ -8,6 +8,8 @@ mkdir: - '{{ dest_dir }}' {% endfor %} copy: +{% for time in time_list %} +{% set prefix = time | to_fv3time %} {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} {% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) @@ -25,3 +27,4 @@ copy: {% endfor %} {% endfor %} {% endfor %} +{% endfor %} From d0d4429eed38b5b35e6ff7d4ca5006db76a1a726 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 6 Jun 2024 15:04:48 +0000 Subject: [PATCH 22/74] bump gdas --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 3b1a5a6f4e6..aaaded8fea6 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 3b1a5a6f4e62734beb71bbd5ce1d9f00a6862483 +Subproject commit aaaded8fea6358f6dedac31f9ef62a15d5e08508 From ee1aa30565ccbebaf5790773a21fb670d1e09268 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 6 Jun 2024 15:28:02 +0000 Subject: [PATCH 23/74] bump gdas --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index bbc4be3dedc..6c7c58aeb22 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit bbc4be3dedc158f4f4809c85daf850a5d4f630f6 +Subproject commit 6c7c58aeb22bbe6d6f98ed51f1089ea2ecb6b9b8 From 09ec0213198ecda06a6aea5f71d6ec52b446d218 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 6 Jun 2024 17:27:47 +0000 Subject: [PATCH 24/74] hashes --- sorc/gdas.cd | 2 +- sorc/jcb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6c7c58aeb22..0e60de32286 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6c7c58aeb22bbe6d6f98ed51f1089ea2ecb6b9b8 +Subproject commit 0e60de322862d91895e0945654b23a08d1fd6009 diff --git a/sorc/jcb b/sorc/jcb index d167c6c2731..cd04b800dab 160000 --- a/sorc/jcb +++ b/sorc/jcb @@ -1 +1 @@ -Subproject commit d167c6c27312ae6fbd9d82c2747fecc6687eb0f9 +Subproject commit cd04b800dab9a2b1153aea5da0de1dc2073f21cb From a5db89e1a5ae16de521bd82266d200e66621f4dc Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 6 Jun 2024 20:06:59 +0000 Subject: [PATCH 25/74] update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index aaaded8fea6..0cda73bab89 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit aaaded8fea6358f6dedac31f9ef62a15d5e08508 +Subproject commit 0cda73bab89af06fa9fccd172cad43fb04e5281c From 1bafbe8909b6176fac4a87e1d7344ebfc23cd7ad Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 6 Jun 2024 20:18:08 +0000 Subject: [PATCH 26/74] bring russ change in --- parm/archive/enkf.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index e05aca29814..ed50a1749d1 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -16,7 +16,7 @@ enkf: {% endfor %} {% if DO_JEDIATMENS %} - {% set steps = ["atmensanlinit", "atmensanlrun", "atmensanlfinal"] %} + {% set steps = ["atmensanlinit", "atmensanlletkf", "atmensanlfv3inc", "atmensanlfinal"] %} {% else %} {% set steps = ["eobs", "eupd"] %} {% if lobsdiag_forenkf %} From e52e0de2b269eeca42c30c57459b937100a19f30 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 6 Jun 2024 20:23:14 +0000 Subject: [PATCH 27/74] follow russ for GSI utils hash --- sorc/gsi_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index d9404061611..bb03e172e0d 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit d9404061611553459394173c3ff33116db306326 +Subproject commit bb03e172e0d0d9c56d6da7788ca033bfb5ef5119 From 4d3150ca5e692104dbad0d0aa57f0a9d3f6a1a82 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 7 Jun 2024 02:09:06 +0000 Subject: [PATCH 28/74] revert gsi utils --- sorc/gsi_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index bb03e172e0d..d9404061611 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit bb03e172e0d0d9c56d6da7788ca033bfb5ef5119 +Subproject commit d9404061611553459394173c3ff33116db306326 From f70b0ad92f0022f478802a7f6394f99b175fa14d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 7 Jun 2024 08:36:47 -0500 Subject: [PATCH 29/74] Remove all references to mytask.config and mytask.runtime_config --- ush/python/pygfs/task/aero_analysis.py | 32 +++++++++++----------- ush/python/pygfs/task/aero_emissions.py | 4 ++- ush/python/pygfs/task/analysis.py | 10 +++---- ush/python/pygfs/task/archive.py | 11 ++++---- ush/python/pygfs/task/atm_analysis.py | 32 +++++++++++----------- ush/python/pygfs/task/atmens_analysis.py | 24 ++++++++-------- ush/python/pygfs/task/oceanice_products.py | 21 +++++++------- ush/python/pygfs/task/snow_analysis.py | 22 +++++++-------- ush/python/pygfs/task/upp.py | 17 ++++++------ 9 files changed, 89 insertions(+), 84 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 16d27350905..e67dc6e2705 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -29,33 +29,33 @@ class AerosolAnalysis(Analysis): def __init__(self, config): super().__init__(config) - _res = int(self.config['CASE'][1:]) - _res_anl = int(self.config['CASE_ANL'][1:]) - _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml") + _res = int(self.task_config['CASE'][1:]) + _res_anl = int(self.task_config['CASE_ANL'][1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.aerovar.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, - 'npz_ges': self.config.LEVS - 1, - 'npz': self.config.LEVS - 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, - 'npz_anl': self.config['LEVS'] - 1, + 'npz_anl': self.task_config['LEVS'] - 1, 'AERO_WINDOW_BEGIN': _window_begin, - 'AERO_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", - 'aero_bkg_fhr': map(int, str(self.config['aero_bkg_times']).split(',')), - 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", + 'AERO_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", + 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), + 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, } ) - # task_config is everything that this task should need - self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) @logit(logger) def initialize(self: Analysis) -> None: @@ -157,8 +157,8 @@ def finalize(self: Analysis) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml") + src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") yaml_copy = { 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], 'copy': [[src, dest]] diff --git a/ush/python/pygfs/task/aero_emissions.py b/ush/python/pygfs/task/aero_emissions.py index 17d2f528e43..5f2d4c68403 100644 --- a/ush/python/pygfs/task/aero_emissions.py +++ b/ush/python/pygfs/task/aero_emissions.py @@ -42,7 +42,9 @@ def __init__(self, config: Dict[str, Any]) -> None: localdict = AttrDict( {'variable_used_repeatedly': local_variable} ) - self.task_config = AttrDict(**self.config, **self.runtime_config, **localdict) + + # Extend task_config with localdict + self.task_config = AttrDict(**self.task_config, **localdict) @staticmethod @logit(logger) diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 4b9d6d8bbdc..e407cf17656 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -27,7 +27,7 @@ class Analysis(Task): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) # Store location of GDASApp jinja2 templates - self.gdasapp_j2tmpl_dir = os.path.join(self.config.PARMgfs, 'gdas') + self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') def initialize(self) -> None: super().initialize() @@ -54,7 +54,7 @@ def get_jedi_config(self, algorithm: Optional[str] = None) -> Dict[str, Any]: ---------- algorithm (optional) : str Name of the algorithm to use in the JEDI configuration. Will override the algorithm - set in the self.config.JCB_<>_YAML file + set in the self.task_config.JCB_<>_YAML file Returns ---------- @@ -120,7 +120,7 @@ def get_obs_dict(self) -> Dict[str, Any]: basename = os.path.basename(obfile) copylist.append([os.path.join(self.task_config['COM_OBS'], basename), obfile]) obs_dict = { - 'mkdir': [os.path.join(self.runtime_config['DATA'], 'obs')], + 'mkdir': [os.path.join(self.task_config['DATA'], 'obs')], 'copy': copylist } return obs_dict @@ -161,7 +161,7 @@ def get_bias_dict(self) -> Dict[str, Any]: # TODO: Why is this specific to ATMOS? bias_dict = { - 'mkdir': [os.path.join(self.runtime_config.DATA, 'bc')], + 'mkdir': [os.path.join(self.task_config.DATA, 'bc')], 'copy': copylist } return bias_dict @@ -180,7 +180,7 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li List of increment variables to add to the background """ - for itile in range(1, self.config.ntiles + 1): + for itile in range(1, self.task_config.ntiles + 1): inc_path = inc_file_tmpl.format(tilenum=itile) bkg_path = bkg_file_tmpl.format(tilenum=itile) with Dataset(inc_path, mode='r') as incfile, Dataset(bkg_path, mode='a') as rstfile: diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 66d94878e81..31ff9021882 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -35,12 +35,13 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - rotdir = self.config.ROTDIR + os.sep + rotdir = self.task_config.ROTDIR + os.sep # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} path_dict = self._gen_relative_paths(rotdir) - self.task_config = AttrDict(**self.config, **self.runtime_config, **path_dict) + # Extend task_config with path_dict + self.task_config = AttrDict(**self.task_config, **path_dict) @logit(logger) def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str, Any]]): @@ -297,7 +298,7 @@ def _create_tarball(target: str, fileset: List) -> None: @logit(logger) def _gen_relative_paths(self, root_path: str) -> Dict: - """Generate a dict of paths in self.config relative to root_path + """Generate a dict of paths in self.task_config relative to root_path Parameters ---------- @@ -308,13 +309,13 @@ def _gen_relative_paths(self, root_path: str) -> Dict: ------ rel_path_dict : Dict Dictionary of paths relative to root_path. Members will be named - based on the dict names in self.config. For COM paths, the names will + based on the dict names in self.task_config. For COM paths, the names will follow COM_ --> _dir. For all other directories, the names will follow --> _dir. """ rel_path_dict = {} - for key, value in self.config.items(): + for key, value in self.task_config.items(): if isinstance(value, str): if root_path in value: rel_path = value.replace(root_path, "") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c57f122808b..394ba892772 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -28,35 +28,35 @@ class AtmAnalysis(Analysis): def __init__(self, config): super().__init__(config) - _res = int(self.config.CASE[1:]) - _res_anl = int(self.config.CASE_ANL[1:]) - _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") + _res = int(self.task_config.CASE[1:]) + _res_anl = int(self.task_config.CASE_ANL[1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, - 'npz_ges': self.config.LEVS - 1, - 'npz': self.config.LEVS - 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, - 'npz_anl': self.config.LEVS - 1, + 'npz_anl': self.task_config.LEVS - 1, 'ATM_WINDOW_BEGIN': _window_begin, - 'ATM_WINDOW_LENGTH': f"PT{self.config.assim_freq}H", - 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", + 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, - 'atm_obsdatain_path': f"{self.runtime_config.DATA}/obs/", - 'atm_obsdataout_path': f"{self.runtime_config.DATA}/diags/", + 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", + 'atm_obsdataout_path': f"{self.task_config.DATA}/diags/", 'BKG_TSTEP': "PT1H" # Placeholder for 4D applications } ) - # task_config is everything that this task should need - self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) @logit(logger) def initialize(self: Analysis) -> None: @@ -140,7 +140,7 @@ def variational(self: Analysis) -> None: @logit(logger) def init_fv3_increment(self: Analysis) -> None: # Setup JEDI YAML file - self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, + self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.JCB_ALGO}.yaml") save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index e6cafae63b7..5603184ed80 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -29,22 +29,22 @@ class AtmEnsAnalysis(Analysis): def __init__(self, config): super().__init__(config) - _res = int(self.config.CASE_ENS[1:]) - _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmens.yaml") + _res = int(self.task_config.CASE_ENS[1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) + _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, - 'npz_ges': self.config.LEVS - 1, - 'npz': self.config.LEVS - 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, 'ATM_WINDOW_BEGIN': _window_begin, - 'ATM_WINDOW_LENGTH': f"PT{self.config.assim_freq}H", - 'OPREFIX': f"{self.config.EUPD_CYC}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", + 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", + 'OPREFIX': f"{self.task_config.EUPD_CYC}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, 'atm_obsdatain_path': f"./obs/", 'atm_obsdataout_path': f"./diags/", @@ -52,8 +52,8 @@ def __init__(self, config): } ) - # task_config is everything that this task should need - self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) @logit(logger) def initialize(self: Analysis) -> None: @@ -145,7 +145,7 @@ def letkf(self: Analysis) -> None: @logit(logger) def init_fv3_increment(self: Analysis) -> None: # Setup JEDI YAML file - self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, + self.task_config.jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.JCB_ALGO}.yaml") save_as_yaml(self.get_jedi_config(self.task_config.JCB_ALGO), self.task_config.jedi_yaml) diff --git a/ush/python/pygfs/task/oceanice_products.py b/ush/python/pygfs/task/oceanice_products.py index c865a9f408a..d630fcd9211 100644 --- a/ush/python/pygfs/task/oceanice_products.py +++ b/ush/python/pygfs/task/oceanice_products.py @@ -49,33 +49,34 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - if self.config.COMPONENT not in self.VALID_COMPONENTS: - raise NotImplementedError(f'{self.config.COMPONENT} is not a valid model component.\n' + + if self.task_config.COMPONENT not in self.VALID_COMPONENTS: + raise NotImplementedError(f'{self.task_config.COMPONENT} is not a valid model component.\n' + 'Valid model components are:\n' + f'{", ".join(self.VALID_COMPONENTS)}') - model_grid = f"mx{self.config[self.COMPONENT_RES_MAP[self.config.COMPONENT]]:03d}" + model_grid = f"mx{self.task_config[self.COMPONENT_RES_MAP[self.task_config.COMPONENT]]:03d}" - valid_datetime = add_to_datetime(self.runtime_config.current_cycle, to_timedelta(f"{self.config.FORECAST_HOUR}H")) + valid_datetime = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.FORECAST_HOUR}H")) # TODO: This is a bit of a hack, but it works for now # FIXME: find a better way to provide the averaging period # This will be different for ocean and ice, so when they are made flexible, this will need to be addressed - avg_period = f"{self.config.FORECAST_HOUR-self.config.FHOUT_OCNICE_GFS:03d}-{self.config.FORECAST_HOUR:03d}" + avg_period = f"{self.task_config.FORECAST_HOUR-self.task_config.FHOUT_OCNICE_GFS:03d}-{self.task_config.FORECAST_HOUR:03d}" + # Extend task_config with localdict localdict = AttrDict( - {'component': self.config.COMPONENT, - 'forecast_hour': self.config.FORECAST_HOUR, + {'component': self.task_config.COMPONENT, + 'forecast_hour': self.task_config.FORECAST_HOUR, 'valid_datetime': valid_datetime, 'avg_period': avg_period, 'model_grid': model_grid, 'product_grids': self.VALID_PRODUCT_GRIDS[model_grid]} ) - self.task_config = AttrDict(**self.config, **self.runtime_config, **localdict) + self.task_config = AttrDict(**self.task_config, **localdict) # Read the oceanice_products.yaml file for common configuration - logger.info(f"Read the ocean ice products configuration yaml file {self.config.OCEANICEPRODUCTS_CONFIG}") - self.task_config.oceanice_yaml = parse_j2yaml(self.config.OCEANICEPRODUCTS_CONFIG, self.task_config) + logger.info(f"Read the ocean ice products configuration yaml file {self.task_config.OCEANICEPRODUCTS_CONFIG}") + self.task_config.oceanice_yaml = parse_j2yaml(self.task_config.OCEANICEPRODUCTS_CONFIG, self.task_config) logger.debug(f"oceanice_yaml:\n{pformat(self.task_config.oceanice_yaml)}") @staticmethod diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 9a5c7fcab01..9656b00a8ee 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -32,27 +32,27 @@ class SnowAnalysis(Analysis): def __init__(self, config): super().__init__(config) - _res = int(self.config['CASE'][1:]) - _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _letkfoi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.letkfoi.yaml") + _res = int(self.task_config['CASE'][1:]) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + _letkfoi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.letkfoi.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( { 'npx_ges': _res + 1, 'npy_ges': _res + 1, - 'npz_ges': self.config.LEVS - 1, - 'npz': self.config.LEVS - 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, 'SNOW_WINDOW_BEGIN': _window_begin, - 'SNOW_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", - 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", - 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", + 'SNOW_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'jedi_yaml': _letkfoi_yaml } ) - # task_config is everything that this task should need - self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + # Extend task_config with local_dict + self.task_config = AttrDict(**self.task_config, **local_dict) @logit(logger) def prepare_GTS(self) -> None: @@ -114,7 +114,7 @@ def _gtsbufr2iodax(exe, yaml_file): # 1. generate bufr2ioda YAML files # 2. execute bufr2ioda.x for name in prep_gts_config.bufr2ioda.keys(): - gts_yaml = os.path.join(self.runtime_config.DATA, f"bufr_{name}_snow.yaml") + gts_yaml = os.path.join(self.task_config.DATA, f"bufr_{name}_snow.yaml") logger.info(f"Generate BUFR2IODA YAML file: {gts_yaml}") temp_yaml = parse_j2yaml(prep_gts_config.bufr2ioda[name], localconf) save_as_yaml(temp_yaml, gts_yaml) diff --git a/ush/python/pygfs/task/upp.py b/ush/python/pygfs/task/upp.py index 7db50e1582c..7e42e07c64a 100644 --- a/ush/python/pygfs/task/upp.py +++ b/ush/python/pygfs/task/upp.py @@ -46,26 +46,27 @@ def __init__(self, config: Dict[str, Any]) -> None: """ super().__init__(config) - if self.config.UPP_RUN not in self.VALID_UPP_RUN: - raise NotImplementedError(f'{self.config.UPP_RUN} is not a valid UPP run type.\n' + + if self.task_config.UPP_RUN not in self.VALID_UPP_RUN: + raise NotImplementedError(f'{self.task_config.UPP_RUN} is not a valid UPP run type.\n' + 'Valid UPP_RUN values are:\n' + f'{", ".join(self.VALID_UPP_RUN)}') - valid_datetime = add_to_datetime(self.runtime_config.current_cycle, to_timedelta(f"{self.config.FORECAST_HOUR}H")) + valid_datetime = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.FORECAST_HOUR}H")) + # Extend task_config with localdict localdict = AttrDict( - {'upp_run': self.config.UPP_RUN, - 'forecast_hour': self.config.FORECAST_HOUR, + {'upp_run': self.task_config.UPP_RUN, + 'forecast_hour': self.task_config.FORECAST_HOUR, 'valid_datetime': valid_datetime, 'atmos_filename': f"atm_{valid_datetime.strftime('%Y%m%d%H%M%S')}.nc", 'flux_filename': f"sfc_{valid_datetime.strftime('%Y%m%d%H%M%S')}.nc" } ) - self.task_config = AttrDict(**self.config, **self.runtime_config, **localdict) + self.task_config = AttrDict(**self.task_config, **localdict) # Read the upp.yaml file for common configuration - logger.info(f"Read the UPP configuration yaml file {self.config.UPP_CONFIG}") - self.task_config.upp_yaml = parse_j2yaml(self.config.UPP_CONFIG, self.task_config) + logger.info(f"Read the UPP configuration yaml file {self.task_config.UPP_CONFIG}") + self.task_config.upp_yaml = parse_j2yaml(self.task_config.UPP_CONFIG, self.task_config) logger.debug(f"upp_yaml:\n{pformat(self.task_config.upp_yaml)}") @staticmethod From 49f8593dbb131842c803c38b97681d2a8f3525ed Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 7 Jun 2024 12:53:58 -0500 Subject: [PATCH 30/74] Update wxflow hash --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index 8406beeea41..c62196fb2ae 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 8406beeea410118cdfbd8300895b2b2878eadba6 +Subproject commit c62196fb2ae8885724e052ff0676b70d811670ea From 2bd99497f38314bdaf044f1555e6873728bbfec3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 11 Jun 2024 14:50:44 -0500 Subject: [PATCH 31/74] Tab out Jinja2 templates for readability --- parm/gdas/staging/atm_berror_bump.yaml.j2 | 12 ++++---- parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 34 ++++++++++----------- parm/gdas/staging/atm_var_bkg.yaml.j2 | 14 ++++----- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 36 +++++++++++------------ 4 files changed, 48 insertions(+), 48 deletions(-) diff --git a/parm/gdas/staging/atm_berror_bump.yaml.j2 b/parm/gdas/staging/atm_berror_bump.yaml.j2 index 123d15e3591..c8767f1010e 100644 --- a/parm/gdas/staging/atm_berror_bump.yaml.j2 +++ b/parm/gdas/staging/atm_berror_bump.yaml.j2 @@ -8,16 +8,16 @@ mkdir: copy: {% for ftype in ftype_list %} -{% set prefix = BERROR_DATE | to_fv3time ~ '.' ~ ftype %} -{% set fname = prefix ~ '.coupler.res' %} + {% set prefix = BERROR_DATE | to_fv3time ~ '.' ~ ftype %} + {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% for itile in range(1,7) %} -{% set fname = prefix ~ '.fv_tracer.res.tile' ~ itile ~ '.nc' %} + {% for itile in range(1,7) %} + {% set fname = prefix ~ '.fv_tracer.res.tile' ~ itile ~ '.nc' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% endfor %} + {% endfor %} {% endfor %} {% for nn in range(1,nprocs+1) %} -{% set fname = 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) %} + {% set fname = 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] {% endfor %} diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 index 427401ff249..c654589073f 100644 --- a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 @@ -3,28 +3,28 @@ mkdir: {% for imem in range(1,NMEM_ENS+1) %} -{% set memchar = 'mem%03d' | format(imem) %} + {% set memchar = 'mem%03d' | format(imem) %} - '{{ DATA }}/bkg/{{ memchar }}' - '{{ DATA }}/anl/{{ memchar }}' {% endfor %} copy: {% for time in time_list %} -{% set prefix = time | to_fv3time %} -{% for imem in range(1,NMEM_ENS+1) %} -{% set memchar = 'mem%03d' | format(imem) %} -{% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) - | replace('${RUN}', 'enkfgdas') - | replace('${YMD}', previous_cycle | to_YMD) - | replace('${HH}', previous_cycle | strftime('%H')) - | replace('${MEMDIR}', memchar) %} -{% set dest_dir = DATA ~ '/bkg/' ~ memchar %} -{% set fname = prefix ~ '.coupler.res' %} + {% set prefix = time | to_fv3time %} + {% for imem in range(1,NMEM_ENS+1) %} + {% set memchar = 'mem%03d' | format(imem) %} + {% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) + | replace('${RUN}', 'enkfgdas') + | replace('${YMD}', previous_cycle | to_YMD) + | replace('${HH}', previous_cycle | strftime('%H')) + | replace('${MEMDIR}', memchar) %} + {% set dest_dir = DATA ~ '/bkg/' ~ memchar %} + {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% for ftype in ftype_list %} -{% for itile in range(1,7) %} -{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} + {% for ftype in ftype_list %} + {% for itile in range(1,7) %} + {% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% endfor %} -{% endfor %} -{% endfor %} + {% endfor %} + {% endfor %} + {% endfor %} {% endfor %} diff --git a/parm/gdas/staging/atm_var_bkg.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2 index 31c83d6ae2c..139b86eea6d 100644 --- a/parm/gdas/staging/atm_var_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_var_bkg.yaml.j2 @@ -7,13 +7,13 @@ mkdir: - '{{ dest_dir }}' copy: {% for time in time_list %} -{% set prefix = time | to_fv3time %} -{% set fname = prefix ~ '.coupler.res' %} + {% set prefix = time | to_fv3time %} + {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% for ftype in ftype_list %} -{% for itile in range(1,7) %} -{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} + {% for ftype in ftype_list %} + {% for itile in range(1,7) %} + {% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% endfor %} -{% endfor %} + {% endfor %} + {% endfor %} {% endfor %} diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 index e62217e818d..5a8aa476e7c 100644 --- a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 +++ b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 @@ -3,28 +3,28 @@ mkdir: {% for imem in range(1,NMEM_ENS+1) %} -{% set memchar = 'mem%03d' | format(imem) %} -{% set dest_dir = DATA ~ '/ens/' ~ memchar %} + {% set memchar = 'mem%03d' | format(imem) %} + {% set dest_dir = DATA ~ '/ens/' ~ memchar %} - '{{ dest_dir }}' {% endfor %} copy: {% for time in time_list %} -{% set prefix = time | to_fv3time %} -{% for imem in range(1,NMEM_ENS+1) %} -{% set memchar = 'mem%03d' | format(imem) %} -{% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) - | replace('${RUN}', 'enkfgdas') - | replace('${YMD}', previous_cycle | to_YMD) - | replace('${HH}', previous_cycle | strftime('%H')) - | replace('${MEMDIR}', memchar) %} -{% set dest_dir = DATA ~ '/ens/' ~ memchar %} -{% set fname = prefix ~ '.coupler.res' %} + {% set prefix = time | to_fv3time %} + {% for imem in range(1,NMEM_ENS+1) %} + {% set memchar = 'mem%03d' | format(imem) %} + {% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) + | replace('${RUN}', 'enkfgdas') + | replace('${YMD}', previous_cycle | to_YMD) + | replace('${HH}', previous_cycle | strftime('%H')) + | replace('${MEMDIR}', memchar) %} + {% set dest_dir = DATA ~ '/ens/' ~ memchar %} + {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% for ftype in ftype_list %} -{% for itile in range(1,7) %} -{% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} + {% for ftype in ftype_list %} + {% for itile in range(1,7) %} + {% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] -{% endfor %} -{% endfor %} -{% endfor %} + {% endfor %} + {% endfor %} + {% endfor %} {% endfor %} From 488a3de2addb1e3124cbfbd050fd061d2d33575b Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 13 Jun 2024 15:29:33 +0000 Subject: [PATCH 32/74] update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 0cda73bab89..368c9c5db9b 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 0cda73bab89af06fa9fccd172cad43fb04e5281c +Subproject commit 368c9c5db9b5ea62e72937b6d1b0f753adb9be40 From 2cb6664bfb80a20af4cd5f8f472c5051df186ece Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 13 Jun 2024 15:41:14 +0000 Subject: [PATCH 33/74] Use replace_tmpl filter --- parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 11 ++++++----- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 index c654589073f..595d9541f25 100644 --- a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 @@ -12,11 +12,12 @@ copy: {% set prefix = time | to_fv3time %} {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} - {% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) - | replace('${RUN}', 'enkfgdas') - | replace('${YMD}', previous_cycle | to_YMD) - | replace('${HH}', previous_cycle | strftime('%H')) - | replace('${MEMDIR}', memchar) %} + {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, + '${RUN}': 'enkfgdas', + '${YMD}': previous_cycle | to_YMD, + '${HH}': previous_cycle | strftime('%H'), + '${MEMDIR}': memchar }) %} + {% set src_dir = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} {% set dest_dir = DATA ~ '/bkg/' ~ memchar %} {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 index 5a8aa476e7c..a0ab1497168 100644 --- a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 +++ b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 @@ -12,11 +12,12 @@ copy: {% set prefix = time | to_fv3time %} {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} - {% set src_dir = COM_ATMOS_RESTART_TMPL | replace('${ROTDIR}', ROTDIR) - | replace('${RUN}', 'enkfgdas') - | replace('${YMD}', previous_cycle | to_YMD) - | replace('${HH}', previous_cycle | strftime('%H')) - | replace('${MEMDIR}', memchar) %} + {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, + '${RUN}': 'enkfgdas', + '${YMD}': previous_cycle | to_YMD, + '${HH}': previous_cycle | strftime('%H'), + '${MEMDIR}': memchar }) %} + {% set src_dir = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %} {% set dest_dir = DATA ~ '/ens/' ~ memchar %} {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] From bd7f7709c7d35f106e75445116e35d6d8653043c Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 13 Jun 2024 15:54:38 +0000 Subject: [PATCH 34/74] Update wxflow hash --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index c62196fb2ae..eaafa49e4de 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit c62196fb2ae8885724e052ff0676b70d811670ea +Subproject commit eaafa49e4de9a4505034e84d17208f4b0fdeaa2a From 23971d23e27162cbf79f444be609084cbab78ac2 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 13 Jun 2024 17:48:20 +0000 Subject: [PATCH 35/74] mod --- .gitmodules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitmodules b/.gitmodules index 6cb9f375ebc..5c9e5692434 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,6 +8,9 @@ [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd url = https://github.com/NOAA-EMC/gfs-utils +[submodule "sorc/ufs_utils.fd"] + path = sorc/ufs_utils.fd + url = https://github.com/ufs-community/UFS_UTILS.git [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git From d5cb822a6d34592d6c2e08fd57b473579930ce23 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Thu, 13 Jun 2024 17:51:52 +0000 Subject: [PATCH 36/74] add back ufsutils --- sorc/ufs_utils.fd | 1 + 1 file changed, 1 insertion(+) create mode 160000 sorc/ufs_utils.fd diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd new file mode 160000 index 00000000000..2794d413d08 --- /dev/null +++ b/sorc/ufs_utils.fd @@ -0,0 +1 @@ +Subproject commit 2794d413d083b43d9ba37a15375d5c61b610d29e From f8a18be122484f9674a5b9fe0ba2490d464e6e47 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 14 Jun 2024 17:20:39 +0000 Subject: [PATCH 37/74] Update wxflow hash and fix big in lgetkf staging file --- parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 9 ++++++++- sorc/wxflow | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 index 595d9541f25..14da620db07 100644 --- a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 @@ -4,8 +4,15 @@ mkdir: {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} + {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, + '${RUN}': RUN, + '${YMD}': current_cycle | to_YMD, + '${HH}': current_cycle | strftime('%H'), + '${MEMDIR}': memchar }) %} + {% set anl_rotdir = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} - '{{ DATA }}/bkg/{{ memchar }}' - '{{ DATA }}/anl/{{ memchar }}' +- '{{ anl_rotdir }}' {% endfor %} copy: {% for time in time_list %} @@ -17,7 +24,7 @@ copy: '${YMD}': previous_cycle | to_YMD, '${HH}': previous_cycle | strftime('%H'), '${MEMDIR}': memchar }) %} - {% set src_dir = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} + {% set src_dir = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %} {% set dest_dir = DATA ~ '/bkg/' ~ memchar %} {% set fname = prefix ~ '.coupler.res' %} - ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] diff --git a/sorc/wxflow b/sorc/wxflow index eaafa49e4de..6a790fcf5d5 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit eaafa49e4de9a4505034e84d17208f4b0fdeaa2a +Subproject commit 6a790fcf5d5c20e1b797bcceea04798f192f9fb3 From 2c8feb5d7e771456ddc65d7e71e186adf20827d2 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 17 Jun 2024 14:50:16 +0000 Subject: [PATCH 38/74] Simplify Jinja2 templates a bit --- parm/gdas/staging/atm_berror_bump.yaml.j2 | 16 ++++------------ parm/gdas/staging/atm_berror_gsibec.yaml.j2 | 6 ++---- parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 12 +++--------- parm/gdas/staging/atm_var_bkg.yaml.j2 | 11 +++-------- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 13 +++---------- 5 files changed, 15 insertions(+), 43 deletions(-) diff --git a/parm/gdas/staging/atm_berror_bump.yaml.j2 b/parm/gdas/staging/atm_berror_bump.yaml.j2 index c8767f1010e..1788e1558a7 100644 --- a/parm/gdas/staging/atm_berror_bump.yaml.j2 +++ b/parm/gdas/staging/atm_berror_bump.yaml.j2 @@ -1,23 +1,15 @@ -{% set src_dir = BERROR_DATA_DIR %} -{% set dest_dir = DATA ~ '/berror' %} {% set ftype_list = ['cor_rh', 'cor_rv', 'stddev'] %} {% set nprocs = ntiles * layout_x * layout_y %} mkdir: -- '{{ dest_dir }}' - +- ''{{ DATA }}/berror'' copy: {% for ftype in ftype_list %} - {% set prefix = BERROR_DATE | to_fv3time ~ '.' ~ ftype %} - {% set fname = prefix ~ '.coupler.res' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ BERROR_DATA_DIR }}/{{ BERROR_DATE | to_fv3time }}.coupler.res', '{{ DATA }}/berror'] {% for itile in range(1,7) %} - {% set fname = prefix ~ '.fv_tracer.res.tile' ~ itile ~ '.nc' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ BERROR_DATA_DIR }}/{{ BERROR_DATE | to_fv3time }}.fv_tracer.res.tile{{ itile }}.nc', '{{ DATA }}/berror'] {% endfor %} {% endfor %} - {% for nn in range(1,nprocs+1) %} - {% set fname = 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ BERROR_DATA_DIR }}/{{ 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) }}', '{{ DATA }}/berror'] {% endfor %} diff --git a/parm/gdas/staging/atm_berror_gsibec.yaml.j2 b/parm/gdas/staging/atm_berror_gsibec.yaml.j2 index 0affe68030c..47df4f7df69 100644 --- a/parm/gdas/staging/atm_berror_gsibec.yaml.j2 +++ b/parm/gdas/staging/atm_berror_gsibec.yaml.j2 @@ -1,10 +1,8 @@ -{% set src_dir = HOMEgfs ~ '/fix/gdas/gsibec/' ~ CASE_ANL %} -{% set dest_dir = DATA ~ '/berror' %} {% set fname_list = ['gfs_gsi_global.nml', 'gsi-coeffs-gfs-global.nc4'] %} mkdir: -- '{{ dest_dir }}' +- '{{ DATA }}/berror' copy: {% for fname in fname_list %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ HOMEgfs }}'/fix/gdas/gsibec/{{ CASE_ANL }}/{{ fname }}', '{{ DATA }}/berror'] {% endfor %} diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 index 14da620db07..c592e1607a2 100644 --- a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 @@ -9,14 +9,12 @@ mkdir: '${YMD}': current_cycle | to_YMD, '${HH}': current_cycle | strftime('%H'), '${MEMDIR}': memchar }) %} - {% set anl_rotdir = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} - '{{ DATA }}/bkg/{{ memchar }}' - '{{ DATA }}/anl/{{ memchar }}' -- '{{ anl_rotdir }}' +- '{{ COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}' {% endfor %} copy: {% for time in time_list %} - {% set prefix = time | to_fv3time %} {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, @@ -24,14 +22,10 @@ copy: '${YMD}': previous_cycle | to_YMD, '${HH}': previous_cycle | strftime('%H'), '${MEMDIR}': memchar }) %} - {% set src_dir = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %} - {% set dest_dir = DATA ~ '/bkg/' ~ memchar %} - {% set fname = prefix ~ '.coupler.res' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/{{ memchar }}/'] {% for ftype in ftype_list %} {% for itile in range(1,7) %} - {% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/'] {% endfor %} {% endfor %} {% endfor %} diff --git a/parm/gdas/staging/atm_var_bkg.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2 index 139b86eea6d..3cb15e6c487 100644 --- a/parm/gdas/staging/atm_var_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_var_bkg.yaml.j2 @@ -1,19 +1,14 @@ -{% set src_dir = COM_ATMOS_RESTART_PREV %} -{% set dest_dir = DATA ~ '/bkg' %} {% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} {% set time_list = [current_cycle] %} mkdir: -- '{{ dest_dir }}' +- '{{ DATA }}/bkg' copy: {% for time in time_list %} - {% set prefix = time | to_fv3time %} - {% set fname = prefix ~ '.coupler.res' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/'] {% for ftype in ftype_list %} {% for itile in range(1,7) %} - {% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/'] {% endfor %} {% endfor %} {% endfor %} diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 index a0ab1497168..f2ceff7e910 100644 --- a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 +++ b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 @@ -3,13 +3,10 @@ mkdir: {% for imem in range(1,NMEM_ENS+1) %} - {% set memchar = 'mem%03d' | format(imem) %} - {% set dest_dir = DATA ~ '/ens/' ~ memchar %} -- '{{ dest_dir }}' +- '{{ DATA }}/ens/{{ 'mem%03d' | format(imem) }}' {% endfor %} copy: {% for time in time_list %} - {% set prefix = time | to_fv3time %} {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, @@ -17,14 +14,10 @@ copy: '${YMD}': previous_cycle | to_YMD, '${HH}': previous_cycle | strftime('%H'), '${MEMDIR}': memchar }) %} - {% set src_dir = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %} - {% set dest_dir = DATA ~ '/ens/' ~ memchar %} - {% set fname = prefix ~ '.coupler.res' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/ens/{{ memchar }}/'] {% for ftype in ftype_list %} {% for itile in range(1,7) %} - {% set fname = prefix ~ '.' ~ ftype ~ '.tile' ~ itile ~ '.nc' %} -- ['{{ src_dir }}/{{ fname }}', '{{ dest_dir }}/{{ fname }}'] +- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/ens/{{ memchar }}/'] {% endfor %} {% endfor %} {% endfor %} From 2c5ef592d13c980b740f22269344c969517843bf Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 17 Jun 2024 14:57:18 +0000 Subject: [PATCH 39/74] Replace CDUMP with RUN --- ush/python/pygfs/task/aero_analysis.py | 10 +++++----- ush/python/pygfs/task/aero_prepobs.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 12 ++++++------ ush/python/pygfs/task/atmens_analysis.py | 12 ++++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index e67dc6e2705..69a992d7d47 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -32,7 +32,7 @@ def __init__(self, config): _res = int(self.task_config['CASE'][1:]) _res_anl = int(self.task_config['CASE_ANL'][1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.aerovar.yaml") + _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.aerovar.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -47,8 +47,8 @@ def __init__(self, config): 'AERO_WINDOW_BEGIN': _window_begin, 'AERO_WINDOW_LENGTH': f"PT{self.task_config['assim_freq']}H", 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), - 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, } @@ -157,8 +157,8 @@ def finalize(self: Analysis) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") + src = os.path.join(self.task_config['DATA'], f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") yaml_copy = { 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], 'copy': [[src, dest]] diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index f2344241a92..bf081ba6623 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -64,7 +64,7 @@ def initialize(self) -> None: self.task_config.prepaero_config = self.get_obsproc_config(sensor) # generate converter YAML file - template = f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml" + template = f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml" _prepaero_yaml = os.path.join(self.runtime_config.DATA, template) self.task_config.prepaero_yaml.append(_prepaero_yaml) logger.debug(f"Generate PrepAeroObs YAML file: {_prepaero_yaml}") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 394ba892772..4e9d37335c5 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -31,7 +31,7 @@ def __init__(self, config): _res = int(self.task_config.CASE[1:]) _res_anl = int(self.task_config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") + _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -45,8 +45,8 @@ def __init__(self, config): 'npz_anl': self.task_config.LEVS - 1, 'ATM_WINDOW_BEGIN': _window_begin, 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", - 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, 'atm_obsdatain_path': f"{self.task_config.DATA}/obs/", @@ -198,8 +198,8 @@ def finalize(self: Analysis) -> None: # copy full YAML from executable to ROTDIR logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") - src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") + src = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmvar.yaml") logger.debug(f"Copying {src} to {dest}") yaml_copy = { 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS], @@ -244,7 +244,7 @@ def finalize(self: Analysis) -> None: cdate = to_fv3time(self.task_config.current_cycle) cdate_inc = cdate.replace('.', '_') src = os.path.join(self.task_config.DATA, 'anl', f"atminc.{cdate_inc}z.nc4") - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f'{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc') + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f'{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atminc.nc') logger.debug(f"Copying {src} to {dest}") inc_copy = { 'copy': [[src, dest]] diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 5603184ed80..bd5112050e8 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -31,7 +31,7 @@ def __init__(self, config): _res = int(self.task_config.CASE_ENS[1:]) _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") + _jedi_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -42,8 +42,8 @@ def __init__(self, config): 'npz': self.task_config.LEVS - 1, 'ATM_WINDOW_BEGIN': _window_begin, 'ATM_WINDOW_LENGTH': f"PT{self.task_config.assim_freq}H", - 'OPREFIX': f"{self.task_config.EUPD_CYC}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'OPREFIX': f"{self.task_config.EUPD_CYC}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, 'atm_obsdatain_path': f"./obs/", @@ -209,8 +209,8 @@ def finalize(self: Analysis) -> None: # copy full YAML from executable to ROTDIR logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS_ENS}") - src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") - dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmens.yaml") + src = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS_ENS, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atmens.yaml") logger.debug(f"Copying {src} to {dest}") yaml_copy = { 'mkdir': [self.task_config.COM_ATMOS_ANALYSIS_ENS], @@ -239,7 +239,7 @@ def finalize(self: Analysis) -> None: tmpl_inc_dict['MEMDIR'] = memchar incdir = Template.substitute_structure(template_inc, TemplateConstants.DOLLAR_CURLY_BRACE, tmpl_inc_dict.get) src = os.path.join(self.task_config.DATA, 'anl', memchar, f"atminc.{cdate_inc}z.nc4") - dest = os.path.join(incdir, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc") + dest = os.path.join(incdir, f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.atminc.nc") # copy increment logger.debug(f"Copying {src} to {dest}") From dcfc40f58499aa90f8100258a32195a9cdf844e5 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 17 Jun 2024 15:13:18 +0000 Subject: [PATCH 40/74] Fix bug --- parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 index c592e1607a2..eda3dad5a7c 100644 --- a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 @@ -25,7 +25,7 @@ copy: - ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/{{ memchar }}/'] {% for ftype in ftype_list %} {% for itile in range(1,7) %} -- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/'] +- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/'] {% endfor %} {% endfor %} {% endfor %} From 584762a6a459f7e520fcf7f1f56b197c467c5871 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Tue, 18 Jun 2024 19:24:22 +0000 Subject: [PATCH 41/74] update gdas.cd hash to bring in updated jcb, jcb-gdas, and jcb-algorithms --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c0876c7962b..b093825a7a3 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c0876c7962b05827e8bd209f651887bfeabef77c +Subproject commit b093825a7a363488dd377024eb822ce623b4732f From dd6613b236afe090763ed08ce7b4ce49bf887723 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Jun 2024 14:11:09 +0000 Subject: [PATCH 42/74] remove path specific logic to identify MSU machines (#2695) --- ush/detect_machine.sh | 3 +-- workflow/hosts.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 683ee0db7fc..106d9033387 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -75,8 +75,7 @@ elif [[ -d /scratch1 ]]; then MACHINE_ID=hera elif [[ -d /work ]]; then # We are on MSU Orion or Hercules - if [[ -d /apps/other ]]; then - # We are on Hercules + if [[ $(findmnt -n -o SOURCE /home) =~ "hercules" ]]; then MACHINE_ID=hercules else MACHINE_ID=orion diff --git a/workflow/hosts.py b/workflow/hosts.py index 2334a3ac356..4b86bf4705b 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +import socket from pathlib import Path from wxflow import YAMLFile @@ -39,10 +40,7 @@ def detect(cls): if os.path.exists('/scratch1/NCEPDEV'): machine = 'HERA' elif os.path.exists('/work/noaa'): - if os.path.exists('/apps/other'): - machine = 'HERCULES' - else: - machine = 'ORION' + machine = socket.gethostname().split("-",1)[0].upper() elif os.path.exists('/lfs4/HFIP'): machine = 'JET' elif os.path.exists('/lfs/f1'): From 826c3c3143bdb3ade5422e4d536c6d03b7881d5f Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Jun 2024 14:13:08 +0000 Subject: [PATCH 43/74] add ocnanalecen to HERCULES.env (#2699) --- env/HERCULES.env | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/env/HERCULES.env b/env/HERCULES.env index d43dedad8d8..92d012f0b7c 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -135,6 +135,16 @@ case ${step} in [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max} export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}" ;; +"ocnanalecen") + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_ocnanalecen)) + + export NTHREADS_OCNANALECEN=${nth_ocnanalecen:-${nth_max}} + [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max} + export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}" +;; "ocnanalchkpt") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" From c14607f3645a07b3ef087b488afc1e126ec56292 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Jun 2024 14:22:15 +0000 Subject: [PATCH 44/74] fix shellcheck error in detect_machine.sh (#2695) --- ush/detect_machine.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 106d9033387..cfd0fa97e27 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -75,7 +75,8 @@ elif [[ -d /scratch1 ]]; then MACHINE_ID=hera elif [[ -d /work ]]; then # We are on MSU Orion or Hercules - if [[ $(findmnt -n -o SOURCE /home) =~ "hercules" ]]; then + mount=$(findmnt -n -o SOURCE /home) + if [[ ${mount} =~ "hercules" ]]; then MACHINE_ID=hercules else MACHINE_ID=orion From 9973c70219d94ae0b0f5401e85bada9d1d0da3f8 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Thu, 20 Jun 2024 14:25:41 +0000 Subject: [PATCH 45/74] fix pycodestyle error in hosts.py (#2695) --- workflow/hosts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts.py b/workflow/hosts.py index 4b86bf4705b..cd0cfe00830 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -40,7 +40,7 @@ def detect(cls): if os.path.exists('/scratch1/NCEPDEV'): machine = 'HERA' elif os.path.exists('/work/noaa'): - machine = socket.gethostname().split("-",1)[0].upper() + machine = socket.gethostname().split("-", 1)[0].upper() elif os.path.exists('/lfs4/HFIP'): machine = 'JET' elif os.path.exists('/lfs/f1'): From ddcd37648657182fb1877efe1b50a2a6400883d9 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 20 Jun 2024 14:41:46 +0000 Subject: [PATCH 46/74] Updates --- parm/gdas/staging/atm_berror_bump.yaml.j2 | 15 --------------- parm/gdas/staging/atm_berror_gsibec.yaml.j2 | 2 +- parm/gdas/staging/atm_var_bkg.yaml.j2 | 2 +- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 8 ++++---- 4 files changed, 6 insertions(+), 21 deletions(-) delete mode 100644 parm/gdas/staging/atm_berror_bump.yaml.j2 diff --git a/parm/gdas/staging/atm_berror_bump.yaml.j2 b/parm/gdas/staging/atm_berror_bump.yaml.j2 deleted file mode 100644 index 1788e1558a7..00000000000 --- a/parm/gdas/staging/atm_berror_bump.yaml.j2 +++ /dev/null @@ -1,15 +0,0 @@ -{% set ftype_list = ['cor_rh', 'cor_rv', 'stddev'] %} -{% set nprocs = ntiles * layout_x * layout_y %} - -mkdir: -- ''{{ DATA }}/berror'' -copy: -{% for ftype in ftype_list %} -- ['{{ BERROR_DATA_DIR }}/{{ BERROR_DATE | to_fv3time }}.coupler.res', '{{ DATA }}/berror'] - {% for itile in range(1,7) %} -- ['{{ BERROR_DATA_DIR }}/{{ BERROR_DATE | to_fv3time }}.fv_tracer.res.tile{{ itile }}.nc', '{{ DATA }}/berror'] - {% endfor %} -{% endfor %} -{% for nn in range(1,nprocs+1) %} -- ['{{ BERROR_DATA_DIR }}/{{ 'nicas_aero_nicas_local_%06d-%06d.nc' | format(nprocs, nn) }}', '{{ DATA }}/berror'] -{% endfor %} diff --git a/parm/gdas/staging/atm_berror_gsibec.yaml.j2 b/parm/gdas/staging/atm_berror_gsibec.yaml.j2 index 47df4f7df69..e6c5e416094 100644 --- a/parm/gdas/staging/atm_berror_gsibec.yaml.j2 +++ b/parm/gdas/staging/atm_berror_gsibec.yaml.j2 @@ -4,5 +4,5 @@ mkdir: - '{{ DATA }}/berror' copy: {% for fname in fname_list %} -- ['{{ HOMEgfs }}'/fix/gdas/gsibec/{{ CASE_ANL }}/{{ fname }}', '{{ DATA }}/berror'] +- ['{{ HOMEgfs }}/fix/gdas/gsibec/{{ CASE_ANL }}/{{ fname }}', '{{ DATA }}/berror'] {% endfor %} diff --git a/parm/gdas/staging/atm_var_bkg.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2 index 3cb15e6c487..37af8336494 100644 --- a/parm/gdas/staging/atm_var_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_var_bkg.yaml.j2 @@ -7,7 +7,7 @@ copy: {% for time in time_list %} - ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/'] {% for ftype in ftype_list %} - {% for itile in range(1,7) %} + {% for itile in range(1,ntiles+1) %} - ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/'] {% endfor %} {% endfor %} diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 index f2ceff7e910..e499c86d574 100644 --- a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 +++ b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 @@ -11,12 +11,12 @@ copy: {% set memchar = 'mem%03d' | format(imem) %} {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, '${RUN}': 'enkfgdas', - '${YMD}': previous_cycle | to_YMD, - '${HH}': previous_cycle | strftime('%H'), - '${MEMDIR}': memchar }) %} + '${YMD}': previous_cycle | to_YMD, + '${HH}': previous_cycle | strftime('%H'), + '${MEMDIR}': memchar }) %} - ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/ens/{{ memchar }}/'] {% for ftype in ftype_list %} - {% for itile in range(1,7) %} + {% for itile in range(1,ntiles+1) %} - ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/ens/{{ memchar }}/'] {% endfor %} {% endfor %} From 7aa041e087c84a2c83f5b597f8e64ff68539b960 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 20 Jun 2024 14:54:43 +0000 Subject: [PATCH 47/74] Refactor aero_prepobs.py like other tasks --- ush/python/pygfs/task/aero_prepobs.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ush/python/pygfs/task/aero_prepobs.py b/ush/python/pygfs/task/aero_prepobs.py index bf081ba6623..d8396fe3cad 100644 --- a/ush/python/pygfs/task/aero_prepobs.py +++ b/ush/python/pygfs/task/aero_prepobs.py @@ -24,23 +24,23 @@ class AerosolObsPrep(Task): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config['assim_freq']}H") / 2) - _window_end = add_to_datetime(self.runtime_config.current_cycle, +to_timedelta(f"{self.config['assim_freq']}H") / 2) + _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config['assim_freq']}H") / 2) + _window_end = add_to_datetime(self.task_config.current_cycle, +to_timedelta(f"{self.task_config['assim_freq']}H") / 2) local_dict = AttrDict( { 'window_begin': _window_begin, 'window_end': _window_end, - 'sensors': str(self.config['SENSORS']).split(','), - 'data_dir': self.config['VIIRS_DATA_DIR'], + 'sensors': str(self.task_config['SENSORS']).split(','), + 'data_dir': self.task_config['VIIRS_DATA_DIR'], 'input_files': '', - 'OPREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z.", - 'APREFIX': f"{self.runtime_config.RUN}.t{self.runtime_config.cyc:02d}z." + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z." } ) # task_config is everything that this task should need - self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + self.task_config = AttrDict(**self.task_config, **local_dict) @logit(logger) def initialize(self) -> None: @@ -64,8 +64,8 @@ def initialize(self) -> None: self.task_config.prepaero_config = self.get_obsproc_config(sensor) # generate converter YAML file - template = f"{self.runtime_config.RUN}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml" - _prepaero_yaml = os.path.join(self.runtime_config.DATA, template) + template = f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml" + _prepaero_yaml = os.path.join(self.task_config.DATA, template) self.task_config.prepaero_yaml.append(_prepaero_yaml) logger.debug(f"Generate PrepAeroObs YAML file: {_prepaero_yaml}") save_as_yaml(self.task_config.prepaero_config, _prepaero_yaml) From e175b2526aedc61e8040e52accd2d1ea8b9d198b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 20 Jun 2024 19:27:32 +0000 Subject: [PATCH 48/74] Update wxflow hash --- sorc/wxflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/wxflow b/sorc/wxflow index 6a790fcf5d5..5dad7dd61ce 160000 --- a/sorc/wxflow +++ b/sorc/wxflow @@ -1 +1 @@ -Subproject commit 6a790fcf5d5c20e1b797bcceea04798f192f9fb3 +Subproject commit 5dad7dd61cebd9b3f2b163b3b06bb75eae1860a9 From 7cc86d950487a24446081a5922c46a234461f085 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 21 Jun 2024 15:41:53 +0000 Subject: [PATCH 49/74] update gdas.cd to resolve wxflow issues and update jcb hashes (#2699) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b093825a7a3..4c58b1edc1a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b093825a7a363488dd377024eb822ce623b4732f +Subproject commit 4c58b1edc1a06ab7296929f26daef5958da72735 From 8663123499ede9ae09ce4179fe336213bf3437b6 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sat, 22 Jun 2024 19:23:44 +0000 Subject: [PATCH 50/74] add wxflow to PYTHONPATH for ufsda jobs; update prep_snow_obs script to use task_config (#2654) --- scripts/exglobal_prep_snow_obs.py | 2 +- ush/load_ufsda_modules.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/exglobal_prep_snow_obs.py b/scripts/exglobal_prep_snow_obs.py index 5107d9c935f..d4998a7d847 100755 --- a/scripts/exglobal_prep_snow_obs.py +++ b/scripts/exglobal_prep_snow_obs.py @@ -21,5 +21,5 @@ # Instantiate the snow prepare task SnowAnl = SnowAnalysis(config) SnowAnl.prepare_GTS() - if f"{ SnowAnl.runtime_config.cyc }" == '18': + if f"{ SnowAnl.task_config.cyc }" == '18': SnowAnl.prepare_IMS() diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index d7aa08e1aee..5a996731738 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -51,6 +51,10 @@ esac module list pip list +# Add wxflow to PYTHONPATH +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}" + # Restore stack soft limit: ulimit -S -s "${ulimit_s}" unset ulimit_s From 076a59d0106275baa577ce6578a9fdc2e7c87743 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sun, 23 Jun 2024 18:39:24 +0000 Subject: [PATCH 51/74] simply wxflow path for ufsda jobs (#2654) --- ush/load_ufsda_modules.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index 5a996731738..8117d3f3592 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -52,8 +52,9 @@ module list pip list # Add wxflow to PYTHONPATH -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +wxflowPATH="${HOMEgfs}/ush/python" PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}" +export PYTHONPATH # Restore stack soft limit: ulimit -S -s "${ulimit_s}" From 19f35e9ae7344bfa009bd7e0610af40566f7edf3 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Sun, 23 Jun 2024 18:44:27 +0000 Subject: [PATCH 52/74] update gdas.cd hash to fix issues with aero and soca DA (#2700) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 4c58b1edc1a..7ef7d08fc56 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 4c58b1edc1a06ab7296929f26daef5958da72735 +Subproject commit 7ef7d08fc56f65697cbd203d9fe9163ef4675555 From 3768b9d21e026ad4a8dc778a5d3a29f03a2de861 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 24 Jun 2024 00:38:51 +0000 Subject: [PATCH 53/74] do not set wxflowPATH in rocoto j-job drivers, add wxflowPATH to load_fv3gfs_modules (#2700) --- jobs/rocoto/aeroanlfinal.sh | 5 ----- jobs/rocoto/aeroanlinit.sh | 6 ------ jobs/rocoto/aeroanlrun.sh | 6 ------ jobs/rocoto/atmanlfinal.sh | 5 ----- jobs/rocoto/atmanlfv3inc.sh | 6 ------ jobs/rocoto/atmanlinit.sh | 6 ------ jobs/rocoto/atmanlvar.sh | 6 ------ jobs/rocoto/atmensanlfinal.sh | 5 ----- jobs/rocoto/atmensanlfv3inc.sh | 6 ------ jobs/rocoto/atmensanlinit.sh | 6 ------ jobs/rocoto/atmensanlletkf.sh | 6 ------ jobs/rocoto/oceanice_products.sh | 6 ------ jobs/rocoto/prepatmiodaobs.sh | 5 ++--- jobs/rocoto/prepobsaero.sh | 6 ------ jobs/rocoto/prepsnowobs.sh | 5 ++--- jobs/rocoto/snowanl.sh | 6 ------ jobs/rocoto/upp.sh | 6 ------ ush/load_fv3gfs_modules.sh | 5 +++++ 18 files changed, 9 insertions(+), 93 deletions(-) diff --git a/jobs/rocoto/aeroanlfinal.sh b/jobs/rocoto/aeroanlfinal.sh index 16bb6887fd7..39dea71810f 100755 --- a/jobs/rocoto/aeroanlfinal.sh +++ b/jobs/rocoto/aeroanlfinal.sh @@ -11,11 +11,6 @@ status=$? export job="aeroanlfinal" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE" diff --git a/jobs/rocoto/aeroanlinit.sh b/jobs/rocoto/aeroanlinit.sh index 9aaf255782c..7a1cf885c10 100755 --- a/jobs/rocoto/aeroanlinit.sh +++ b/jobs/rocoto/aeroanlinit.sh @@ -11,12 +11,6 @@ status=$? export job="aeroanlinit" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE" diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlrun.sh index bcd86e3fbf6..529bb2d7d1f 100755 --- a/jobs/rocoto/aeroanlrun.sh +++ b/jobs/rocoto/aeroanlrun.sh @@ -11,12 +11,6 @@ status=$? export job="aeroanlrun" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN" diff --git a/jobs/rocoto/atmanlfinal.sh b/jobs/rocoto/atmanlfinal.sh index 3d3c3ba9e65..a12894ed1e7 100755 --- a/jobs/rocoto/atmanlfinal.sh +++ b/jobs/rocoto/atmanlfinal.sh @@ -11,11 +11,6 @@ status=$? export job="atmanlfinal" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE" diff --git a/jobs/rocoto/atmanlfv3inc.sh b/jobs/rocoto/atmanlfv3inc.sh index effc18cee50..5261c15f090 100755 --- a/jobs/rocoto/atmanlfv3inc.sh +++ b/jobs/rocoto/atmanlfv3inc.sh @@ -11,12 +11,6 @@ status=$? export job="atmanlfv3inc" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT" diff --git a/jobs/rocoto/atmanlinit.sh b/jobs/rocoto/atmanlinit.sh index 13c7d8710b9..53292005908 100755 --- a/jobs/rocoto/atmanlinit.sh +++ b/jobs/rocoto/atmanlinit.sh @@ -11,12 +11,6 @@ status=$? export job="atmanlinit" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE" diff --git a/jobs/rocoto/atmanlvar.sh b/jobs/rocoto/atmanlvar.sh index 812e3c706a6..7df7f59dd10 100755 --- a/jobs/rocoto/atmanlvar.sh +++ b/jobs/rocoto/atmanlvar.sh @@ -11,12 +11,6 @@ status=$? export job="atmanlvar" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL" diff --git a/jobs/rocoto/atmensanlfinal.sh b/jobs/rocoto/atmensanlfinal.sh index 5ffaa92754c..fc29bdd9af0 100755 --- a/jobs/rocoto/atmensanlfinal.sh +++ b/jobs/rocoto/atmensanlfinal.sh @@ -11,11 +11,6 @@ status=$? export job="atmensanlfinal" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE" diff --git a/jobs/rocoto/atmensanlfv3inc.sh b/jobs/rocoto/atmensanlfv3inc.sh index bb44ddc3a0d..7f57e8d6188 100755 --- a/jobs/rocoto/atmensanlfv3inc.sh +++ b/jobs/rocoto/atmensanlfv3inc.sh @@ -11,12 +11,6 @@ status=$? export job="atmensanlfv3inc" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT" diff --git a/jobs/rocoto/atmensanlinit.sh b/jobs/rocoto/atmensanlinit.sh index 2c2204548ac..1cd8129df63 100755 --- a/jobs/rocoto/atmensanlinit.sh +++ b/jobs/rocoto/atmensanlinit.sh @@ -11,12 +11,6 @@ status=$? export job="atmensanlinit" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE" diff --git a/jobs/rocoto/atmensanlletkf.sh b/jobs/rocoto/atmensanlletkf.sh index b4a1a73a804..0ca86bfb43c 100755 --- a/jobs/rocoto/atmensanlletkf.sh +++ b/jobs/rocoto/atmensanlletkf.sh @@ -11,12 +11,6 @@ status=$? export job="atmensanlletkf" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF" diff --git a/jobs/rocoto/oceanice_products.sh b/jobs/rocoto/oceanice_products.sh index eb704fb35f3..2a3b617d054 100755 --- a/jobs/rocoto/oceanice_products.sh +++ b/jobs/rocoto/oceanice_products.sh @@ -12,12 +12,6 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? if (( status != 0 )); then exit "${status}"; fi -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - export job="oceanice_products" export jobid="${job}.$$" diff --git a/jobs/rocoto/prepatmiodaobs.sh b/jobs/rocoto/prepatmiodaobs.sh index 0e69eda5c9d..26629a514f5 100755 --- a/jobs/rocoto/prepatmiodaobs.sh +++ b/jobs/rocoto/prepatmiodaobs.sh @@ -12,11 +12,10 @@ export job="prepatmobs" export jobid="${job}.$$" ############################################################### -# setup python path for workflow and ioda utilities -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +# setup python path for ioda utilities # shellcheck disable=SC2311 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/" -PYTHONPATH="${pyiodaPATH}:${wxflowPATH}:${PYTHONPATH}" +PYTHONPATH="${pyiodaPATH}:${PYTHONPATH}" export PYTHONPATH ############################################################### diff --git a/jobs/rocoto/prepobsaero.sh b/jobs/rocoto/prepobsaero.sh index 89da7547e8d..5d65ff8a02d 100755 --- a/jobs/rocoto/prepobsaero.sh +++ b/jobs/rocoto/prepobsaero.sh @@ -11,12 +11,6 @@ status=$? export job="prepobsaero" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_PREP_OBS_AERO" diff --git a/jobs/rocoto/prepsnowobs.sh b/jobs/rocoto/prepsnowobs.sh index cff082bab2c..3f23bc16a55 100755 --- a/jobs/rocoto/prepsnowobs.sh +++ b/jobs/rocoto/prepsnowobs.sh @@ -12,12 +12,11 @@ export job="prepsnowobs" export jobid="${job}.$$" ############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +# setup python path for ioda utilities # shellcheck disable=SC2311 pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/" gdasappPATH="${HOMEgfs}/sorc/gdas.cd/sorc/iodaconv/src:${pyiodaPATH}" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}:${gdasappPATH}" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}:${gdasappPATH}" export PYTHONPATH ############################################################### diff --git a/jobs/rocoto/snowanl.sh b/jobs/rocoto/snowanl.sh index 627dd860f49..97df7a46c71 100755 --- a/jobs/rocoto/snowanl.sh +++ b/jobs/rocoto/snowanl.sh @@ -11,12 +11,6 @@ status=$? export job="snowanl" export jobid="${job}.$$" -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - ############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGLOBAL_SNOW_ANALYSIS" diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh index da0180472d4..9f2f092a006 100755 --- a/jobs/rocoto/upp.sh +++ b/jobs/rocoto/upp.sh @@ -35,12 +35,6 @@ else if (( status != 0 )); then exit "${status}"; fi fi -############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH - export job="upp" export jobid="${job}.$$" diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index ae0e381db43..5f6afb7e35f 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -30,6 +30,11 @@ esac module list +# Add wxflow to PYTHONPATH +wxflowPATH="${HOMEgfs}/ush/python" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}" +export PYTHONPATH + # Restore stack soft limit: ulimit -S -s "${ulimit_s}" unset ulimit_s From 32a2002d7a281a2f51c233de4c7a99bc867c4c16 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 24 Jun 2024 12:59:07 +0000 Subject: [PATCH 54/74] update gdas.cd hash to simplify wxflowpath in GDASApp ctests (#2700) --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 7ef7d08fc56..e3644a98c36 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 7ef7d08fc56f65697cbd203d9fe9163ef4675555 +Subproject commit e3644a98c362d7321f9e3081a4e55947885ed2bf From 8fc02e29e75aad7d8ce274176a35cc2d03e31900 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Mon, 24 Jun 2024 18:05:11 +0000 Subject: [PATCH 55/74] add g-w python to PYTHONPATH on WCOSS2 in upp.sh (#2700) --- jobs/rocoto/upp.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh index 9f2f092a006..c3f128ab022 100755 --- a/jobs/rocoto/upp.sh +++ b/jobs/rocoto/upp.sh @@ -29,6 +29,12 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then module load python/3.8.6 module load crtm/2.4.0 # TODO: This is only needed when UPP_RUN=goes. Is there a better way to handle this? set_trace + + # Add wxflow to PYTHONPATH + wxflowPATH="${HOMEgfs}/ush/python" + PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}" + export PYTHONPATH + else . "${HOMEgfs}/ush/load_fv3gfs_modules.sh" status=$? From c1ef4b30c19a896e40b84634f5fb261b18459d63 Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA Date: Fri, 28 Jun 2024 13:40:58 +0000 Subject: [PATCH 56/74] comment out find lines for purge_every_days (#2719) --- scripts/exglobal_cleanup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh index 1150ca6d1d5..876fa3538b5 100755 --- a/scripts/exglobal_cleanup.sh +++ b/scripts/exglobal_cleanup.sh @@ -17,10 +17,10 @@ rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}" purge_every_days=3 # Find and delete files older than ${purge_every_days} days -find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \; +#find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \; # Find and delete directories older than ${purge_every_days} days -find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \; +#find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \; echo "Cleanup ${DATAROOT} completed!" ############################################################### From f551018c5593c0e27064ff6854975a7ae5b12646 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 28 Jun 2024 14:57:27 -0400 Subject: [PATCH 57/74] Update scripts/exglobal_cleanup.sh --- scripts/exglobal_cleanup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh index 876fa3538b5..dcf1baef313 100755 --- a/scripts/exglobal_cleanup.sh +++ b/scripts/exglobal_cleanup.sh @@ -14,7 +14,7 @@ rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}" # Search and delete files/directories from DATAROOT/ older than ${purge_every_days} days # purge_every_days should be a positive integer -purge_every_days=3 +#purge_every_days=3 # Find and delete files older than ${purge_every_days} days #find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \; From cc99a5aff4991843f8a46c0807925750f55c2949 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 28 Jun 2024 19:46:44 +0000 Subject: [PATCH 58/74] Update --- parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 | 29 ++++++++---------------- parm/gdas/staging/atm_var_bkg.yaml.j2 | 13 ++--------- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 27 ++++++++-------------- ush/forecast_postdet.sh | 2 ++ ush/parsing_model_configure_FV3.sh | 4 ++-- 5 files changed, 25 insertions(+), 50 deletions(-) diff --git a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 index eda3dad5a7c..b10a1ddb6b5 100644 --- a/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_lgetkf_bkg.yaml.j2 @@ -1,6 +1,3 @@ -{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} -{% set time_list = [current_cycle] %} - mkdir: {% for imem in range(1,NMEM_ENS+1) %} {% set memchar = 'mem%03d' | format(imem) %} @@ -14,19 +11,13 @@ mkdir: - '{{ COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}' {% endfor %} copy: -{% for time in time_list %} - {% for imem in range(1,NMEM_ENS+1) %} - {% set memchar = 'mem%03d' | format(imem) %} - {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, - '${RUN}': 'enkfgdas', - '${YMD}': previous_cycle | to_YMD, - '${HH}': previous_cycle | strftime('%H'), - '${MEMDIR}': memchar }) %} -- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/{{ memchar }}/'] - {% for ftype in ftype_list %} - {% for itile in range(1,7) %} -- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/'] - {% endfor %} - {% endfor %} - {% endfor %} -{% endfor %} +{% for imem in range(1,NMEM_ENS+1) %} + {% set memchar = 'mem%03d' | format(imem) %} + {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, + '${RUN}': 'enkfgdas', + '${YMD}': previous_cycle | to_YMD, + '${HH}': previous_cycle | strftime('%H'), + '${MEMDIR}': memchar }) %} +- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/bkg/{{ memchar }}/'] +- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/{{ memchar }}/'] +{% endfor %} \ No newline at end of file diff --git a/parm/gdas/staging/atm_var_bkg.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2 index 37af8336494..d30e938b863 100644 --- a/parm/gdas/staging/atm_var_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_var_bkg.yaml.j2 @@ -1,14 +1,5 @@ -{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} -{% set time_list = [current_cycle] %} - mkdir: - '{{ DATA }}/bkg' copy: -{% for time in time_list %} -- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/'] - {% for ftype in ftype_list %} - {% for itile in range(1,ntiles+1) %} -- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/'] - {% endfor %} - {% endfor %} -{% endfor %} +- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/bkg/'] +- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/'] \ No newline at end of file diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 index e499c86d574..776a74304dc 100644 --- a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 +++ b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 @@ -1,24 +1,15 @@ -{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %} -{% set time_list = [current_cycle] %} - mkdir: {% for imem in range(1,NMEM_ENS+1) %} - '{{ DATA }}/ens/{{ 'mem%03d' | format(imem) }}' {% endfor %} copy: -{% for time in time_list %} - {% for imem in range(1,NMEM_ENS+1) %} - {% set memchar = 'mem%03d' | format(imem) %} - {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, - '${RUN}': 'enkfgdas', - '${YMD}': previous_cycle | to_YMD, - '${HH}': previous_cycle | strftime('%H'), - '${MEMDIR}': memchar }) %} -- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/ens/{{ memchar }}/'] - {% for ftype in ftype_list %} - {% for itile in range(1,ntiles+1) %} -- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/ens/{{ memchar }}/'] - {% endfor %} - {% endfor %} - {% endfor %} +{% for imem in range(1,NMEM_ENS+1) %} + {% set memchar = 'mem%03d' | format(imem) %} + {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR, + '${RUN}': 'enkfgdas', + '${YMD}': previous_cycle | to_YMD, + '${HH}': previous_cycle | strftime('%H'), + '${MEMDIR}': memchar }) %} +- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/ens/{{ memchar }}/'] +- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/ens/{{ memchar }}/'] {% endfor %} diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 993331d70b6..b3f334de6b4 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -209,6 +209,8 @@ EOF ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atmf${FH3}.nc" "atmf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.sfcf${FH3}.nc" "sfcf${FH3}.nc" ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.atm.logf${FH3}.txt" "log.atm.f${FH3}" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_atmf${FH3}.nc" "cubed_sphere_grid_atmf${FH3}.nc" + ${NLN} "${COMOUT_ATMOS_HISTORY}/${RUN}.t${cyc}z.cubed_sphere_grid_sfcf${FH3}.nc" "cubed_sphere_grid_sfcf${FH3}.nc" if [[ "${WRITE_DOPOST}" == ".true." ]]; then ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.master.grb2f${FH3}" "GFSPRS.GrbF${FH2}" ${NLN} "${COMOUT_ATMOS_MASTER}/${RUN}.t${cyc}z.sfluxgrbf${FH3}.grib2" "GFSFLX.GrbF${FH2}" diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh index 7d64ab38f90..d2dd434fffc 100755 --- a/ush/parsing_model_configure_FV3.sh +++ b/ush/parsing_model_configure_FV3.sh @@ -31,7 +31,7 @@ local WRITE_GROUP=${WRITE_GROUP:-1} local WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP:-24} local ITASKS=1 local OUTPUT_HISTORY=${OUTPUT_HISTORY:-".true."} -local HISTORY_FILE_ON_NATIVE_GRID=".false." +local HISTORY_FILE_ON_NATIVE_GRID=".true." local WRITE_DOPOST=${WRITE_DOPOST:-".false."} local WRITE_NSFLIP=${WRITE_NSFLIP:-".false."} local NUM_FILES=${NUM_FILES:-2} @@ -39,7 +39,7 @@ local FILENAME_BASE="'atm' 'sfc'" # OUTPUT_GRID local OUTPUT_FILE="'${OUTPUT_FILETYPE_ATM}' '${OUTPUT_FILETYPE_SFC}'" local ZSTANDARD_LEVEL=0 -local IDEFLATE=1 # netCDF zlib lossless compression (0-9); 0: no compression +local IDEFLATE=0 # netCDF zlib lossless compression (0-9); 0: no compression local QUANTIZE_NSD=${QUANTIZE_NSD:-0} # netCDF compression local ICHUNK2D=$((4*restile)) local JCHUNK2D=$((2*restile)) From 16eaef37fa70443a01ef1fe748aaf9964f8b0b0a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Jul 2024 13:27:01 +0000 Subject: [PATCH 59/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index e3644a98c36..af8531d3e39 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit e3644a98c362d7321f9e3081a4e55947885ed2bf +Subproject commit af8531d3e39be89f189dac3a3927f6fa667fcd94 From a3928d2707bb5dae4d2718cbffad9aebd0ac96f8 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Jul 2024 13:29:59 +0000 Subject: [PATCH 60/74] Revert "Address issues in creating XML for GFS forecast-only with app S2SWA (#2757)" This reverts commit fc668aa422ebbad76ceda1b3bbf8dc0ea432defd. --- workflow/applications/applications.py | 6 ------ workflow/applications/gfs_forecast_only.py | 8 +++----- workflow/rocoto/gfs_tasks.py | 4 +--- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 64440830c18..6a4d240fe5f 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -79,18 +79,12 @@ def __init__(self, conf: Configuration) -> None: self.wave_cdumps = [wave_cdump] self.aero_anl_cdumps = None - self.aero_fcst_cdumps = None if self.do_aero: aero_anl_cdump = _base.get('AERO_ANL_CDUMP', 'BOTH').lower() if aero_anl_cdump in ['both']: self.aero_anl_cdumps = ['gfs', 'gdas'] elif aero_anl_cdump in ['gfs', 'gdas']: self.aero_anl_cdumps = [aero_anl_cdump] - aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', None).lower() - if aero_fcst_cdump in ['both']: - self.aero_fcst_cdumps = ['gfs', 'gdas'] - elif aero_fcst_cdump in ['gfs', 'gdas']: - self.aero_fcst_cdumps = [aero_fcst_cdump] def _init_finalize(self, conf: Configuration): print("Finalizing initialize") diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index 938a514896e..e219032551c 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -25,8 +25,7 @@ def _get_app_configs(self): configs += ['atmos_products'] if self.do_aero: - if not self._base['EXP_WARM_START']: - configs += ['aerosol_init'] + configs += ['aerosol_init'] if self.do_tracker: configs += ['tracker'] @@ -88,10 +87,9 @@ def get_task_names(self): tasks = ['stage_ic'] if self.do_aero: - aero_fcst_cdump = self._base.get('AERO_FCST_CDUMP', 'BOTH').lower() + aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', 'BOTH').lower() if self._base['CDUMP'] in aero_fcst_cdump or aero_fcst_cdump == "both": - if not self._base['EXP_WARM_START']: - tasks += ['aerosol_init'] + tasks += ['aerosol_init'] if self.do_wave: tasks += ['waveinit'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index c14c3e2898f..530ea465c41 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -856,9 +856,7 @@ def _fcst_forecast_only(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_aero and \ - self.cdump in self.app_config.aero_fcst_cdumps and \ - not self._base['EXP_WARM_START']: + if self.app_config.do_aero and self.cdump in self.app_config.aero_fcst_cdumps: # Calculate offset based on CDUMP = gfs | gdas interval = None if self.cdump in ['gfs']: From 9c7a5b6b28e604d144e6cdd75fe9c3903780d312 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Jul 2024 13:30:22 +0000 Subject: [PATCH 61/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index e3644a98c36..af8531d3e39 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit e3644a98c362d7321f9e3081a4e55947885ed2bf +Subproject commit af8531d3e39be89f189dac3a3927f6fa667fcd94 From b6c43168ad00e5e295d79e38ac612ce268e762b0 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Jul 2024 13:49:13 +0000 Subject: [PATCH 62/74] Revert "Revert "Address issues in creating XML for GFS forecast-only with app S2SWA (#2757)"" This reverts commit a3928d2707bb5dae4d2718cbffad9aebd0ac96f8. --- workflow/applications/applications.py | 6 ++++++ workflow/applications/gfs_forecast_only.py | 8 +++++--- workflow/rocoto/gfs_tasks.py | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index 6a4d240fe5f..64440830c18 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -79,12 +79,18 @@ def __init__(self, conf: Configuration) -> None: self.wave_cdumps = [wave_cdump] self.aero_anl_cdumps = None + self.aero_fcst_cdumps = None if self.do_aero: aero_anl_cdump = _base.get('AERO_ANL_CDUMP', 'BOTH').lower() if aero_anl_cdump in ['both']: self.aero_anl_cdumps = ['gfs', 'gdas'] elif aero_anl_cdump in ['gfs', 'gdas']: self.aero_anl_cdumps = [aero_anl_cdump] + aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', None).lower() + if aero_fcst_cdump in ['both']: + self.aero_fcst_cdumps = ['gfs', 'gdas'] + elif aero_fcst_cdump in ['gfs', 'gdas']: + self.aero_fcst_cdumps = [aero_fcst_cdump] def _init_finalize(self, conf: Configuration): print("Finalizing initialize") diff --git a/workflow/applications/gfs_forecast_only.py b/workflow/applications/gfs_forecast_only.py index e219032551c..938a514896e 100644 --- a/workflow/applications/gfs_forecast_only.py +++ b/workflow/applications/gfs_forecast_only.py @@ -25,7 +25,8 @@ def _get_app_configs(self): configs += ['atmos_products'] if self.do_aero: - configs += ['aerosol_init'] + if not self._base['EXP_WARM_START']: + configs += ['aerosol_init'] if self.do_tracker: configs += ['tracker'] @@ -87,9 +88,10 @@ def get_task_names(self): tasks = ['stage_ic'] if self.do_aero: - aero_fcst_cdump = _base.get('AERO_FCST_CDUMP', 'BOTH').lower() + aero_fcst_cdump = self._base.get('AERO_FCST_CDUMP', 'BOTH').lower() if self._base['CDUMP'] in aero_fcst_cdump or aero_fcst_cdump == "both": - tasks += ['aerosol_init'] + if not self._base['EXP_WARM_START']: + tasks += ['aerosol_init'] if self.do_wave: tasks += ['waveinit'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 530ea465c41..c14c3e2898f 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -856,7 +856,9 @@ def _fcst_forecast_only(self): dep_dict = {'type': 'task', 'name': f'{self.cdump}{wave_job}'} dependencies.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_aero and self.cdump in self.app_config.aero_fcst_cdumps: + if self.app_config.do_aero and \ + self.cdump in self.app_config.aero_fcst_cdumps and \ + not self._base['EXP_WARM_START']: # Calculate offset based on CDUMP = gfs | gdas interval = None if self.cdump in ['gfs']: From 0fdd8894332b04e74b29afffb307f3620bff0d67 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Jul 2024 13:50:08 +0000 Subject: [PATCH 63/74] Revert "Update GDAS hash" This reverts commit 16eaef37fa70443a01ef1fe748aaf9964f8b0b0a. --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index af8531d3e39..e3644a98c36 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit af8531d3e39be89f189dac3a3927f6fa667fcd94 +Subproject commit e3644a98c362d7321f9e3081a4e55947885ed2bf From 41f50430bb8b0529e380c9ec065a4ebd0f59573a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 19 Jul 2024 14:30:48 +0000 Subject: [PATCH 64/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index af8531d3e39..5927257ca34 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit af8531d3e39be89f189dac3a3927f6fa667fcd94 +Subproject commit 5927257ca344e9733bb9a1e3f2e7e35ba9fc8b13 From f32bb4503e50cf8947ad01e43b56ab28578bdf5b Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 19 Jul 2024 14:34:50 +0000 Subject: [PATCH 65/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 5927257ca34..d579d8bbd7d 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 5927257ca344e9733bb9a1e3f2e7e35ba9fc8b13 +Subproject commit d579d8bbd7dcc48d0dce4c148736b54b04539f77 From b1fc7569999d2bb89db8a70718cf58a932b9b7c3 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 25 Jul 2024 12:41:26 +0000 Subject: [PATCH 66/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index d0a98cec7ae..9bd93a5abd1 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit d0a98cec7ae32e9cf1d7445f33ff8a5c97360316 +Subproject commit 9bd93a5abd19927a7e41b2c3fc5c4f1e571508c5 From 3e30c8ba4d33fc75eb5bba73ea8db6446cfd5fec Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 25 Jul 2024 09:11:23 -0400 Subject: [PATCH 67/74] Update atm_var_bkg.yaml.j2 --- parm/gdas/staging/atm_var_bkg.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/staging/atm_var_bkg.yaml.j2 b/parm/gdas/staging/atm_var_bkg.yaml.j2 index d30e938b863..92704e1f51d 100644 --- a/parm/gdas/staging/atm_var_bkg.yaml.j2 +++ b/parm/gdas/staging/atm_var_bkg.yaml.j2 @@ -2,4 +2,4 @@ mkdir: - '{{ DATA }}/bkg' copy: - ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/bkg/'] -- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/'] \ No newline at end of file +- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/'] From 018f24b9ed69027c9348e7b89e76bc440e481ceb Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA <134300700+DavidNew-NOAA@users.noreply.github.com> Date: Thu, 25 Jul 2024 09:11:50 -0400 Subject: [PATCH 68/74] Update atm_var_fv3ens.yaml.j2 --- parm/gdas/staging/atm_var_fv3ens.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 index 383a3a59315..776a74304dc 100644 --- a/parm/gdas/staging/atm_var_fv3ens.yaml.j2 +++ b/parm/gdas/staging/atm_var_fv3ens.yaml.j2 @@ -12,4 +12,4 @@ copy: '${MEMDIR}': memchar }) %} - ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/ens/{{ memchar }}/'] - ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/ens/{{ memchar }}/'] -{% endfor %} \ No newline at end of file +{% endfor %} From 6e856360eac3cd507e7ba47d101c78fe905c0bda Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Thu, 25 Jul 2024 14:26:52 -0400 Subject: [PATCH 69/74] Temporarily disable METplus jobs (#2796) --- parm/config/gfs/config.base | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 9fd66bf0c72..fca3fd99edb 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -453,6 +453,8 @@ export binary_diag=".false." # Verification options export DO_METP="@DO_METP@" # Run METPLUS jobs - set METPLUS settings in config.metp +# TODO Reenable METplus jobs when issue #2790 is resolved +export DO_METP="NO" export DO_FIT2OBS="YES" # Run fit to observations package export DO_VRFY_OCEANDA="@DO_VRFY_OCEANDA@" # Run SOCA Ocean and Seaice DA verification tasks From 48442e9a9d0962c0992c1b80e3c9afea9d0e5a69 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 26 Jul 2024 13:49:26 +0000 Subject: [PATCH 70/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 9bd93a5abd1..0cf2015c074 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 9bd93a5abd19927a7e41b2c3fc5c4f1e571508c5 +Subproject commit 0cf2015c074114c3504778d9fe51f8df4a9825a4 From 8f7e91b67b851af69337237b561183f87eb17531 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 30 Jul 2024 14:05:46 +0000 Subject: [PATCH 71/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 0cf2015c074..f229880aab6 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 0cf2015c074114c3504778d9fe51f8df4a9825a4 +Subproject commit f229880aab64279d6acfba25947f99406a7f4021 From 586800cbcd2ca31f0f9b3902ea753201fd5b6937 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 1 Aug 2024 14:09:04 +0000 Subject: [PATCH 72/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index f229880aab6..baa8740e246 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit f229880aab64279d6acfba25947f99406a7f4021 +Subproject commit baa8740e2464cd4ba38bbfdd6786261a24630c12 From 352b8c2ea19d717790f43c07c100151375054495 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 6 Aug 2024 09:06:55 -0500 Subject: [PATCH 73/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index baa8740e246..a8faa7bb720 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit baa8740e2464cd4ba38bbfdd6786261a24630c12 +Subproject commit a8faa7bb720fd92edd7409e31495a84129011ebe From 0197f3fefe19cd7b07410d07cbbc3c1f49b1b10f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 6 Aug 2024 12:47:29 -0500 Subject: [PATCH 74/74] Update GDAS hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index a8faa7bb720..f3fa26d4d66 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit a8faa7bb720fd92edd7409e31495a84129011ebe +Subproject commit f3fa26d4d6693fcf451184d5ecabb86c1b4190ca