From bbc238ed98d87f3af557fc0705c68940a93c39bb Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 18 Mar 2024 17:37:59 +0000 Subject: [PATCH 01/28] Adding rocoto job for converting JEDI increment to FV3 increment --- jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT | 37 ++++++++++ jobs/rocoto/atmanlinc.sh | 24 +++++++ parm/config/gfs/config.atmanl | 2 + parm/config/gfs/config.atmanlinc | 11 +++ parm/config/gfs/config.resources | 16 ++++- parm/gdas/atm_jedi_fix.yaml | 2 + scripts/exglobal_atm_analysis_increment.py | 22 ++++++ sorc/link_workflow.sh | 1 + ush/python/pygfs/task/analysis.py | 29 ++++++++ ush/python/pygfs/task/atm_analysis.py | 81 ++++++++++------------ workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 26 ++++++- workflow/rocoto/tasks.py | 2 +- 13 files changed, 209 insertions(+), 48 deletions(-) create mode 100755 jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT create mode 100755 jobs/rocoto/atmanlinc.sh create mode 100644 parm/config/gfs/config.atmanlinc create mode 100755 scripts/exglobal_atm_analysis_increment.py diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT b/jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT new file mode 100755 index 00000000000..de2b3d40b56 --- /dev/null +++ b/jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}atmanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinc" -c "base atmanl atmanlinc" + +############################################## +# Set variables used in the script +############################################## + + +############################################## +# Begin JOB SPECIFIC work +############################################## + + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_increment.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/atmanlinc.sh b/jobs/rocoto/atmanlinc.sh new file mode 100755 index 00000000000..ce82254ac6f --- /dev/null +++ b/jobs/rocoto/atmanlinc.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="atmanlinc" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT" +status=$? +exit "${status}" diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 11358de8a84..a4b277c1f2a 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -7,6 +7,7 @@ echo "BEGIN: config.atmanl" export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/gdas_prototype_3d.yaml.j2" export JEDIYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2" +export JEDIINC2FV3YAML="${PARMgfs}/gdas/atm/utils/gdas_jediinc2fv3.yaml.j2" export STATICB_TYPE="gsibec" export INTERP_METHOD='barycentric' @@ -26,5 +27,6 @@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE=${EXECgfs}/fv3jedi_var.x +export JEDIINC2FV3EXE=${EXECgfs}/gdas_fv3jedi_jediinc2fv3.x echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlinc b/parm/config/gfs/config.atmanlinc new file mode 100644 index 00000000000..ef770e4ab11 --- /dev/null +++ b/parm/config/gfs/config.atmanlinc @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.atmanlinc ########## +# Atm Var Analysis specific + +echo "BEGIN: config.atmanlinc" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmanlinc + +echo "END: config.atmanlinc" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 21f0a111ef3..6266eb5d584 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -10,7 +10,7 @@ if (( $# != 1 )); then echo "argument can be any one of the following:" echo "stage_ic aerosol_init" echo "prep prepsnowobs prepatmiodaobs" - echo "atmanlinit atmanlrun atmanlfinal" + echo "atmanlinit atmanlrun atmanlinc atmanlfinal" echo "atmensanlinit atmensanlrun atmensanlfinal" echo "snowanl" echo "aeroanlinit aeroanlrun aeroanlfinal" @@ -223,6 +223,20 @@ case ${step} in export is_exclusive=True ;; + "atmanlinc") + export layout_x=${layout_x_atmanl} + export layout_y=${layout_y_atmanl} + + export wtime_atmanlinc="00:30:00" + export npe_atmanlinc=$(( layout_x * layout_y * 1 )) + export npe_atmanlinc_gfs=$(( layout_x * layout_y * 1 )) + export nth_atmanlinc=1 + export nth_atmanlinc_gfs=${nth_atmanlinc} + export npe_node_atmanlinc=$(( npe_node_max / nth_atmanlinc )) + export memory_atmanlinc="96GB" + export is_exclusive=True + ;; + "atmanlfinal") export wtime_atmanlfinal="00:30:00" export npe_atmanlfinal=${npe_node_max} diff --git a/parm/gdas/atm_jedi_fix.yaml b/parm/gdas/atm_jedi_fix.yaml index 69039baddf7..a9cb2309f07 100644 --- a/parm/gdas/atm_jedi_fix.yaml +++ b/parm/gdas/atm_jedi_fix.yaml @@ -5,3 +5,5 @@ copy: - ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml'] - ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table'] - ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'] +- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_history.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_history.yaml'] +- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_fv3inc.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_fv3inc.yaml'] diff --git a/scripts/exglobal_atm_analysis_increment.py b/scripts/exglobal_atm_analysis_increment.py new file mode 100755 index 00000000000..27be45a9c30 --- /dev/null +++ b/scripts/exglobal_atm_analysis_increment.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# exglobal_atm_analysis_increment.py +# This script creates an AtmAnalysis object +# and runs the increment method +# which converts the JEDI increment into an FV3 increment +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.atm_analysis import AtmAnalysis + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the atm analysis task + AtmAnl = AtmAnalysis(config) + AtmAnl.increment() diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 2b54f3ea10b..82f7c0b339c 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -323,6 +323,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "fv3jedi_enshofx.x" \ "fv3jedi_hofx_nomodel.x" \ "fv3jedi_testdata_downloader.py" \ + "gdas_fv3jedi_jediinc2fv3.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ "gdas_obsprovider2ioda.x" \ diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 5709bc130e9..72b0ee8d928 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -41,6 +41,9 @@ def initialize(self) -> None: # link jedi executable to run directory self.link_jediexe() + # link jediinc2fv3 app to run directory + self.link_jediinc2fv3exe() + @logit(logger) def get_obs_dict(self) -> Dict[str, Any]: """Compile a dictionary of observation files to copy @@ -203,6 +206,32 @@ def link_jediexe(self) -> None: return + @logit(logger) + def link_jediinc2fv3exe(self) -> None: + """Compile a dictionary of background error files to copy + + This method links the jediinc2fv3 OOPS app to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.JEDIINC2FV3EXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + @staticmethod @logit(logger) def get_fv3ens_dict(config: Dict[str, Any]) -> Dict[str, Any]: diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 5a90a89e34b..7db07e230f2 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -32,6 +32,7 @@ def __init__(self, config): _res_anl = int(self.config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) _fv3jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") + _jediinc2fv3_yaml = os.path.join(self.runtime_config.DATA, 'gdas_fv3jedi_jediinc2fv3.yaml') # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -49,6 +50,7 @@ def __init__(self, config): 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", 'fv3jedi_yaml': _fv3jedi_yaml, + 'jediinc2fv3_yaml': _jediinc2fv3_yaml, } ) @@ -107,6 +109,12 @@ def initialize(self: Analysis) -> None: save_as_yaml(varda_yaml, self.task_config.fv3jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.fv3jedi_yaml}") + # generate jediinc2fv3 YAML file + logger.debug(f"Generate jediinc2fv3 YAML file: {self.task_config.jediinc2fv3_yaml}") + jediinc2fv3_yaml = parse_j2yaml(self.task_config.JEDIINC2FV3YAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(jediinc2fv3_yaml, self.task_config.jediinc2fv3_yaml) + logger.info(f"Wrote jediinc2fv3 YAML to: {self.task_config.jediinc2fv3_yaml}") + # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -135,6 +143,26 @@ def execute(self: Analysis) -> None: pass + @logit(logger) + def increment(self: Analysis) -> None: + + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_ATMANL) + exec_name = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_jediinc2fv3.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.jediinc2fv3_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + @logit(logger) def finalize(self: Analysis) -> None: """Finalize a global atm analysis @@ -212,9 +240,16 @@ def finalize(self: Analysis) -> None: } FileHandler(bias_copy).sync() - # Create UFS model readable atm increment file from UFS-DA atm increment - logger.info("Create UFS model readable atm increment file from UFS-DA atm increment") - self.jedi2fv3inc() + # Copy FV3 atm increment to comrot directory + cdate = to_fv3time(self.task_config.current_cycle) + cdate_inc = cdate.replace('.', '_') + src = os.path.join(self.task_config.DATA, 'anl', f"atminc.{cdate_inc}z.nc4") + dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f'{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc') + logger.debug(f"Copying {src} to {dest}") + inc_copy = { + 'copy': [[src, dest]] + } + FileHandler(inc_copy).sync() def clean(self): super().clean() @@ -404,43 +439,3 @@ def _get_berror_dict_gsibec(config: Dict[str, Any]) -> Dict[str, List[str]]: 'copy': berror_list, } return berror_dict - - @logit(logger) - def jedi2fv3inc(self: Analysis) -> None: - """Generate UFS model readable analysis increment - - This method writes a UFS DA atm increment in UFS model readable format. - This includes: - - write UFS-DA atm increments using variable names expected by UFS model - - compute and write delp increment - - compute and write hydrostatic delz increment - - Please note that some of these steps are temporary and will be modified - once the modle is able to directly read atm increments. - - """ - # Select the atm guess file based on the analysis and background resolutions - # Fields from the atm guess are used to compute the delp and delz increments - case_anl = int(self.task_config.CASE_ANL[1:]) - case = int(self.task_config.CASE[1:]) - - file = f"{self.task_config.GPREFIX}" + "atmf006" + f"{'' if case_anl == case else '.ensres'}" + ".nc" - atmges_fv3 = os.path.join(self.task_config.COM_ATMOS_HISTORY_PREV, file) - - # Set the path/name to the input UFS-DA atm increment file (atminc_jedi) - # and the output UFS model atm increment file (atminc_fv3) - cdate = to_fv3time(self.task_config.current_cycle) - cdate_inc = cdate.replace('.', '_') - atminc_jedi = os.path.join(self.task_config.DATA, 'anl', f'atminc.{cdate_inc}z.nc4') - atminc_fv3 = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atminc.nc") - - # Reference the python script which does the actual work - incpy = os.path.join(self.task_config.HOMEgfs, 'ush/jediinc2fv3.py') - - # Execute incpy to create the UFS model atm increment file - cmd = Executable(incpy) - cmd.add_default_arg(atmges_fv3) - cmd.add_default_arg(atminc_jedi) - cmd.add_default_arg(atminc_fv3) - logger.debug(f"Executing {cmd}") - cmd(output='stdout', error='stderr') diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 51b430a996a..6059382bca4 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -38,7 +38,7 @@ def _get_app_configs(self): configs = ['prep'] if self.do_jediatmvar: - configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlfinal'] + configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlinc', 'atmanlfinal'] else: configs += ['anal', 'analdiag'] @@ -132,7 +132,7 @@ def get_task_names(self): gdas_gfs_common_cleanup_tasks = ['arch', 'cleanup'] if self.do_jediatmvar: - gdas_gfs_common_tasks_before_fcst += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlfinal'] + gdas_gfs_common_tasks_before_fcst += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlinc', 'atmanlfinal'] else: gdas_gfs_common_tasks_before_fcst += ['anal'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index b91e974c740..afaf9a49406 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -435,11 +435,35 @@ def atmanlrun(self): return task - def atmanlfinal(self): + def atmanlinc(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanlrun'} deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('atmanlinc') + task_name = f'{self.cdump}atmanlinc' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlinc.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + + def atmanlfinal(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanlinc'} + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('atmanlfinal') diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 01bf586af13..af2e8a2c314 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -13,7 +13,7 @@ class Tasks: SERVICE_TASKS = ['arch', 'earc'] VALID_TASKS = ['aerosol_init', 'stage_ic', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", - 'prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlfinal', + 'prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlinc', 'atmanlfinal', 'prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', From afc57a91f3af67a15b9a2fa4f58bfae6b4401221 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 19 Mar 2024 19:24:41 +0000 Subject: [PATCH 02/28] Added HPC environments and bug fixes --- env/HERA.env | 10 +++++++++- env/JET.env | 10 +++++++++- env/ORION.env | 10 +++++++++- env/S4.env | 10 +++++++++- env/WCOSS2.env | 10 +++++++++- parm/config/gfs/config.resources | 4 ++-- ush/python/pygfs/task/atm_analysis.py | 2 +- 7 files changed, 48 insertions(+), 8 deletions(-) diff --git a/env/HERA.env b/env/HERA.env index f55434e8d9d..87fca708a8e 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -79,6 +79,14 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "atmanlinc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmanlinc)) + + export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} + [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} + export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc} --cpus-per-task=${NTHREADS_ATMANLINC}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/JET.env b/env/JET.env index df6666d8dc2..4b2119f2e5e 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -80,6 +80,14 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "atmanlinc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmanlinc)) + + export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} + [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} + export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc}" + elif [[ "${step}" = "ocnanalbmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" diff --git a/env/ORION.env b/env/ORION.env index 17d0d24d978..2a28372799c 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlrun" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -88,6 +88,14 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "atmanlinc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmanlinc)) + + export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} + [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} + export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc} --cpus-per-task=${NTHREADS_ATMANLINC}" + elif [[ "${step}" = "ocnanalbmat" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" diff --git a/env/S4.env b/env/S4.env index ab564eb9745..74dccf8aafe 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -80,6 +80,14 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "atmanlinc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmanlinc)) + + export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} + [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} + export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc}" + elif [[ "${step}" = "ocnanalbmat" ]]; then echo "WARNING: ${step} is not enabled on S4!" diff --git a/env/WCOSS2.env b/env/WCOSS2.env index ebb55a40771..90d81e6bf6d 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen esfc efcs epos" echo "postsnd awips gempak" @@ -74,6 +74,14 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" +elif [[ "${step}" = "atmanlinc" ]]; then + + nth_max=$((npe_node_max / npe_node_atmanlinc)) + + export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} + [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} + export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc}" + elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then export OMP_PLACES=cores diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 6266eb5d584..e84ed81665b 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -228,8 +228,8 @@ case ${step} in export layout_y=${layout_y_atmanl} export wtime_atmanlinc="00:30:00" - export npe_atmanlinc=$(( layout_x * layout_y * 1 )) - export npe_atmanlinc_gfs=$(( layout_x * layout_y * 1 )) + export npe_atmanlinc=$(( layout_x * layout_y * 6 )) + export npe_atmanlinc_gfs=$(( layout_x * layout_y * 6 )) export nth_atmanlinc=1 export nth_atmanlinc_gfs=${nth_atmanlinc} export npe_node_atmanlinc=$(( npe_node_max / nth_atmanlinc )) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 7db07e230f2..9430cd19952 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -148,7 +148,7 @@ def increment(self: Analysis) -> None: chdir(self.task_config.DATA) - exec_cmd = Executable(self.task_config.APRUN_ATMANL) + exec_cmd = Executable(self.task_config.APRUN_ATMANLINC) exec_name = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_jediinc2fv3.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg(self.task_config.jediinc2fv3_yaml) From 051d08f3efa6682cf898c69ca30bd8d316a40d93 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Wed, 20 Mar 2024 18:35:04 +0000 Subject: [PATCH 03/28] Remove jediinc2fv3.py from link_workflow.sh --- sorc/link_workflow.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 82f7c0b339c..a8dc3c452e6 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -217,7 +217,6 @@ fi if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then cd "${HOMEgfs}/ush" || exit 1 ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ufsda" . - ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/jediinc2fv3.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/scripts/exglobal_prep_ocean_obs.py" . From 5bf1b25c54a039cb3e9d1ab49e59ccdc4bdc48a1 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 21 Mar 2024 08:20:46 -0500 Subject: [PATCH 04/28] Fixed small omission --- parm/gdas/atm_jedi_fix.yaml.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parm/gdas/atm_jedi_fix.yaml.j2 b/parm/gdas/atm_jedi_fix.yaml.j2 index 69039baddf7..a9cb2309f07 100644 --- a/parm/gdas/atm_jedi_fix.yaml.j2 +++ b/parm/gdas/atm_jedi_fix.yaml.j2 @@ -5,3 +5,5 @@ copy: - ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml'] - ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table'] - ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'] +- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_history.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_history.yaml'] +- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_fv3inc.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_fv3inc.yaml'] From 57caeec1ccc7cc772e057cf0d1a29c7b2cabd43e Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 21 Mar 2024 15:49:19 -0500 Subject: [PATCH 05/28] Rename atmanlrun and atmanlinc to atmanlvar and atmanlfv3 respectively --- env/HERA.env | 12 +++---- env/JET.env | 12 +++---- env/ORION.env | 10 +++--- env/S4.env | 12 +++---- env/WCOSS2.env | 12 +++---- ...RUN => JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT} | 4 +-- ...EMENT => JGLOBAL_ATM_ANALYSIS_VARIATIONAL} | 4 +-- jobs/rocoto/{atmanlrun.sh => atmanlfv3inc.sh} | 4 +-- jobs/rocoto/{atmanlinc.sh => atmanlvar.sh} | 4 +-- parm/config/gfs/config.atmanl | 8 ++--- parm/config/gfs/config.atmanlfv3inc | 11 ++++++ parm/config/gfs/config.atmanlinc | 11 ------ parm/config/gfs/config.atmanlrun | 11 ------ parm/config/gfs/config.atmanlvar | 11 ++++++ parm/config/gfs/config.resources | 34 +++++++++--------- ... => exglobal_aero_analysis_variational.py} | 0 ...=> exglobal_atm_analysis_fv3_increment.py} | 4 +-- ...y => exglobal_atm_analysis_variational.py} | 4 +-- sorc/link_workflow.sh | 2 +- ush/python/pygfs/task/analysis.py | 16 ++++----- ush/python/pygfs/task/atm_analysis.py | 36 +++++++++---------- workflow/applications/gfs_cycled.py | 4 +-- workflow/rocoto/gfs_tasks.py | 20 +++++------ workflow/rocoto/tasks.py | 2 +- 24 files changed, 124 insertions(+), 124 deletions(-) rename jobs/{JGLOBAL_ATM_ANALYSIS_RUN => JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT} (83%) rename jobs/{JGLOBAL_ATM_ANALYSIS_INCREMENT => JGLOBAL_ATM_ANALYSIS_VARIATIONAL} (84%) rename jobs/rocoto/{atmanlrun.sh => atmanlfv3inc.sh} (89%) rename jobs/rocoto/{atmanlinc.sh => atmanlvar.sh} (89%) create mode 100644 parm/config/gfs/config.atmanlfv3inc delete mode 100644 parm/config/gfs/config.atmanlinc delete mode 100644 parm/config/gfs/config.atmanlrun create mode 100644 parm/config/gfs/config.atmanlvar rename scripts/{exglobal_aero_analysis_run.py => exglobal_aero_analysis_variational.py} (100%) rename scripts/{exglobal_atm_analysis_increment.py => exglobal_atm_analysis_fv3_increment.py} (89%) rename scripts/{exglobal_atm_analysis_run.py => exglobal_atm_analysis_variational.py} (89%) diff --git a/env/HERA.env b/env/HERA.env index 87fca708a8e..201f37f9ca0 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -79,13 +79,13 @@ elif [[ "${step}" = "aeroanlrun" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" -elif [[ "${step}" = "atmanlinc" ]]; then +elif [[ "${step}" = "atmanlfv3inc" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlinc)) + nth_max=$((npe_node_max / npe_node_atmanlfv3inc)) - export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} - [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} - export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc} --cpus-per-task=${NTHREADS_ATMANLINC}" + export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} + export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/JET.env b/env/JET.env index 4b2119f2e5e..0d58564b354 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -80,13 +80,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" -elif [[ "${step}" = "atmanlinc" ]]; then +elif [[ "${step}" = "atmanlfv3inc" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlinc)) + nth_max=$((npe_node_max / npe_node_atmanlfv3inc)) - export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} - [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} - export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc}" + export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} + export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}" elif [[ "${step}" = "ocnanalbmat" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 2a28372799c..b23ceb3a4d8 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -88,13 +88,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" -elif [[ "${step}" = "atmanlinc" ]]; then +elif [[ "${step}" = "atmanlfv3inc" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlinc)) + nth_max=$((npe_node_max / npe_node_atmanlfv3inc)) - export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} - [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} - export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc} --cpus-per-task=${NTHREADS_ATMANLINC}" + export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} + export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" elif [[ "${step}" = "ocnanalbmat" ]]; then diff --git a/env/S4.env b/env/S4.env index 74dccf8aafe..60039a72b20 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -80,13 +80,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" -elif [[ "${step}" = "atmanlinc" ]]; then +elif [[ "${step}" = "atmanlfv3inc" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlinc)) + nth_max=$((npe_node_max / npe_node_atmanlfv3inc)) - export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} - [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} - export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc}" + export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} + export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}" elif [[ "${step}" = "ocnanalbmat" ]]; then echo "WARNING: ${step} is not enabled on S4!" diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 90d81e6bf6d..3e13d08d7ca 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlinc" + echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen esfc efcs epos" echo "postsnd awips gempak" @@ -74,13 +74,13 @@ elif [[ "${step}" = "snowanl" ]]; then export APRUN_APPLY_INCR="${launcher} -n 6" -elif [[ "${step}" = "atmanlinc" ]]; then +elif [[ "${step}" = "atmanlfv3inc" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlinc)) + nth_max=$((npe_node_max / npe_node_atmanlfv3inc)) - export NTHREADS_ATMANLINC=${nth_atmanlinc:-${nth_max}} - [[ ${NTHREADS_ATMANLINC} -gt ${nth_max} ]] && export NTHREADS_ATMANLINC=${nth_max} - export APRUN_ATMANLINC="${launcher} -n ${npe_atmanlinc}" + export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} + export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_RUN b/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT similarity index 83% rename from jobs/JGLOBAL_ATM_ANALYSIS_RUN rename to jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT index 2105d719dee..9a9a4760653 100755 --- a/jobs/JGLOBAL_ATM_ANALYSIS_RUN +++ b/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}atmanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlrun" -c "base atmanl atmanlrun" +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlfv3inc" -c "base atmanl atmanlfv3inc" ############################################## # Set variables used in the script @@ -18,7 +18,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlrun" -c "base atmanl atmanlrun" ############################################################### # Run relevant script -EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_run.py} +EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_fv3_increment.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT b/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL similarity index 84% rename from jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT rename to jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL index de2b3d40b56..552eccf9116 100755 --- a/jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT +++ b/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}atmanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinc" -c "base atmanl atmanlinc" +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlvar" -c "base atmanl atmanlvar" ############################################## # Set variables used in the script @@ -18,7 +18,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinc" -c "base atmanl atmanlinc" ############################################################### # Run relevant script -EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_increment.py} +EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atm_analysis_variational.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/atmanlrun.sh b/jobs/rocoto/atmanlfv3inc.sh similarity index 89% rename from jobs/rocoto/atmanlrun.sh rename to jobs/rocoto/atmanlfv3inc.sh index 1b87cb40747..effc18cee50 100755 --- a/jobs/rocoto/atmanlrun.sh +++ b/jobs/rocoto/atmanlfv3inc.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="atmanlrun" +export job="atmanlfv3inc" export jobid="${job}.$$" ############################################################### @@ -19,6 +19,6 @@ export PYTHONPATH ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN" +"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FV3_INCREMENT" status=$? exit "${status}" diff --git a/jobs/rocoto/atmanlinc.sh b/jobs/rocoto/atmanlvar.sh similarity index 89% rename from jobs/rocoto/atmanlinc.sh rename to jobs/rocoto/atmanlvar.sh index ce82254ac6f..812e3c706a6 100755 --- a/jobs/rocoto/atmanlinc.sh +++ b/jobs/rocoto/atmanlvar.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="atmanlinc" +export job="atmanlvar" export jobid="${job}.$$" ############################################################### @@ -19,6 +19,6 @@ export PYTHONPATH ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INCREMENT" +"${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_VARIATIONAL" status=$? exit "${status}" diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index da7d8bf8629..c6c4bfb89da 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -6,8 +6,8 @@ echo "BEGIN: config.atmanl" export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/gdas_prototype_3d.yaml.j2" -export JEDIYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2" -export JEDIINC2FV3YAML="${PARMgfs}/gdas/atm/utils/gdas_jediinc2fv3.yaml.j2" +export VARYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2" +export FV3INCYAML="${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc.yaml.j2" export STATICB_TYPE="gsibec" export INTERP_METHOD='barycentric' @@ -29,7 +29,7 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/fv3jedi_var.x -export JEDIINC2FV3EXE=${EXECgfs}/gdas_fv3jedi_jediinc2fv3.x +export VAREXE=${EXECgfs}/fv3jedi_var.x +export FV3INCEXE=${EXECgfs}/fv3jedi_fv3inc.x echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc new file mode 100644 index 00000000000..4e7714628e6 --- /dev/null +++ b/parm/config/gfs/config.atmanlfv3inc @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.atmanlfv3inc ########## +# Atm Var Analysis specific + +echo "BEGIN: config.atmanlfv3inc" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmanlfv3inc + +echo "END: config.atmanlfv3inc" diff --git a/parm/config/gfs/config.atmanlinc b/parm/config/gfs/config.atmanlinc deleted file mode 100644 index ef770e4ab11..00000000000 --- a/parm/config/gfs/config.atmanlinc +++ /dev/null @@ -1,11 +0,0 @@ -#! /usr/bin/env bash - -########## config.atmanlinc ########## -# Atm Var Analysis specific - -echo "BEGIN: config.atmanlinc" - -# Get task specific resources -. "${EXPDIR}/config.resources" atmanlinc - -echo "END: config.atmanlinc" diff --git a/parm/config/gfs/config.atmanlrun b/parm/config/gfs/config.atmanlrun deleted file mode 100644 index 68b76157186..00000000000 --- a/parm/config/gfs/config.atmanlrun +++ /dev/null @@ -1,11 +0,0 @@ -#! /usr/bin/env bash - -########## config.atmanlrun ########## -# Atm Var Analysis specific - -echo "BEGIN: config.atmanlrun" - -# Get task specific resources -. "${EXPDIR}/config.resources" atmanlrun - -echo "END: config.atmanlrun" diff --git a/parm/config/gfs/config.atmanlvar b/parm/config/gfs/config.atmanlvar new file mode 100644 index 00000000000..cbc0334a08c --- /dev/null +++ b/parm/config/gfs/config.atmanlvar @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.atmanlvar ########## +# Atm Var Analysis specific + +echo "BEGIN: config.atmanlvar" + +# Get task specific resources +. "${EXPDIR}/config.resources" atmanlvar + +echo "END: config.atmanlvar" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index e84ed81665b..0ba3016c409 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -10,7 +10,7 @@ if (( $# != 1 )); then echo "argument can be any one of the following:" echo "stage_ic aerosol_init" echo "prep prepsnowobs prepatmiodaobs" - echo "atmanlinit atmanlrun atmanlinc atmanlfinal" + echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlrun atmensanlfinal" echo "snowanl" echo "aeroanlinit aeroanlrun aeroanlfinal" @@ -209,31 +209,31 @@ case ${step} in export memory_atmanlinit="3072M" ;; - "atmanlrun") + "atmanlvar") export layout_x=${layout_x_atmanl} export layout_y=${layout_y_atmanl} - export wtime_atmanlrun="00:30:00" - export npe_atmanlrun=$(( layout_x * layout_y * 6 )) - export npe_atmanlrun_gfs=$(( layout_x * layout_y * 6 )) - export nth_atmanlrun=1 - export nth_atmanlrun_gfs=${nth_atmanlrun} - export npe_node_atmanlrun=$(( npe_node_max / nth_atmanlrun )) - export memory_atmanlrun="96GB" + export wtime_atmanlvar="00:30:00" + export npe_atmanlvar=$(( layout_x * layout_y * 6 )) + export npe_atmanlvar_gfs=$(( layout_x * layout_y * 6 )) + export nth_atmanlvar=1 + export nth_atmanlvar_gfs=${nth_atmanlvar} + export npe_node_atmanlvar=$(( npe_node_max / nth_atmanlvar )) + export memory_atmanlvar="96GB" export is_exclusive=True ;; - "atmanlinc") + "atmanlfv3inc") export layout_x=${layout_x_atmanl} export layout_y=${layout_y_atmanl} - export wtime_atmanlinc="00:30:00" - export npe_atmanlinc=$(( layout_x * layout_y * 6 )) - export npe_atmanlinc_gfs=$(( layout_x * layout_y * 6 )) - export nth_atmanlinc=1 - export nth_atmanlinc_gfs=${nth_atmanlinc} - export npe_node_atmanlinc=$(( npe_node_max / nth_atmanlinc )) - export memory_atmanlinc="96GB" + export wtime_atmanlfv3inc="00:30:00" + export npe_atmanlfv3inc=$(( layout_x * layout_y * 6 )) + export npe_atmanlfv3inc_gfs=$(( layout_x * layout_y * 6 )) + export nth_atmanlfv3inc=1 + export nth_atmanlfv3inc_gfs=${nth_atmanlfv3inc} + export npe_node_atmanlfv3inc=$(( npe_node_max / nth_atmanlfv3inc )) + export memory_atmanlfv3inc="96GB" export is_exclusive=True ;; diff --git a/scripts/exglobal_aero_analysis_run.py b/scripts/exglobal_aero_analysis_variational.py similarity index 100% rename from scripts/exglobal_aero_analysis_run.py rename to scripts/exglobal_aero_analysis_variational.py diff --git a/scripts/exglobal_atm_analysis_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py similarity index 89% rename from scripts/exglobal_atm_analysis_increment.py rename to scripts/exglobal_atm_analysis_fv3_increment.py index 27be45a9c30..04d42cd4d13 100755 --- a/scripts/exglobal_atm_analysis_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# exglobal_atm_analysis_increment.py +# exglobal_atm_analysis_fv3_increment.py # This script creates an AtmAnalysis object # and runs the increment method # which converts the JEDI increment into an FV3 increment @@ -19,4 +19,4 @@ # Instantiate the atm analysis task AtmAnl = AtmAnalysis(config) - AtmAnl.increment() + AtmAnl.fv3_increment() diff --git a/scripts/exglobal_atm_analysis_run.py b/scripts/exglobal_atm_analysis_variational.py similarity index 89% rename from scripts/exglobal_atm_analysis_run.py rename to scripts/exglobal_atm_analysis_variational.py index 8adbe4a2677..07bc2083311 100755 --- a/scripts/exglobal_atm_analysis_run.py +++ b/scripts/exglobal_atm_analysis_variational.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# exglobal_atm_analysis_run.py +# exglobal_atm_analysis_variational.py # This script creates an AtmAnalysis object # and runs the execute method # which executes the global atm variational analysis @@ -19,4 +19,4 @@ # Instantiate the atm analysis task AtmAnl = AtmAnalysis(config) - AtmAnl.execute() + AtmAnl.variational() diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index a8dc3c452e6..106107123d4 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -322,7 +322,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "fv3jedi_enshofx.x" \ "fv3jedi_hofx_nomodel.x" \ "fv3jedi_testdata_downloader.py" \ - "gdas_fv3jedi_jediinc2fv3.x" \ + "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ "gdas_obsprovider2ioda.x" \ diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 04492ce7fa5..a72b5acc0c8 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -43,11 +43,11 @@ def initialize(self) -> None: bias_dict = self.get_bias_dict() FileHandler(bias_dict).sync() - # link jedi executable to run directory - self.link_jediexe() + # link jedi variational executable to run directory + self.link_varexe() - # link jediinc2fv3 app to run directory - self.link_jediinc2fv3exe() + # link fv3 increment converter executable to run directory + self.link_fv3incexe() @logit(logger) def get_jedi_config(self) -> Dict[str, Any]: @@ -209,7 +209,7 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: return berror_dict @logit(logger) - def link_jediexe(self) -> None: + def link_varexe(self) -> None: """Compile a dictionary of background error files to copy This method links a JEDI executable to the run directory @@ -222,7 +222,7 @@ def link_jediexe(self) -> None: ---------- None """ - exe_src = self.task_config.JEDIEXE + exe_src = self.task_config.VAREXE # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. logger.info(f"Link executable {exe_src} to DATA/") @@ -235,7 +235,7 @@ def link_jediexe(self) -> None: return @logit(logger) - def link_jediinc2fv3exe(self) -> None: + def link_fv3incexe(self) -> None: """Compile a dictionary of background error files to copy This method links the jediinc2fv3 OOPS app to the run directory @@ -248,7 +248,7 @@ def link_jediinc2fv3exe(self) -> None: ---------- None """ - exe_src = self.task_config.JEDIINC2FV3EXE + exe_src = self.task_config.FV3INCEXE # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. logger.info(f"Link executable {exe_src} to DATA/") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index da4542bad9e..86c185b37b0 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -31,8 +31,8 @@ def __init__(self, config): _res = int(self.config.CASE[1:]) _res_anl = int(self.config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) - _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") - _jediinc2fv3_yaml = os.path.join(self.runtime_config.DATA, 'gdas_fv3jedi_jediinc2fv3.yaml') + _var_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") + _fv3inc_yaml = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc.yaml') # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -49,8 +49,8 @@ def __init__(self, config): 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", - 'jedi_yaml': _jedi_yaml, - 'jediinc2fv3_yaml': _jediinc2fv3_yaml, + 'var_yaml': _var_yaml, + 'fv3inc_yaml': _fv3inc_yaml, } ) @@ -102,15 +102,15 @@ def initialize(self: Analysis) -> None: FileHandler(self.get_bkg_dict(AttrDict(self.task_config))).sync() # generate variational YAML file - logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") - save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) - logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") + logger.debug(f"Generate variational YAML file: {self.task_config.var_yaml}") + save_as_yaml(self.task_config.jedi_config, self.task_config.var_yaml) + logger.info(f"Wrote variational YAML to: {self.task_config.var_yaml}") - # generate jediinc2fv3 YAML file - logger.debug(f"Generate jediinc2fv3 YAML file: {self.task_config.jediinc2fv3_yaml}") - jediinc2fv3_yaml = parse_j2yaml(self.task_config.JEDIINC2FV3YAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(jediinc2fv3_yaml, self.task_config.jediinc2fv3_yaml) - logger.info(f"Wrote jediinc2fv3 YAML to: {self.task_config.jediinc2fv3_yaml}") + # generate FV3 increment converter YAML file + logger.debug(f"Generate FV3 increment converter YAML file: {self.task_config.fv3inc_yaml}") + fv3inc_yaml = parse_j2yaml(self.task_config.FV3INCYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(fv3inc_yaml, self.task_config.fv3inc_yaml) + logger.info(f"Wrote FV3 increment converter YAML to: {self.task_config.fv3inc_yaml}") # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -121,14 +121,14 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def execute(self: Analysis) -> None: + def variational(self: Analysis) -> None: chdir(self.task_config.DATA) exec_cmd = Executable(self.task_config.APRUN_ATMANL) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) + exec_cmd.add_default_arg(self.task_config.var_yaml) try: logger.debug(f"Executing {exec_cmd}") @@ -141,14 +141,14 @@ def execute(self: Analysis) -> None: pass @logit(logger) - def increment(self: Analysis) -> None: + def fv3_increment(self: Analysis) -> None: chdir(self.task_config.DATA) exec_cmd = Executable(self.task_config.APRUN_ATMANLINC) - exec_name = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_jediinc2fv3.x') + exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_fv3inc.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.jediinc2fv3_yaml) + exec_cmd.add_default_arg(self.task_config.fv3inc_yaml) try: logger.debug(f"Executing {exec_cmd}") @@ -195,7 +195,7 @@ def finalize(self: Analysis) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") + logger.info(f"Copying {self.task_config.var_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") logger.debug(f"Copying {src} to {dest}") diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 6059382bca4..2558ff230ec 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -38,7 +38,7 @@ def _get_app_configs(self): configs = ['prep'] if self.do_jediatmvar: - configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlinc', 'atmanlfinal'] + configs += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] else: configs += ['anal', 'analdiag'] @@ -132,7 +132,7 @@ def get_task_names(self): gdas_gfs_common_cleanup_tasks = ['arch', 'cleanup'] if self.do_jediatmvar: - gdas_gfs_common_tasks_before_fcst += ['prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlinc', 'atmanlfinal'] + gdas_gfs_common_tasks_before_fcst += ['prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal'] else: gdas_gfs_common_tasks_before_fcst += ['anal'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index f8e66442fce..6d2c9819411 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -411,21 +411,21 @@ def atmanlinit(self): return task - def atmanlrun(self): + def atmanlvar(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanlinit'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('atmanlrun') - task_name = f'{self.cdump}atmanlrun' + resources = self.get_resource('atmanlvar') + task_name = f'{self.cdump}atmanlvar' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -435,21 +435,21 @@ def atmanlrun(self): return task - def atmanlinc(self): + def atmanlfv3inc(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanlvar'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('atmanlinc') - task_name = f'{self.cdump}atmanlinc' + resources = self.get_resource('atmanlfv3inc') + task_name = f'{self.cdump}atmanlfv3inc' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlinc.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/atmanlfv3inc.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -462,7 +462,7 @@ def atmanlinc(self): def atmanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanlinc'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}atmanlfv3inc'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index af2e8a2c314..e88898d76dc 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -13,7 +13,7 @@ class Tasks: SERVICE_TASKS = ['arch', 'earc'] VALID_TASKS = ['aerosol_init', 'stage_ic', 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", - 'prepatmiodaobs', 'atmanlinit', 'atmanlrun', 'atmanlinc', 'atmanlfinal', + 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', 'ocnanalprep', 'ocnanalbmat', 'ocnanalrun', 'ocnanalchkpt', 'ocnanalpost', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', From b72ccfdbc0d866326f56d011c66f7e502fb0b6fc Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 25 Mar 2024 10:42:57 -0500 Subject: [PATCH 06/28] Finalized HPC environment file changes and bugs in analysis scripts --- env/AWSPW.env | 2 +- env/CONTAINER.env | 2 +- env/HERA.env | 12 ++++++------ env/HERCULES.env | 10 +++++----- env/JET.env | 12 ++++++------ env/ORION.env | 12 ++++++------ env/S4.env | 12 ++++++------ env/WCOSS2.env | 12 ++++++------ ush/python/pygfs/task/analysis.py | 10 +++++----- ush/python/pygfs/task/atm_analysis.py | 4 ++-- 10 files changed, 44 insertions(+), 44 deletions(-) diff --git a/env/AWSPW.env b/env/AWSPW.env index 2dbba67eb31..f5473cf019e 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlvar atmensanlrun aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/CONTAINER.env b/env/CONTAINER.env index bc2d64b4cef..a6de7ecbe79 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlvar atmensanlrun aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/HERA.env b/env/HERA.env index 201f37f9ca0..d9cc47f26b5 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -53,13 +53,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} -elif [[ "${step}" = "atmanlrun" ]]; then +elif [[ "${step}" = "atmanlvar" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlrun)) + nth_max=$((npe_node_max / npe_node_atmanlvar)) - export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}} - [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max} - export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}" + export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}} + [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} + export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}" elif [[ "${step}" = "atmensanlrun" ]]; then diff --git a/env/HERCULES.env b/env/HERCULES.env index 7d2aa5f8d0f..69d415b6094 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -53,13 +53,13 @@ case ${step} in export wave_mpmd=${mpmd_opt} ;; - "atmanlrun") + "atmanlvar") - nth_max=$((npe_node_max / npe_node_atmanlrun)) + nth_max=$((npe_node_max / npe_node_atmanlvar)) - export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}} - [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max} - export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}" + export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}} + [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} + export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}" ;; "atmensanlrun") diff --git a/env/JET.env b/env/JET.env index 0d58564b354..3b4c2c2c53d 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -44,13 +44,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} -elif [[ "${step}" = "atmanlrun" ]]; then +elif [[ "${step}" = "atmanlvar" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlrun)) + nth_max=$((npe_node_max / npe_node_atmanlvar)) - export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}} - [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max} - export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun}" + export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}} + [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} + export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}" elif [[ "${step}" = "atmensanlrun" ]]; then diff --git a/env/ORION.env b/env/ORION.env index b23ceb3a4d8..6aac84a1694 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlrun" + echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -52,13 +52,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} -elif [[ "${step}" = "atmanlrun" ]]; then +elif [[ "${step}" = "atmanlvar" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlrun)) + nth_max=$((npe_node_max / npe_node_atmanlvar)) - export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}} - [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max} - export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}" + export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}} + [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} + export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}" elif [[ "${step}" = "atmensanlrun" ]]; then diff --git a/env/S4.env b/env/S4.env index 60039a72b20..9cbf8b7bdbf 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -44,13 +44,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} -elif [[ "${step}" = "atmanlrun" ]]; then +elif [[ "${step}" = "atmanlvar" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlrun)) + nth_max=$((npe_node_max / npe_node_atmanlvar)) - export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}} - [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max} - export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun}" + export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}} + [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} + export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}" elif [[ "${step}" = "atmensanlrun" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 3e13d08d7ca..9669a7f7a85 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl atmanlfv3inc" + echo "atmanlvar atmensanlrun aeroanlrun snowanl atmanlfv3inc" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen esfc efcs epos" echo "postsnd awips gempak" @@ -38,13 +38,13 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec="${launcher} -np" export wave_mpmd=${mpmd_opt} -elif [[ "${step}" = "atmanlrun" ]]; then +elif [[ "${step}" = "atmanlvar" ]]; then - nth_max=$((npe_node_max / npe_node_atmanlrun)) + nth_max=$((npe_node_max / npe_node_atmanlvar)) - export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}} - [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max} - export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun}" + export NTHREADS_ATMANLVAR=${nth_atmanlvar:-${nth_max}} + [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} + export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar}" elif [[ "${step}" = "atmensanlrun" ]]; then diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index a72b5acc0c8..615ce31c1e5 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -51,7 +51,7 @@ def initialize(self) -> None: @logit(logger) def get_jedi_config(self) -> Dict[str, Any]: - """Compile a dictionary of JEDI configuration from JEDIYAML template file + """Compile a dictionary of JEDI configuration from VARYAML template file Parameters ---------- @@ -63,8 +63,8 @@ def get_jedi_config(self) -> Dict[str, Any]: """ # generate JEDI YAML file - logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}") - jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + logger.info(f"Generate JEDI variational YAML config: {self.task_config.var_yaml}") + jedi_config = parse_j2yaml(self.task_config.VARYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) logger.debug(f"JEDI config:\n{pformat(jedi_config)}") return jedi_config @@ -86,7 +86,7 @@ def get_obs_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation files to copy for FileHandler """ - logger.info(f"Extracting a list of observation files from {self.task_config.JEDIYAML}") + logger.info(f"Extracting a list of observation files from {self.task_config.VARYAML}") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -120,7 +120,7 @@ def get_bias_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation bias files to copy for FileHandler """ - logger.info(f"Extracting a list of bias correction files from {self.task_config.JEDIYAML}") + logger.info(f"Extracting a list of bias correction files from {self.task_config.VARYAML}") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 86c185b37b0..4c7f3c3a8d6 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -125,7 +125,7 @@ def variational(self: Analysis) -> None: chdir(self.task_config.DATA) - exec_cmd = Executable(self.task_config.APRUN_ATMANL) + exec_cmd = Executable(self.task_config.APRUN_ATMANLVAR) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg(self.task_config.var_yaml) @@ -145,7 +145,7 @@ def fv3_increment(self: Analysis) -> None: chdir(self.task_config.DATA) - exec_cmd = Executable(self.task_config.APRUN_ATMANLINC) + exec_cmd = Executable(self.task_config.APRUN_ATMANLFV3INC) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_fv3inc.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg(self.task_config.fv3inc_yaml) From 98d297c8a10c2386e68d1d4d8e1b1b9799375d1f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 25 Mar 2024 11:12:27 -0500 Subject: [PATCH 07/28] Minor documentation update --- ush/python/pygfs/task/atm_analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 4c7f3c3a8d6..5f8d40d976c 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -67,7 +67,8 @@ def initialize(self: Analysis) -> None: - staging FV3-JEDI fix files - staging B error files - staging model backgrounds - - generating a YAML file for the JEDI executable + - generating a YAML file for the JEDI variational executable + - generating a YAML file for the JEDI FV3 increment converter executable - creating output directories """ super().initialize() From 88c414c7b9c1c176d95a55e58aa7967e039ce317 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 25 Mar 2024 11:54:35 -0500 Subject: [PATCH 08/28] Undo change --- parm/gdas/aero_jedi_fix.yaml.j2 | 2 -- 1 file changed, 2 deletions(-) diff --git a/parm/gdas/aero_jedi_fix.yaml.j2 b/parm/gdas/aero_jedi_fix.yaml.j2 index a9cb2309f07..69039baddf7 100644 --- a/parm/gdas/aero_jedi_fix.yaml.j2 +++ b/parm/gdas/aero_jedi_fix.yaml.j2 @@ -5,5 +5,3 @@ copy: - ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/fmsmpp.nml', '{{ DATA }}/fv3jedi/fmsmpp.nml'] - ['{{ FIXgfs }}/gdas/fv3jedi/fv3files/field_table_gfdl', '{{ DATA }}/fv3jedi/field_table'] - ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_restart.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_restart.yaml'] -- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_history.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_history.yaml'] -- ['{{ PARMgfs }}/gdas/io/fv3jedi_fieldmetadata_fv3inc.yaml', '{{ DATA }}/fv3jedi/fv3jedi_fieldmetadata_fv3inc.yaml'] From 8aa9ffc8d30eaec8a0eb7abd0d7db87b0ef82689 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 26 Mar 2024 08:55:56 -0500 Subject: [PATCH 09/28] Fix trailing whitespace --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 6d2c9819411..6ebe6fc09e2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -457,7 +457,7 @@ def atmanlfv3inc(self): task = rocoto.create_task(task_dict) - return task + return task def atmanlfinal(self): From 47dde9ff1d86018d5f5506f327ca98859745cc32 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 26 Mar 2024 12:34:37 -0500 Subject: [PATCH 10/28] Revert changes that completely eliminate jediinc2fv3.py interactions since still needed for ensemble analysis job --- sorc/link_workflow.sh | 1 + ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 106107123d4..d49b7f4dcdc 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -217,6 +217,7 @@ fi if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then cd "${HOMEgfs}/ush" || exit 1 ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ufsda" . + ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/jediinc2fv3.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/ioda/bufr2ioda/run_bufr2ioda.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py" . ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/scripts/exglobal_prep_ocean_obs.py" . diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 615ce31c1e5..0ee4a71d6a7 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -238,7 +238,7 @@ def link_varexe(self) -> None: def link_fv3incexe(self) -> None: """Compile a dictionary of background error files to copy - This method links the jediinc2fv3 OOPS app to the run directory + This method links the FV3 increment converter app to the run directory Parameters ---------- diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 5f8d40d976c..c8fb2327186 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -170,7 +170,7 @@ def finalize(self: Analysis) -> None: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - copy the updated bias correction files to ROTDIR - - write UFS model readable atm incrment file + - copy UFS model readable atm FV3 increment file to ROTDIR """ # ---- tar up diags From 8fe51fbb16871ae576f7256bdaee40cd6401d0b0 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 28 Mar 2024 09:50:54 -0500 Subject: [PATCH 11/28] Update GDASApp submodule with PR #983 --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index dd350d7e4da..18ba5da17f3 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit dd350d7e4daab0977407e388711807f13b204f6f +Subproject commit 18ba5da17f3a0e9329ddda4021b78cf4544037fb From 32d6e76a0f964dcd60b9fc4994ac251a9285b471 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 29 Mar 2024 10:12:55 -0500 Subject: [PATCH 12/28] Reverting accidental renaming --- ...aero_analysis_variational.py => exglobal_aero_analysis_run.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/{exglobal_aero_analysis_variational.py => exglobal_aero_analysis_run.py} (100%) diff --git a/scripts/exglobal_aero_analysis_variational.py b/scripts/exglobal_aero_analysis_run.py similarity index 100% rename from scripts/exglobal_aero_analysis_variational.py rename to scripts/exglobal_aero_analysis_run.py From 31dc53cb27a44852bfe581a5c6cf7bd15346db50 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 29 Mar 2024 15:32:14 -0500 Subject: [PATCH 13/28] Reverting some changes and revising some things. --- jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE | 2 +- parm/config/gfs/config.atmanl | 6 ++-- parm/config/gfs/config.atmanlfv3inc | 3 ++ ush/python/pygfs/task/analysis.py | 47 +++++---------------------- ush/python/pygfs/task/atm_analysis.py | 25 +++++++++----- workflow/applications/applications.py | 2 +- 6 files changed, 33 insertions(+), 52 deletions(-) diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE index b7e2eeacc64..f00ffdfc1a5 100755 --- a/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE @@ -2,7 +2,7 @@ source "${HOMEgfs}/ush/preamble.sh" export DATA=${DATA:-${DATAROOT}/${RUN}atmanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinit" -c "base atmanl atmanlinit" +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinit" -c "base atmanl atmanlinit atmanlfv3inc" ############################################## # Set variables used in the script diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index c6c4bfb89da..7cfd0cb47fb 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -6,8 +6,7 @@ echo "BEGIN: config.atmanl" export OBS_LIST="${PARMgfs}/gdas/atm/obs/lists/gdas_prototype_3d.yaml.j2" -export VARYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2" -export FV3INCYAML="${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc.yaml.j2" +export JEDIYAML="${PARMgfs}/gdas/atm/variational/3dvar_drpcg.yaml.j2" export STATICB_TYPE="gsibec" export INTERP_METHOD='barycentric' @@ -29,7 +28,6 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export VAREXE=${EXECgfs}/fv3jedi_var.x -export FV3INCEXE=${EXECgfs}/fv3jedi_fv3inc.x +export JEDIEXE=${EXECgfs}/fv3jedi_var.x echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc index 4e7714628e6..93135fd12a0 100644 --- a/parm/config/gfs/config.atmanlfv3inc +++ b/parm/config/gfs/config.atmanlfv3inc @@ -5,6 +5,9 @@ echo "BEGIN: config.atmanlfv3inc" +export FV3INCYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc.yaml.j2 +export FV3INCEXE=${EXECgfs}/fv3jedi_fv3inc.x + # Get task specific resources . "${EXPDIR}/config.resources" atmanlfv3inc diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 0ee4a71d6a7..2221fb7b340 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -43,15 +43,12 @@ def initialize(self) -> None: bias_dict = self.get_bias_dict() FileHandler(bias_dict).sync() - # link jedi variational executable to run directory - self.link_varexe() - - # link fv3 increment converter executable to run directory - self.link_fv3incexe() + # link jedi executable to run directory + self.link_jediexe() @logit(logger) def get_jedi_config(self) -> Dict[str, Any]: - """Compile a dictionary of JEDI configuration from VARYAML template file + """Compile a dictionary of JEDI configuration from JEDIYAML template file Parameters ---------- @@ -63,8 +60,8 @@ def get_jedi_config(self) -> Dict[str, Any]: """ # generate JEDI YAML file - logger.info(f"Generate JEDI variational YAML config: {self.task_config.var_yaml}") - jedi_config = parse_j2yaml(self.task_config.VARYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + logger.info(f"Generate JEDI YAML config: {self.task_config.jedi_yaml}") + jedi_config = parse_j2yaml(self.task_config.JEDIYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) logger.debug(f"JEDI config:\n{pformat(jedi_config)}") return jedi_config @@ -86,7 +83,7 @@ def get_obs_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation files to copy for FileHandler """ - logger.info(f"Extracting a list of observation files from {self.task_config.VARYAML}") + logger.info(f"Extracting a list of observation files from {self.task_config.JEDIYAML}") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -120,7 +117,7 @@ def get_bias_dict(self) -> Dict[str, Any]: a dictionary containing the list of observation bias files to copy for FileHandler """ - logger.info(f"Extracting a list of bias correction files from {self.task_config.VARYAML}") + logger.info(f"Extracting a list of bias correction files from {self.task_config.JEDIYAML}") observations = find_value_in_nested_dict(self.task_config.jedi_config, 'observations') logger.debug(f"observations:\n{pformat(observations)}") @@ -209,7 +206,7 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: return berror_dict @logit(logger) - def link_varexe(self) -> None: + def link_jediexe(self) -> None: """Compile a dictionary of background error files to copy This method links a JEDI executable to the run directory @@ -222,33 +219,7 @@ def link_varexe(self) -> None: ---------- None """ - exe_src = self.task_config.VAREXE - - # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - logger.info(f"Link executable {exe_src} to DATA/") - logger.warn("Linking is not permitted per EE2.") - exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - - return - - @logit(logger) - def link_fv3incexe(self) -> None: - """Compile a dictionary of background error files to copy - - This method links the FV3 increment converter app to the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - exe_src = self.task_config.FV3INCEXE + exe_src = self.task_config.JEDIEXE # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. logger.info(f"Link executable {exe_src} to DATA/") diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index c8fb2327186..0f3771958f6 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -10,7 +10,7 @@ from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, - chdir, + chdir, rm_p, parse_j2yaml, save_as_yaml, logit, Executable, @@ -31,8 +31,9 @@ def __init__(self, config): _res = int(self.config.CASE[1:]) _res_anl = int(self.config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) - _var_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") + _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") _fv3inc_yaml = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc.yaml') + _fv3inc_exe = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc.x') # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -49,8 +50,9 @@ def __init__(self, config): 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", - 'var_yaml': _var_yaml, + 'jedi_yaml': _jedi_yaml, 'fv3inc_yaml': _fv3inc_yaml, + 'fv3inc_exe': _fv3inc_exe, } ) @@ -103,9 +105,9 @@ def initialize(self: Analysis) -> None: FileHandler(self.get_bkg_dict(AttrDict(self.task_config))).sync() # generate variational YAML file - logger.debug(f"Generate variational YAML file: {self.task_config.var_yaml}") - save_as_yaml(self.task_config.jedi_config, self.task_config.var_yaml) - logger.info(f"Wrote variational YAML to: {self.task_config.var_yaml}") + logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") + save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) + logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") # generate FV3 increment converter YAML file logger.debug(f"Generate FV3 increment converter YAML file: {self.task_config.fv3inc_yaml}") @@ -113,6 +115,13 @@ def initialize(self: Analysis) -> None: save_as_yaml(fv3inc_yaml, self.task_config.fv3inc_yaml) logger.info(f"Wrote FV3 increment converter YAML to: {self.task_config.fv3inc_yaml}") + # link FV3 increment converter executable + logger.debug(f"Link FV3 increment converter executable: {self.task_config.fv3inc_exe}") + if os.path.exists(self.task_config.fv3inc_exe): + rm_p(self.task_config.fv3inc_exe) + os.symlink(self.task_config.FV3INCEXE, self.task_config.fv3inc_exe) + logger.debug(f"Linked FV3 increment converter executable: {self.task_config.fv3inc_exe}") + # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -129,7 +138,7 @@ def variational(self: Analysis) -> None: exec_cmd = Executable(self.task_config.APRUN_ATMANLVAR) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.var_yaml) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) try: logger.debug(f"Executing {exec_cmd}") @@ -196,7 +205,7 @@ def finalize(self: Analysis) -> None: archive.add(diaggzip, arcname=os.path.basename(diaggzip)) # copy full YAML from executable to ROTDIR - logger.info(f"Copying {self.task_config.var_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") + logger.info(f"Copying {self.task_config.jedi_yaml} to {self.task_config.COM_ATMOS_ANALYSIS}") src = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") dest = os.path.join(self.task_config.COM_ATMOS_ANALYSIS, f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.atmvar.yaml") logger.debug(f"Copying {src} to {dest}") diff --git a/workflow/applications/applications.py b/workflow/applications/applications.py index b20c5a7c286..3a8d23f744c 100644 --- a/workflow/applications/applications.py +++ b/workflow/applications/applications.py @@ -145,7 +145,7 @@ def _source_configs(self, conf: Configuration) -> Dict[str, Any]: files += ['config.anal', 'config.eupd'] elif config in ['efcs']: files += ['config.fcst', 'config.efcs'] - elif config in ['atmanlinit', 'atmanlrun']: + elif config in ['atmanlinit', 'atmanlvar', 'atmanlfv3inc']: files += ['config.atmanl', f'config.{config}'] elif config in ['atmensanlinit', 'atmensanlrun']: files += ['config.atmensanl', f'config.{config}'] From 63d2b63d2fde26abba2e8b2963c2c12d2553e708 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 29 Mar 2024 15:56:34 -0500 Subject: [PATCH 14/28] Update GDASApp develop --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 18ba5da17f3..b0ef0cea512 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 18ba5da17f3a0e9329ddda4021b78cf4544037fb +Subproject commit b0ef0cea512eecebf579d25123d90274d1202368 From 114a05442285a7b6765c11a58815ba6cc123ad55 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 4 Apr 2024 13:39:12 -0500 Subject: [PATCH 15/28] Update name of FV3 increment converter YAML for variational job --- parm/config/gfs/config.atmanlfv3inc | 2 +- ush/python/pygfs/task/atm_analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc index 93135fd12a0..4e04d614fdd 100644 --- a/parm/config/gfs/config.atmanlfv3inc +++ b/parm/config/gfs/config.atmanlfv3inc @@ -5,7 +5,7 @@ echo "BEGIN: config.atmanlfv3inc" -export FV3INCYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc.yaml.j2 +export FV3INCYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc_variational.yaml.j2 export FV3INCEXE=${EXECgfs}/fv3jedi_fv3inc.x # Get task specific resources diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 0f3771958f6..6cfbcd07e63 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -32,7 +32,7 @@ def __init__(self, config): _res_anl = int(self.config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") - _fv3inc_yaml = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc.yaml') + _fv3inc_yaml = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc_variational.yaml') _fv3inc_exe = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc.x') # Create a local dictionary that is repeatedly used across this class From 2ec1d3027e2e760a953941817e76eadfba172287 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 9 Apr 2024 08:37:15 -0500 Subject: [PATCH 16/28] Slight refactoring of analysis jobs --- parm/config/gfs/config.atmanl | 2 - parm/config/gfs/config.atmanlfv3inc | 6 +-- parm/config/gfs/config.atmanlvar | 2 + ush/python/pygfs/task/aero_analysis.py | 4 ++ ush/python/pygfs/task/analysis.py | 3 -- ush/python/pygfs/task/atm_analysis.py | 47 ++++++++---------------- ush/python/pygfs/task/atmens_analysis.py | 4 ++ ush/python/pygfs/task/snow_analysis.py | 4 ++ 8 files changed, 33 insertions(+), 39 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 7cfd0cb47fb..08af7205093 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -28,6 +28,4 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ -export JEDIEXE=${EXECgfs}/fv3jedi_var.x - echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlfv3inc b/parm/config/gfs/config.atmanlfv3inc index 4e04d614fdd..14c11d3dd3f 100644 --- a/parm/config/gfs/config.atmanlfv3inc +++ b/parm/config/gfs/config.atmanlfv3inc @@ -5,10 +5,10 @@ echo "BEGIN: config.atmanlfv3inc" -export FV3INCYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc_variational.yaml.j2 -export FV3INCEXE=${EXECgfs}/fv3jedi_fv3inc.x - # Get task specific resources . "${EXPDIR}/config.resources" atmanlfv3inc +export JEDIYAML=${PARMgfs}/gdas/atm/utils/fv3jedi_fv3inc_variational.yaml.j2 +export JEDIEXE=${EXECgfs}/fv3jedi_fv3inc.x + echo "END: config.atmanlfv3inc" diff --git a/parm/config/gfs/config.atmanlvar b/parm/config/gfs/config.atmanlvar index cbc0334a08c..bc9e17e02aa 100644 --- a/parm/config/gfs/config.atmanlvar +++ b/parm/config/gfs/config.atmanlvar @@ -8,4 +8,6 @@ echo "BEGIN: config.atmanlvar" # Get task specific resources . "${EXPDIR}/config.resources" atmanlvar +export JEDIEXE=${EXECgfs}/fv3jedi_var.x + echo "END: config.atmanlvar" diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index a61b7c82f38..e0f228f4556 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -68,6 +68,7 @@ def initialize(self: Analysis) -> None: - staging B error files - staging model backgrounds - generating a YAML file for the JEDI executable + - linking JEDI variational executable - creating output directories """ super().initialize() @@ -95,6 +96,9 @@ def initialize(self: Analysis) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") + # link variational JEDI executable to run directory + self.link_jediexe() + # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 2221fb7b340..1d1b4582690 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -43,9 +43,6 @@ def initialize(self) -> None: bias_dict = self.get_bias_dict() FileHandler(bias_dict).sync() - # link jedi executable to run directory - self.link_jediexe() - @logit(logger) def get_jedi_config(self) -> Dict[str, Any]: """Compile a dictionary of JEDI configuration from JEDIYAML template file diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 6cfbcd07e63..9e7c8c4b969 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -32,8 +32,6 @@ def __init__(self, config): _res_anl = int(self.config.CASE_ANL[1:]) _window_begin = add_to_datetime(self.runtime_config.current_cycle, -to_timedelta(f"{self.config.assim_freq}H") / 2) _jedi_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.atmvar.yaml") - _fv3inc_yaml = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc_variational.yaml') - _fv3inc_exe = os.path.join(self.runtime_config.DATA, 'fv3jedi_fv3inc.x') # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -51,8 +49,6 @@ def __init__(self, config): 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", 'jedi_yaml': _jedi_yaml, - 'fv3inc_yaml': _fv3inc_yaml, - 'fv3inc_exe': _fv3inc_exe, } ) @@ -70,7 +66,7 @@ def initialize(self: Analysis) -> None: - staging B error files - staging model backgrounds - generating a YAML file for the JEDI variational executable - - generating a YAML file for the JEDI FV3 increment converter executable + - linking JEDI variational executable - creating output directories """ super().initialize() @@ -109,18 +105,8 @@ def initialize(self: Analysis) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # generate FV3 increment converter YAML file - logger.debug(f"Generate FV3 increment converter YAML file: {self.task_config.fv3inc_yaml}") - fv3inc_yaml = parse_j2yaml(self.task_config.FV3INCYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(fv3inc_yaml, self.task_config.fv3inc_yaml) - logger.info(f"Wrote FV3 increment converter YAML to: {self.task_config.fv3inc_yaml}") - - # link FV3 increment converter executable - logger.debug(f"Link FV3 increment converter executable: {self.task_config.fv3inc_exe}") - if os.path.exists(self.task_config.fv3inc_exe): - rm_p(self.task_config.fv3inc_exe) - os.symlink(self.task_config.FV3INCEXE, self.task_config.fv3inc_exe) - logger.debug(f"Linked FV3 increment converter executable: {self.task_config.fv3inc_exe}") + # link variational JEDI executable to run directory + self.link_jediexe() # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") @@ -153,23 +139,22 @@ def variational(self: Analysis) -> None: @logit(logger) def fv3_increment(self: Analysis) -> None: - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMANLFV3INC) - exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_fv3inc.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.fv3inc_yaml) + # Setup task configuration + self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIYAML)) + self.task_config.jedi_exe = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIEXE)) - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + # Stage JEDI YAML file + save_as_yaml(self.get_jedi_config(), self.task_config.jedi_yaml) - pass + # Link JEDI executable + self.link_jediexe() + # Run executable + self.execute_jediexe(self.runtime_config.DATA, \ + self.task_config.APRUN_ATMANLFV3INC, \ + self.task_config.jedi_exe, \ + self.task_config.jedi_yaml) + @logit(logger) def finalize(self: Analysis) -> None: """Finalize a global atm analysis diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 1037b557c24..08f932a728e 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -62,6 +62,7 @@ def initialize(self: Analysis) -> None: - staging FV3-JEDI fix files - staging model backgrounds - generating a YAML file for the JEDI executable + - linking JEDI ensemble da executable - creating output directories Parameters @@ -120,6 +121,9 @@ def initialize(self: Analysis) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote ensemble da YAML to: {self.task_config.jedi_yaml}") + # link ensemble da JEDI executable to run directory + self.link_jediexe() + # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index c149f140b61..99152ba8f58 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -236,6 +236,7 @@ def initialize(self) -> None: This method: - creates artifacts in the DATA directory by copying fix files - creates the JEDI LETKF yaml from the template + - links JEDI LETKF executable - stages backgrounds, observations and ensemble members Parameters @@ -272,6 +273,9 @@ def initialize(self) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}") + # link letkfoi JEDI executable to run directory + self.link_jediexe() + # need output dir for diags and anl logger.info("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ From 7b40fce1cb7aa1e63731eb545b5b59b7ae513241 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Tue, 9 Apr 2024 12:46:19 -0500 Subject: [PATCH 17/28] Move executable linking to executation functions in analysis jobs --- jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE | 2 +- ush/python/pygfs/task/aero_analysis.py | 25 +++++------------ ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/atm_analysis.py | 35 +++++++----------------- ush/python/pygfs/task/atmens_analysis.py | 29 ++++++-------------- ush/python/pygfs/task/snow_analysis.py | 11 ++++---- 6 files changed, 33 insertions(+), 71 deletions(-) diff --git a/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE index 2d98996f31c..5b264839222 100755 --- a/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE @@ -2,7 +2,7 @@ source "${HOMEgfs}/ush/preamble.sh" export DATA=${DATA:-${DATAROOT}/${RUN}atmanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinit" -c "base atmanl atmanlinit atmanlfv3inc" +source "${HOMEgfs}/ush/jjob_header.sh" -e "atmanlinit" -c "base atmanl atmanlinit" ############################################## # Set variables used in the script diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index e0f228f4556..231049c6c05 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -96,9 +96,6 @@ def initialize(self: Analysis) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # link variational JEDI executable to run directory - self.link_jediexe() - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -110,22 +107,14 @@ def initialize(self: Analysis) -> None: @logit(logger) def execute(self: Analysis) -> None: - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_AEROANL) - exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + # link JEDI executable to run directory + self.task_config.jedi_exe = self.link_jediexe() - pass + # Run executable + self.execute_jediexe(self.runtime_config.DATA, \ + self.task_config.APRUN_AEROANL, \ + self.task_config.jedi_exe, \ + self.task_config.jedi_yaml) @logit(logger) def finalize(self: Analysis) -> None: diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 1d1b4582690..90fb0dda3ea 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -226,7 +226,7 @@ def link_jediexe(self) -> None: rm_p(exe_dest) os.symlink(exe_src, exe_dest) - return + return exe_dest @staticmethod @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 9e7c8c4b969..16eadd02992 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -66,7 +66,6 @@ def initialize(self: Analysis) -> None: - staging B error files - staging model backgrounds - generating a YAML file for the JEDI variational executable - - linking JEDI variational executable - creating output directories """ super().initialize() @@ -105,9 +104,6 @@ def initialize(self: Analysis) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # link variational JEDI executable to run directory - self.link_jediexe() - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -119,35 +115,24 @@ def initialize(self: Analysis) -> None: @logit(logger) def variational(self: Analysis) -> None: - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMANLVAR) - exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") + # Link JEDI executable + self.task_config.jedi_exe = self.link_jediexe() - pass + # Run executable + self.execute_jediexe(self.runtime_config.DATA, \ + self.task_config.APRUN_ATMANLVAR, \ + self.task_config.jedi_exe, \ + self.task_config.jedi_yaml) @logit(logger) def fv3_increment(self: Analysis) -> None: - # Setup task configuration + # Setup JEDI YAML file self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIYAML)) - self.task_config.jedi_exe = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIEXE)) - - # Stage JEDI YAML file save_as_yaml(self.get_jedi_config(), self.task_config.jedi_yaml) - # Link JEDI executable - self.link_jediexe() + # Link JEDI executable to run directory + self.task_config.jedi_exe = self.link_jediexe() # Run executable self.execute_jediexe(self.runtime_config.DATA, \ diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 08f932a728e..a028baf6e4c 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -62,7 +62,6 @@ def initialize(self: Analysis) -> None: - staging FV3-JEDI fix files - staging model backgrounds - generating a YAML file for the JEDI executable - - linking JEDI ensemble da executable - creating output directories Parameters @@ -121,9 +120,6 @@ def initialize(self: Analysis) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote ensemble da YAML to: {self.task_config.jedi_yaml}") - # link ensemble da JEDI executable to run directory - self.link_jediexe() - # need output dir for diags and anl logger.debug("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -149,22 +145,15 @@ def execute(self: Analysis) -> None: ---------- None """ - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_ATMENSANL) - exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_letkf.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.jedi_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass + + # link JEDI executable to run directory + self.task_config.jedi_exe = self.link_jediexe() + + # Run executable + self.execute_jediexe(self.runtime_config.DATA, \ + self.task_config.APRUN_ATMENSANL, \ + self.task_config.jedi_exe, \ + self.task_config.jedi_yaml) @logit(logger) def finalize(self: Analysis) -> None: diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 99152ba8f58..81213163cd9 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -236,7 +236,6 @@ def initialize(self) -> None: This method: - creates artifacts in the DATA directory by copying fix files - creates the JEDI LETKF yaml from the template - - links JEDI LETKF executable - stages backgrounds, observations and ensemble members Parameters @@ -273,9 +272,6 @@ def initialize(self) -> None: save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote letkfoi YAML to: {self.task_config.jedi_yaml}") - # link letkfoi JEDI executable to run directory - self.link_jediexe() - # need output dir for diags and anl logger.info("Create empty output [anl, diags] directories to receive output from executable") newdirs = [ @@ -298,12 +294,15 @@ def execute(self) -> None: Instance of the SnowAnalysis object """ + # link JEDI executable to run directory + self.task_config.jedi_exe = self.link_jediexe() + # create a temporary dict of all keys needed in this method localconf = AttrDict() keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_ATMOS_RESTART_PREV', 'COM_SNOW_ANALYSIS', 'APREFIX', 'SNOWDEPTHVAR', 'BESTDDEV', 'CASE', 'OCNRES', 'ntiles', - 'APRUN_SNOWANL', 'JEDIEXE', 'jedi_yaml', + 'APRUN_SNOWANL', 'JEDIEXE', 'jedi_yaml', 'jedi_exe', 'APPLY_INCR_NML_TMPL', 'APPLY_INCR_EXE', 'APRUN_APPLY_INCR'] for key in keys: localconf[key] = self.task_config[key] @@ -316,7 +315,7 @@ def execute(self) -> None: logger.info("Running JEDI LETKF") self.execute_jediexe(localconf.DATA, localconf.APRUN_SNOWANL, - os.path.basename(localconf.JEDIEXE), + localconf.jedi_exe, localconf.jedi_yaml) logger.info("Creating analysis from backgrounds and increments") From f098e557c3215149f3d18513b930ccda4f68a01d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 11 Apr 2024 07:37:17 -0500 Subject: [PATCH 18/28] Coding norms --- ush/python/pygfs/task/aero_analysis.py | 6 +++--- ush/python/pygfs/task/atm_analysis.py | 14 +++++++------- ush/python/pygfs/task/atmens_analysis.py | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 231049c6c05..f7efd503d9e 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -111,9 +111,9 @@ def execute(self: Analysis) -> None: self.task_config.jedi_exe = self.link_jediexe() # Run executable - self.execute_jediexe(self.runtime_config.DATA, \ - self.task_config.APRUN_AEROANL, \ - self.task_config.jedi_exe, \ + self.execute_jediexe(self.runtime_config.DATA, + self.task_config.APRUN_AEROANL, + self.task_config.jedi_exe, self.task_config.jedi_yaml) @logit(logger) diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 16eadd02992..e3d2c519ed6 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -119,9 +119,9 @@ def variational(self: Analysis) -> None: self.task_config.jedi_exe = self.link_jediexe() # Run executable - self.execute_jediexe(self.runtime_config.DATA, \ - self.task_config.APRUN_ATMANLVAR, \ - self.task_config.jedi_exe, \ + self.execute_jediexe(self.runtime_config.DATA, + self.task_config.APRUN_ATMANLVAR, + self.task_config.jedi_exe, self.task_config.jedi_yaml) @logit(logger) @@ -135,11 +135,11 @@ def fv3_increment(self: Analysis) -> None: self.task_config.jedi_exe = self.link_jediexe() # Run executable - self.execute_jediexe(self.runtime_config.DATA, \ - self.task_config.APRUN_ATMANLFV3INC, \ - self.task_config.jedi_exe, \ + self.execute_jediexe(self.runtime_config.DATA, + self.task_config.APRUN_ATMANLFV3INC, + self.task_config.jedi_exe, self.task_config.jedi_yaml) - + @logit(logger) def finalize(self: Analysis) -> None: """Finalize a global atm analysis diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index a028baf6e4c..81c99b3755b 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -150,9 +150,9 @@ def execute(self: Analysis) -> None: self.task_config.jedi_exe = self.link_jediexe() # Run executable - self.execute_jediexe(self.runtime_config.DATA, \ - self.task_config.APRUN_ATMENSANL, \ - self.task_config.jedi_exe, \ + self.execute_jediexe(self.runtime_config.DATA, + self.task_config.APRUN_ATMENSANL, + self.task_config.jedi_exe, self.task_config.jedi_yaml) @logit(logger) From 2fcc64b55a5fa26fa05a44b246640e0b52a3bc26 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 11 Apr 2024 10:25:15 -0500 Subject: [PATCH 19/28] Revert some things --- ush/python/pygfs/task/aero_analysis.py | 23 +++++++++++------- ush/python/pygfs/task/analysis.py | 3 +++ ush/python/pygfs/task/atm_analysis.py | 30 +++++++++++++++--------- ush/python/pygfs/task/atmens_analysis.py | 25 +++++++++++++------- ush/python/pygfs/task/snow_analysis.py | 7 ++---- 5 files changed, 55 insertions(+), 33 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index f7efd503d9e..a61b7c82f38 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -68,7 +68,6 @@ def initialize(self: Analysis) -> None: - staging B error files - staging model backgrounds - generating a YAML file for the JEDI executable - - linking JEDI variational executable - creating output directories """ super().initialize() @@ -107,14 +106,22 @@ def initialize(self: Analysis) -> None: @logit(logger) def execute(self: Analysis) -> None: - # link JEDI executable to run directory - self.task_config.jedi_exe = self.link_jediexe() + chdir(self.task_config.DATA) - # Run executable - self.execute_jediexe(self.runtime_config.DATA, - self.task_config.APRUN_AEROANL, - self.task_config.jedi_exe, - self.task_config.jedi_yaml) + exec_cmd = Executable(self.task_config.APRUN_AEROANL) + exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass @logit(logger) def finalize(self: Analysis) -> None: diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 90fb0dda3ea..078e013e7f8 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -43,6 +43,9 @@ def initialize(self) -> None: bias_dict = self.get_bias_dict() FileHandler(bias_dict).sync() + # link jedi executable to run directory + self.link_jediexe() + @logit(logger) def get_jedi_config(self) -> Dict[str, Any]: """Compile a dictionary of JEDI configuration from JEDIYAML template file diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index e3d2c519ed6..56e4ec2a449 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -10,7 +10,7 @@ from wxflow import (AttrDict, FileHandler, add_to_datetime, to_fv3time, to_timedelta, to_YMDH, - chdir, rm_p, + chdir, parse_j2yaml, save_as_yaml, logit, Executable, @@ -65,7 +65,7 @@ def initialize(self: Analysis) -> None: - staging FV3-JEDI fix files - staging B error files - staging model backgrounds - - generating a YAML file for the JEDI variational executable + - generating a YAML file for the JEDI executable - creating output directories """ super().initialize() @@ -115,18 +115,25 @@ def initialize(self: Analysis) -> None: @logit(logger) def variational(self: Analysis) -> None: - # Link JEDI executable - self.task_config.jedi_exe = self.link_jediexe() + chdir(self.task_config.DATA) - # Run executable - self.execute_jediexe(self.runtime_config.DATA, - self.task_config.APRUN_ATMANLVAR, - self.task_config.jedi_exe, - self.task_config.jedi_yaml) + exec_cmd = Executable(self.task_config.APRUN_ATMANL) + exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass @logit(logger) def fv3_increment(self: Analysis) -> None: - # Setup JEDI YAML file self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIYAML)) save_as_yaml(self.get_jedi_config(), self.task_config.jedi_yaml) @@ -149,7 +156,7 @@ def finalize(self: Analysis) -> None: - tar output diag files and place in ROTDIR - copy the generated YAML file from initialize to the ROTDIR - copy the updated bias correction files to ROTDIR - - copy UFS model readable atm FV3 increment file to ROTDIR + - write UFS model readable atm incrment file """ # ---- tar up diags @@ -218,6 +225,7 @@ def finalize(self: Analysis) -> None: FileHandler(bias_copy).sync() # Copy FV3 atm increment to comrot directory + logger.info("Copy UFS model readable atm increment file") cdate = to_fv3time(self.task_config.current_cycle) cdate_inc = cdate.replace('.', '_') src = os.path.join(self.task_config.DATA, 'anl', f"atminc.{cdate_inc}z.nc4") diff --git a/ush/python/pygfs/task/atmens_analysis.py b/ush/python/pygfs/task/atmens_analysis.py index 81c99b3755b..1037b557c24 100644 --- a/ush/python/pygfs/task/atmens_analysis.py +++ b/ush/python/pygfs/task/atmens_analysis.py @@ -145,15 +145,22 @@ def execute(self: Analysis) -> None: ---------- None """ - - # link JEDI executable to run directory - self.task_config.jedi_exe = self.link_jediexe() - - # Run executable - self.execute_jediexe(self.runtime_config.DATA, - self.task_config.APRUN_ATMENSANL, - self.task_config.jedi_exe, - self.task_config.jedi_yaml) + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_ATMENSANL) + exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_letkf.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.jedi_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass @logit(logger) def finalize(self: Analysis) -> None: diff --git a/ush/python/pygfs/task/snow_analysis.py b/ush/python/pygfs/task/snow_analysis.py index 81213163cd9..c149f140b61 100644 --- a/ush/python/pygfs/task/snow_analysis.py +++ b/ush/python/pygfs/task/snow_analysis.py @@ -294,15 +294,12 @@ def execute(self) -> None: Instance of the SnowAnalysis object """ - # link JEDI executable to run directory - self.task_config.jedi_exe = self.link_jediexe() - # create a temporary dict of all keys needed in this method localconf = AttrDict() keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_ATMOS_RESTART_PREV', 'COM_SNOW_ANALYSIS', 'APREFIX', 'SNOWDEPTHVAR', 'BESTDDEV', 'CASE', 'OCNRES', 'ntiles', - 'APRUN_SNOWANL', 'JEDIEXE', 'jedi_yaml', 'jedi_exe', + 'APRUN_SNOWANL', 'JEDIEXE', 'jedi_yaml', 'APPLY_INCR_NML_TMPL', 'APPLY_INCR_EXE', 'APRUN_APPLY_INCR'] for key in keys: localconf[key] = self.task_config[key] @@ -315,7 +312,7 @@ def execute(self) -> None: logger.info("Running JEDI LETKF") self.execute_jediexe(localconf.DATA, localconf.APRUN_SNOWANL, - localconf.jedi_exe, + os.path.basename(localconf.JEDIEXE), localconf.jedi_yaml) logger.info("Creating analysis from backgrounds and increments") From 2d757c0e773c8f34fa87204b0eb3e6ce63e14e7f Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 11 Apr 2024 11:02:39 -0500 Subject: [PATCH 20/28] Hopefully final changes and reversions --- parm/config/gfs/config.atmanl | 2 ++ parm/config/gfs/config.atmanlvar | 2 -- scripts/exglobal_atm_analysis_fv3_increment.py | 1 + ush/python/pygfs/task/atm_analysis.py | 6 ++++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/parm/config/gfs/config.atmanl b/parm/config/gfs/config.atmanl index 08af7205093..7cfd0cb47fb 100644 --- a/parm/config/gfs/config.atmanl +++ b/parm/config/gfs/config.atmanl @@ -28,4 +28,6 @@ export layout_y_atmanl=@LAYOUT_Y_ATMANL@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ +export JEDIEXE=${EXECgfs}/fv3jedi_var.x + echo "END: config.atmanl" diff --git a/parm/config/gfs/config.atmanlvar b/parm/config/gfs/config.atmanlvar index bc9e17e02aa..cbc0334a08c 100644 --- a/parm/config/gfs/config.atmanlvar +++ b/parm/config/gfs/config.atmanlvar @@ -8,6 +8,4 @@ echo "BEGIN: config.atmanlvar" # Get task specific resources . "${EXPDIR}/config.resources" atmanlvar -export JEDIEXE=${EXECgfs}/fv3jedi_var.x - echo "END: config.atmanlvar" diff --git a/scripts/exglobal_atm_analysis_fv3_increment.py b/scripts/exglobal_atm_analysis_fv3_increment.py index 04d42cd4d13..57f2e7c9ee0 100755 --- a/scripts/exglobal_atm_analysis_fv3_increment.py +++ b/scripts/exglobal_atm_analysis_fv3_increment.py @@ -19,4 +19,5 @@ # Instantiate the atm analysis task AtmAnl = AtmAnalysis(config) + AtmAnl.init_fv3_increment() AtmAnl.fv3_increment() diff --git a/ush/python/pygfs/task/atm_analysis.py b/ush/python/pygfs/task/atm_analysis.py index 56e4ec2a449..ebeb0c7ba6d 100644 --- a/ush/python/pygfs/task/atm_analysis.py +++ b/ush/python/pygfs/task/atm_analysis.py @@ -117,7 +117,7 @@ def variational(self: Analysis) -> None: chdir(self.task_config.DATA) - exec_cmd = Executable(self.task_config.APRUN_ATMANL) + exec_cmd = Executable(self.task_config.APRUN_ATMANLVAR) exec_name = os.path.join(self.task_config.DATA, 'fv3jedi_var.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg(self.task_config.jedi_yaml) @@ -133,7 +133,7 @@ def variational(self: Analysis) -> None: pass @logit(logger) - def fv3_increment(self: Analysis) -> None: + def init_fv3_increment(self: Analysis) -> None: # Setup JEDI YAML file self.task_config.jedi_yaml = os.path.join(self.runtime_config.DATA, os.path.basename(self.task_config.JEDIYAML)) save_as_yaml(self.get_jedi_config(), self.task_config.jedi_yaml) @@ -141,6 +141,8 @@ def fv3_increment(self: Analysis) -> None: # Link JEDI executable to run directory self.task_config.jedi_exe = self.link_jediexe() + @logit(logger) + def fv3_increment(self: Analysis) -> None: # Run executable self.execute_jediexe(self.runtime_config.DATA, self.task_config.APRUN_ATMANLFV3INC, From a4a8c7d9c869e530bbfca2f8581e6ea3a0d33779 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 12 Apr 2024 08:57:58 -0500 Subject: [PATCH 21/28] Fix calcanl_gfs.py to allow time dimension in increment. Also, update gsi_utils and gdasapp submodules --- sorc/gdas.cd | 2 +- sorc/gsi_utils.fd | 2 +- ush/calcanl_gfs.py | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b0ef0cea512..0d7f9852b36 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b0ef0cea512eecebf579d25123d90274d1202368 +Subproject commit 0d7f9852b362ce4c60bb6a406efcfc24c9ec0ca6 diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index 67b014d8d3e..a70ffee59b0 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit 67b014d8d3e5acc1d21aca15e3fe2d66d327a206 +Subproject commit a70ffee59b0c4e279d0d61908f593be67e29fb8b diff --git a/ush/calcanl_gfs.py b/ush/calcanl_gfs.py index 25bbae4bce6..0f2ebe9300e 100755 --- a/ush/calcanl_gfs.py +++ b/ush/calcanl_gfs.py @@ -12,12 +12,13 @@ from collections import OrderedDict import datetime +python2fortran_bool = {True: '.true.', False: '.false.'} # function to calculate analysis from a given increment file and background def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ComIn_Ges, GPrefix, FixDir, atmges_ens_mean, RunDir, NThreads, NEMSGet, IAUHrs, - ExecCMD, ExecCMDMPI, ExecAnl, ExecChgresInc, Cdump): + ExecCMD, ExecCMDMPI, ExecAnl, ExecChgresInc, Cdump, JEDI): print('calcanl_gfs beginning at: ', datetime.datetime.utcnow()) IAUHH = IAUHrs @@ -273,6 +274,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, "firstguess_filename": "'ges'", "increment_filename": "'inc.fullres'", "fhr": 6, + "jedi": python2fortran_bool[JEDI], } gsi_utils.write_nml(namelist, CalcAnlDir6 + '/calc_analysis.nml') @@ -311,6 +313,7 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, "firstguess_filename": "'ges.ensres'", "increment_filename": "'siginc.nc'", "fhr": fh, + "jedi": python2fortran_bool[JEDI], } gsi_utils.write_nml(namelist, CalcAnlDir6 + '/calc_analysis.nml') @@ -356,10 +359,11 @@ def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, NEMSGet = os.getenv('NEMSIOGET', 'nemsio_get') IAUHrs = list(map(int, os.getenv('IAUFHRS', '6').split(','))) Cdump = os.getenv('CDUMP', 'gdas') + JEDI = gsi_utils.isTrue(os.getenv('DO_JEDIATMVAR', 'YES')) print(locals()) calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ComIn_Ges, GPrefix, FixDir, atmges_ens_mean, RunDir, NThreads, NEMSGet, IAUHrs, ExecCMD, ExecCMDMPI, ExecAnl, ExecChgresInc, - Cdump) + Cdump, JEDI) From 90673bad14a5110fcfc5791aff917e39937c9e62 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Fri, 12 Apr 2024 09:05:34 -0500 Subject: [PATCH 22/28] Coding norms --- ush/calcanl_gfs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/calcanl_gfs.py b/ush/calcanl_gfs.py index 0f2ebe9300e..8c5c6434170 100755 --- a/ush/calcanl_gfs.py +++ b/ush/calcanl_gfs.py @@ -14,6 +14,7 @@ python2fortran_bool = {True: '.true.', False: '.false.'} + # function to calculate analysis from a given increment file and background def calcanl_gfs(DoIAU, l4DEnsVar, Write4Danl, ComOut, APrefix, ComIn_Ges, GPrefix, From cbedaef5d38731027d62157e3ffbc49284aab6dd Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 15 Apr 2024 14:27:03 +0000 Subject: [PATCH 23/28] Update GDASApp hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 0d7f9852b36..c2fdf1e652b 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 0d7f9852b362ce4c60bb6a406efcfc24c9ec0ca6 +Subproject commit c2fdf1e652b8bb367e1e33780acc8f94811a25f5 From 333a52d6e82176b74b2dfa2917ac896aeba60a23 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 15 Apr 2024 14:31:50 +0000 Subject: [PATCH 24/28] Add atmanlfv3inc to HERCULES.env --- env/HERCULES.env | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/env/HERCULES.env b/env/HERCULES.env index 69d415b6094..da5ad972f28 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -61,6 +61,14 @@ case ${step} in [[ ${NTHREADS_ATMANLVAR} -gt ${nth_max} ]] && export NTHREADS_ATMANLVAR=${nth_max} export APRUN_ATMANLVAR="${launcher} -n ${npe_atmanlvar} --cpus-per-task=${NTHREADS_ATMANLVAR}" ;; + "atmanlfv3inc") + + nth_max=$((npe_node_max / npe_node_atmanlfv3inc)) + + export NTHREADS_ATMANLFV3INC=${nth_atmanlfv3inc:-${nth_max}} + [[ ${NTHREADS_ATMANLFV3INC} -gt ${nth_max} ]] && export NTHREADS_ATMANLFV3INC=${nth_max} + export APRUN_ATMANLFV3INC="${launcher} -n ${npe_atmanlfv3inc} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" + ;; "atmensanlrun") nth_max=$((npe_node_max / npe_node_atmensanlrun)) From 44b9f33381ca96274ef08946ff2d02b7b142cc7a Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 15 Apr 2024 14:57:27 +0000 Subject: [PATCH 25/28] Update GSI Utils hash --- sorc/gsi_utils.fd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd index a70ffee59b0..68bc14d30b3 160000 --- a/sorc/gsi_utils.fd +++ b/sorc/gsi_utils.fd @@ -1 +1 @@ -Subproject commit a70ffee59b0c4e279d0d61908f593be67e29fb8b +Subproject commit 68bc14d30b3ca8f890f2761c8bdd0a3cea635cf1 From 06eca748e0f597b000c699d58d5e50fc6330a760 Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 15 Apr 2024 15:16:34 +0000 Subject: [PATCH 26/28] Make placeholder in AWSPW.env and CONTAINER.env for atmanlfv3inc job --- env/AWSPW.env | 2 +- env/CONTAINER.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/env/AWSPW.env b/env/AWSPW.env index f5473cf019e..5cf819ba2b3 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmensanlrun aeroanlrun snowanl" + echo "atmanlvar atmanlfv3inc atmensanlrun aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/CONTAINER.env b/env/CONTAINER.env index a6de7ecbe79..700460b7553 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlvar atmensanlrun aeroanlrun snowanl" + echo "atmanlvar atmanlfv3inc atmensanlrun aeroanlrun snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" From a1de80239373db049686dd33141a88a684c9e10d Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Thu, 18 Apr 2024 15:40:11 -0500 Subject: [PATCH 27/28] Update GDASApp hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c2fdf1e652b..5b464c6421a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c2fdf1e652b8bb367e1e33780acc8f94811a25f5 +Subproject commit 5b464c6421a72443e0b9dca4ab28ce07dd3c7ca7 From 5d98365f421b2e09c6274ec2bded9a2fa5c986be Mon Sep 17 00:00:00 2001 From: DavidNew-NOAA Date: Mon, 22 Apr 2024 10:50:25 -0500 Subject: [PATCH 28/28] Update GDASApp hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 5b464c6421a..fc62ef5f4dd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 5b464c6421a72443e0b9dca4ab28ce07dd3c7ca7 +Subproject commit fc62ef5f4ddcd2c13df515b593e5ce1c7b5dd10b