From 8766c0dddc94363f3f16446838d8385a5fd53713 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 17 Apr 2024 15:50:00 -0400 Subject: [PATCH 01/69] Change run to variational for aerosol analysis --- ...nalysis_run.py => exglobal_aero_analysis_variational.py} | 6 +++--- ush/python/pygfs/task/aero_analysis.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename scripts/{exglobal_aero_analysis_run.py => exglobal_aero_analysis_variational.py} (84%) diff --git a/scripts/exglobal_aero_analysis_run.py b/scripts/exglobal_aero_analysis_variational.py similarity index 84% rename from scripts/exglobal_aero_analysis_run.py rename to scripts/exglobal_aero_analysis_variational.py index 85f4b963a4f..dd5bb4f65a6 100755 --- a/scripts/exglobal_aero_analysis_run.py +++ b/scripts/exglobal_aero_analysis_variational.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# exglobal_aero_analysis_run.py +# exglobal_aero_analysis_variational.py # This script creates an AerosolAnalysis object -# and runs the execute method +# and runs the variational method # which executes the global aerosol variational analysis import os @@ -19,4 +19,4 @@ # Instantiate the aerosol analysis task AeroAnl = AerosolAnalysis(config) - AeroAnl.execute() + AeroAnl.variational() diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index a61b7c82f38..8fa2bff6c5b 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -104,7 +104,7 @@ def initialize(self: Analysis) -> None: FileHandler({'mkdir': newdirs}).sync() @logit(logger) - def execute(self: Analysis) -> None: + def variational(self: Analysis) -> None: chdir(self.task_config.DATA) From 34eb98a04cdf61b4cb87dec17a5ad81e9a8c44d2 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 17 Apr 2024 15:51:50 -0400 Subject: [PATCH 02/69] More places for run to var --- ...AL_AERO_ANALYSIS_RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} | 4 ++-- jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename jobs/{JGLOBAL_AERO_ANALYSIS_RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} (83%) rename jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} (92%) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_RUN b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL similarity index 83% rename from jobs/JGLOBAL_AERO_ANALYSIS_RUN rename to jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL index 43749b78c5f..290d7225dd4 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_RUN +++ b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar" ############################################## # Set variables used in the script @@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr ############################################################### # Run relevant script -EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py} +EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlvar.sh similarity index 92% rename from jobs/rocoto/aeroanlrun.sh rename to jobs/rocoto/aeroanlvar.sh index bcd86e3fbf6..5232cc63b0f 100755 --- a/jobs/rocoto/aeroanlrun.sh +++ b/jobs/rocoto/aeroanlvar.sh @@ -19,6 +19,6 @@ export PYTHONPATH ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN" +"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL" status=$? exit "${status}" From 4bfd5e599be6da06bae4a7e3d5a43379ac3e3fbe Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 17 Apr 2024 15:56:45 -0400 Subject: [PATCH 03/69] run to var with a find/replace --- env/AWSPW.env | 2 +- env/CONTAINER.env | 2 +- env/HERA.env | 4 ++-- env/HERCULES.env | 8 ++++---- env/JET.env | 10 +++++----- env/ORION.env | 10 +++++----- env/S4.env | 10 +++++----- env/WCOSS2.env | 10 +++++----- jobs/rocoto/aeroanlvar.sh | 2 +- parm/config/gfs/config.aeroanlrun | 11 ----------- parm/config/gfs/config.aeroanlvar | 11 +++++++++++ parm/config/gfs/config.resources | 16 ++++++++-------- workflow/applications/gfs_cycled.py | 4 ++-- workflow/rocoto/gfs_tasks.py | 10 +++++----- workflow/rocoto/tasks.py | 2 +- 15 files changed, 56 insertions(+), 56 deletions(-) delete mode 100644 parm/config/gfs/config.aeroanlrun create mode 100644 parm/config/gfs/config.aeroanlvar diff --git a/env/AWSPW.env b/env/AWSPW.env index 2dbba67eb31..f1674770146 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlvar snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/CONTAINER.env b/env/CONTAINER.env index bc2d64b4cef..6b0a1ba4f73 100755 --- a/env/CONTAINER.env +++ b/env/CONTAINER.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlvar snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" diff --git a/env/HERA.env b/env/HERA.env index f55434e8d9d..6a02ab4059a 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlvar snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -69,7 +69,7 @@ elif [[ "${step}" = "atmensanlrun" ]]; then [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" diff --git a/env/HERCULES.env b/env/HERCULES.env index 7d2aa5f8d0f..4abf06bcfbf 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -69,15 +69,15 @@ case ${step} in [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}" ;; - "aeroanlrun") + "aeroanlvar") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_aeroanlrun)) + nth_max=$((npe_node_max / npe_node_aeroanlvar)) - export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} + export NTHREADS_AEROANL=${nth_aeroanlvar:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} - export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" + export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar} --cpus-per-task=${NTHREADS_AEROANL}" ;; "snowanl") diff --git a/env/JET.env b/env/JET.env index df6666d8dc2..ea53407a237 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlvar snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -60,15 +60,15 @@ elif [[ "${step}" = "atmensanlrun" ]]; then [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} export APRUN_ATMENSANL="${launcher} ${npe_atmensanlrun}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_aeroanlrun)) + nth_max=$((npe_node_max / npe_node_aeroanlvar)) - export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} + export NTHREADS_AEROANL=${nth_aeroanlvar:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} - export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" + export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 17d0d24d978..82e28bd5f2b 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlvar snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -68,15 +68,15 @@ elif [[ "${step}" = "atmensanlrun" ]]; then [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_aeroanlrun)) + nth_max=$((npe_node_max / npe_node_aeroanlvar)) - export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} + export NTHREADS_AEROANL=${nth_aeroanlvar:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} - export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" + export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar} --cpus-per-task=${NTHREADS_AEROANL}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/S4.env b/env/S4.env index ab564eb9745..200639b303e 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlvar snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -60,15 +60,15 @@ elif [[ "${step}" = "atmensanlrun" ]]; then [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_aeroanlrun)) + nth_max=$((npe_node_max / npe_node_aeroanlvar)) - export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} + export NTHREADS_AEROANL=${nth_aeroanlvar:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} - export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" + export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 4533629edce..a0cd36205fa 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlrun snowanl" + echo "atmanlrun atmensanlrun aeroanlvar snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen esfc efcs epos" echo "postsnd awips gempak" @@ -54,15 +54,15 @@ elif [[ "${step}" = "atmensanlrun" ]]; then [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max} export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_aeroanlrun)) + nth_max=$((npe_node_max / npe_node_aeroanlvar)) - export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} + export NTHREADS_AEROANL=${nth_aeroanlvar:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} - export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun}" + export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/jobs/rocoto/aeroanlvar.sh b/jobs/rocoto/aeroanlvar.sh index 5232cc63b0f..cac7aa8b89c 100755 --- a/jobs/rocoto/aeroanlvar.sh +++ b/jobs/rocoto/aeroanlvar.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="aeroanlrun" +export job="aeroanlvar" export jobid="${job}.$$" ############################################################### diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun deleted file mode 100644 index 012e5b79f3b..00000000000 --- a/parm/config/gfs/config.aeroanlrun +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -x - -########## config.aeroanlrun ########## -# Aerosol Analysis specific - -echo "BEGIN: config.aeroanlrun" - -# Get task specific resources -source "${EXPDIR}/config.resources" aeroanlrun - -echo "END: config.aeroanlrun" diff --git a/parm/config/gfs/config.aeroanlvar b/parm/config/gfs/config.aeroanlvar new file mode 100644 index 00000000000..4282b6c840b --- /dev/null +++ b/parm/config/gfs/config.aeroanlvar @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlvar ########## +# Aerosol Analysis specific + +echo "BEGIN: config.aeroanlvar" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlvar + +echo "END: config.aeroanlvar" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 98fc3b26686..55f1d5c1853 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -13,7 +13,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlrun atmanlfinal" echo "atmensanlinit atmensanlrun atmensanlfinal" echo "snowanl" - echo "aeroanlinit aeroanlrun aeroanlfinal" + echo "aeroanlinit aeroanlvar aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -294,7 +294,7 @@ case ${step} in export memory_aeroanlinit="3072M" ;; - "aeroanlrun") + "aeroanlvar") case ${CASE} in "C768") layout_x=8 @@ -321,12 +321,12 @@ case ${step} in export layout_x export layout_y - export wtime_aeroanlrun="00:30:00" - export npe_aeroanlrun=$(( layout_x * layout_y * 6 )) - export npe_aeroanlrun_gfs=$(( layout_x * layout_y * 6 )) - export nth_aeroanlrun=1 - export nth_aeroanlrun_gfs=1 - export npe_node_aeroanlrun=$(( npe_node_max / nth_aeroanlrun )) + export wtime_aeroanlvar="00:30:00" + export npe_aeroanlvar=$(( layout_x * layout_y * 6 )) + export npe_aeroanlvar_gfs=$(( layout_x * layout_y * 6 )) + export nth_aeroanlvar=1 + export nth_aeroanlvar_gfs=1 + export npe_node_aeroanlvar=$(( npe_node_max / nth_aeroanlvar )) export is_exclusive=True ;; diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index fdb81205db4..470525d1b73 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + configs += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] @@ -151,7 +151,7 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] if self.do_aero: - gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 8789a418f98..0229c57125b 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -483,21 +483,21 @@ def aeroanlinit(self): return task - def aeroanlrun(self): + def aeroanlvar(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlinit'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('aeroanlrun') - task_name = f'{self.cdump}aeroanlrun' + resources = self.get_resource('aeroanlvar') + task_name = f'{self.cdump}aeroanlvar' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.cdump.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -510,7 +510,7 @@ def aeroanlrun(self): def aeroanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlvar'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index e917f2c642e..3d05de0a994 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -19,7 +19,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlrun', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From da333e9e04751b11cbc267514a79d940f715c199 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 17 Apr 2024 16:16:44 -0400 Subject: [PATCH 04/69] More changes needed; end of day commit --- env/HERA.env | 18 +++++++-- env/JET.env | 12 +++++- env/ORION.env | 12 +++++- env/S4.env | 12 +++++- env/WCOSS2.env | 10 +++++ jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX | 35 ++++++++++++++++++ jobs/rocoto/aeroanlgenb.sh | 24 ++++++++++++ parm/config/gfs/config.aeroanlgenb | 11 ++++++ parm/config/gfs/config.resources | 37 +++++++++++++++++++ ...exglobal_aero_analysis_generate_bmatrix.py | 25 +++++++++++++ 10 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX create mode 100644 jobs/rocoto/aeroanlgenb.sh create mode 100644 parm/config/gfs/config.aeroanlgenb create mode 100644 scripts/exglobal_aero_analysis_generate_bmatrix.py diff --git a/env/HERA.env b/env/HERA.env index 6a02ab4059a..54e60230f3b 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlvar snowanl" + echo "atmanlrun atmensanlrun aeroanlvar aeroanlgenb snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -73,11 +73,21 @@ elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - nth_max=$((npe_node_max / npe_node_aeroanlrun)) + nth_max=$((npe_node_max / npe_node_aeroanlvar)) - export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}} + export NTHREADS_AEROANL=${nth_aeroanlvar:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} - export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}" + export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar} --cpus-per-task=${NTHREADS_AEROANL}" + +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_aeroanlgenb)) + + export NTHREADS_AEROGENB=${nth_aeroanlgenb:-${nth_max}} + [[ ${NTHREADS_AEROGENB} -gt ${nth_max} ]] && export NTHREADS_AEROGENB=${nth_max} + export APRUN_AEROGENB="${launcher} -n ${npe_aeroanlgenb} --cpus-per-task=${NTHREADS_AEROGENB}" elif [[ "${step}" = "snowanl" ]]; then diff --git a/env/JET.env b/env/JET.env index ea53407a237..47ee385e50a 100755 --- a/env/JET.env +++ b/env/JET.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlvar snowanl" + echo "atmanlrun atmensanlrun aeroanlvar aeroanlgenb snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -70,6 +70,16 @@ elif [[ "${step}" = "aeroanlvar" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_aeroanlgenb)) + + export NTHREADS_AEROGENB=${nth_aeroanlgenb:-${nth_max}} + [[ ${NTHREADS_AEROGENB} -gt ${nth_max} ]] && export NTHREADS_AEROGENB=${nth_max} + export APRUN_AEROGENB="${launcher} -n ${npe_aeroanlgenb}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/ORION.env b/env/ORION.env index 82e28bd5f2b..660cc10052d 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlvar snowanl" + echo "atmanlrun atmensanlrun aeroanlvar aeroanlgenb snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -78,6 +78,16 @@ elif [[ "${step}" = "aeroanlvar" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_aeroanlgenb)) + + export NTHREADS_AEROGENB=${nth_aeroanlgenb:-${nth_max}} + [[ ${NTHREADS_AEROGENB} -gt ${nth_max} ]] && export NTHREADS_AEROGENB=${nth_max} + export APRUN_AEROGENB="${launcher} -n ${npe_aeroanlgenb} --cpus-per-task=${NTHREADS_AEROGENB}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/S4.env b/env/S4.env index 200639b303e..a000e32509e 100755 --- a/env/S4.env +++ b/env/S4.env @@ -4,7 +4,7 @@ if [[ $# -ne 1 ]]; then echo "Must specify an input argument to set runtime environment variables!" echo "argument can be any one of the following:" - echo "atmanlrun atmensanlrun aeroanlvar snowanl" + echo "atmanlrun atmensanlrun aeroanlvar aeroanlgenb snowanl" echo "anal sfcanl fcst post metp" echo "eobs eupd ecen efcs epos" echo "postsnd awips gempak" @@ -70,6 +70,16 @@ elif [[ "${step}" = "aeroanlvar" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_aeroanlgenb)) + + export NTHREADS_AEROGENB=${nth_aeroanlgenb:-${nth_max}} + [[ ${NTHREADS_AEROGENB} -gt ${nth_max} ]] && export NTHREADS_AEROGENB=${nth_max} + export APRUN_AEROGENB="${launcher} -n ${npe_aeroanlgenb}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index a0cd36205fa..5e87237202e 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -64,6 +64,16 @@ elif [[ "${step}" = "aeroanlvar" ]]; then [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_aeroanlgenb)) + + export NTHREADS_AEROGENB=${nth_aeroanlgenb:-${nth_max}} + [[ ${NTHREADS_AEROGENB} -gt ${nth_max} ]] && export NTHREADS_AEROGENB=${nth_max} + export APRUN_AEROGENB="${launcher} -n ${npe_aeroanlgenb}" + elif [[ "${step}" = "snowanl" ]]; then nth_max=$((npe_node_max / npe_node_snowanl)) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX new file mode 100644 index 00000000000..5439d3a4a3b --- /dev/null +++ b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX @@ -0,0 +1,35 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +export WIPE_DATA="NO" +export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_generate_bmatrix.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +exit 0 diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh new file mode 100644 index 00000000000..7478a0185f6 --- /dev/null +++ b/jobs/rocoto/aeroanlgenb.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="aeroanlvar" +export jobid="${job}.$$" + +############################################################### +# setup python path for workflow utilities and tasks +wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" +PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" +export PYTHONPATH + +############################################################### +# Execute the JJOB +"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX" +status=$? +exit "${status}" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb new file mode 100644 index 00000000000..9e855fbe6e6 --- /dev/null +++ b/parm/config/gfs/config.aeroanlgenb @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlgenb ########## +# Aerosol Variance specific + +echo "BEGIN: config.aeroanlgenb" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlgenb + +echo "END: config.aeroanlgenb" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 55f1d5c1853..860e89cdcfa 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -330,6 +330,43 @@ case ${step} in export is_exclusive=True ;; + "aeroanlgenb") + case ${CASE} in + "C768") + layout_x=8 + layout_y=8 + ;; + "C384") + layout_x=8 + layout_y=8 + ;; + "C192" | "C96") + layout_x=8 + layout_y=8 + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + export wtime_aeroanlgenb="00:30:00" + export npe_aeroanlgenb=$(( layout_x * layout_y * 6 )) + export npe_aeroanlgenb_gfs=$(( layout_x * layout_y * 6 )) + export nth_aeroanlgenb=1 + export nth_aeroanlgenb_gfs=1 + export npe_node_aeroanlgenb=$(( npe_node_max / nth_aeroanlgenb )) + export is_exclusive=True + ;; + + "aeroanlfinal") export wtime_aeroanlfinal="00:10:00" export npe_aeroanlfinal=1 diff --git a/scripts/exglobal_aero_analysis_generate_bmatrix.py b/scripts/exglobal_aero_analysis_generate_bmatrix.py new file mode 100644 index 00000000000..13bf174bda9 --- /dev/null +++ b/scripts/exglobal_aero_analysis_generate_bmatrix.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# exglobal_aero_analysis_generate_bmatrix.py +# This script creates an AerosolVariance object +# and runs the methods needed +# to stage files, compute the variance, and write to com +# files needed for the variational solver +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.aero_variance import AerosolVariance + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the aerosol variance tasks + AeroB = AerosolVariance(config) + AeroB.initialize() + AeroB.computeVariance() + AeroB.finalize() From a6645a701d003eb22ff9676eec46c09042ecbe9a Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 18 Apr 2024 10:35:29 -0400 Subject: [PATCH 05/69] Missed one --- env/HERCULES.env | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/env/HERCULES.env b/env/HERCULES.env index 4abf06bcfbf..d7a220297c9 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -78,6 +78,16 @@ case ${step} in export NTHREADS_AEROANL=${nth_aeroanlvar:-${nth_max}} [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max} export APRUN_AEROANL="${launcher} -n ${npe_aeroanlvar} --cpus-per-task=${NTHREADS_AEROANL}" + ;; + "aeroanlgenb") + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + nth_max=$((npe_node_max / npe_node_aeroanlgenb)) + + export NTHREADS_AEROGENB=${nth_aeroanlgenb:-${nth_max}} + [[ ${NTHREADS_AEROGENB} -gt ${nth_max} ]] && export NTHREADS_AEROGENB=${nth_max} + export APRUN_AEROGENB="${launcher} -n ${npe_aeroanlgenb} --cpus-per-task=${NTHREADS_AEROGENB}" ;; "snowanl") From 217db3b5d8d7a84f7b30cb6a9e7163f006c97a9a Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Thu, 18 Apr 2024 11:19:56 -0400 Subject: [PATCH 06/69] Save to start building and iterating on HPC --- ...exglobal_aero_analysis_generate_bmatrix.py | 6 ++--- ush/python/pygfs/task/aero_bmatrix.py | 21 ++++++++++++++++ ush/python/pygfs/task/bmatrix.py | 24 +++++++++++++++++++ 3 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 ush/python/pygfs/task/aero_bmatrix.py create mode 100644 ush/python/pygfs/task/bmatrix.py diff --git a/scripts/exglobal_aero_analysis_generate_bmatrix.py b/scripts/exglobal_aero_analysis_generate_bmatrix.py index 13bf174bda9..cd4a60166e9 100644 --- a/scripts/exglobal_aero_analysis_generate_bmatrix.py +++ b/scripts/exglobal_aero_analysis_generate_bmatrix.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 # exglobal_aero_analysis_generate_bmatrix.py -# This script creates an AerosolVariance object +# This script creates an AerosolBMatrix object # and runs the methods needed # to stage files, compute the variance, and write to com # files needed for the variational solver import os from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs.task.aero_variance import AerosolVariance +from pygfs.task.aero_bmatrix import AerosolBMatrix # Initialize root logger logger = Logger(level='DEBUG', colored_log=True) @@ -19,7 +19,7 @@ config = cast_strdict_as_dtypedict(os.environ) # Instantiate the aerosol variance tasks - AeroB = AerosolVariance(config) + AeroB = AerosolBMatrix(config) AeroB.initialize() AeroB.computeVariance() AeroB.finalize() diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py new file mode 100644 index 00000000000..2e4aec79934 --- /dev/null +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolBMatrix(BMatrix): + """ + Class for global aerosol BMatrix tasks + """ + @logit(logger, name="AerosolBMatrix") + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + @logit(logger) + def initialize(self: Analysis) -> None: + super().initialize() + diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py new file mode 100644 index 00000000000..68d92d84c1b --- /dev/null +++ b/ush/python/pygfs/task/bmatrix.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +logger = getLogger(__name__.split('.')[-1]) + + +class BMatrix(Task): + """Parent class for GDAS BMatrix tasks + + The BMatrix class is the parent class for all + Global Data Assimilation System (GDAS) BMatrix tasks + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + self.config.ntiles = 6 + # Store location of GDASApp jinja2 templates + self.gdasapp_j2tmpl_dir = os.path.join(self.config.PARMgfs, 'gdas') + + def initialize(self) -> None: + super().initialize() + From 520b3ac9f74cd5c0089d9bb01056b894d27ee2ea Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 18 Apr 2024 17:10:51 +0000 Subject: [PATCH 07/69] update gdas submodule and link new exe --- sorc/gdas.cd | 2 +- sorc/link_workflow.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 2198b419567..5b464c6421a 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 2198b419567cf7efa7404cd076e76e01d86f9e58 +Subproject commit 5b464c6421a72443e0b9dca4ab28ce07dd3c7ca7 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 45645aac9ba..aeb31640338 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -340,6 +340,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "fv3jedi_enshofx.x" \ "fv3jedi_hofx_nomodel.x" \ "fv3jedi_testdata_downloader.py" \ + "gdasapp_chem_diagb.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ "gdas_obsprovider2ioda.x" \ From 20885a53ae8173af16517b15d4f630a4b7d7e548 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 18 Apr 2024 17:29:25 +0000 Subject: [PATCH 08/69] commit before starting to build test experiment --- ush/python/pygfs/task/aero_bmatrix.py | 3 +++ ush/python/pygfs/task/bmatrix.py | 2 ++ workflow/applications/gfs_cycled.py | 4 ++-- workflow/rocoto/gfs_tasks.py | 31 +++++++++++++++++++++++++-- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 2e4aec79934..063f1c1f97c 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -19,3 +19,6 @@ def __init__(self, config: Dict[str, Any]) -> None: def initialize(self: Analysis) -> None: super().initialize() + @logit(logger) + def finalize(self) -> None: + super().finalize() diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index 68d92d84c1b..eff147edc02 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -22,3 +22,5 @@ def __init__(self, config: Dict[str, Any]) -> None: def initialize(self) -> None: super().initialize() + def finalize(self) -> None: + super().finalize() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 470525d1b73..07e298c46d4 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_jedisnowda: configs += ['prepsnowobs', 'snowanl'] @@ -151,7 +151,7 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] if self.do_aero: - gdas_gfs_common_tasks_before_fcst += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + gdas_gfs_common_tasks_before_fcst += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 0229c57125b..1ba0f9a6106 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -459,6 +459,30 @@ def atmanlfinal(self): return task + def aeroanlgenb(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('aeroanlgenb') + task_name = f'{self.cdump}aeroanlgenb' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.cdump.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def aeroanlinit(self): deps = [] @@ -486,9 +510,12 @@ def aeroanlinit(self): def aeroanlvar(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlinit'} + dep_dict = { + 'type': 'task', 'name': f'{self.cdump}aeroanlinit', + 'type': 'task', 'name': f'{self.cdump}aeroanlgenb', + } deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlvar') task_name = f'{self.cdump}aeroanlvar' From e872517625bf1bae05caaf4b8a0586603413301c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 18 Apr 2024 18:52:57 +0000 Subject: [PATCH 09/69] make executable --- jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX | 0 jobs/rocoto/aeroanlgenb.sh | 0 scripts/exglobal_aero_analysis_generate_bmatrix.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX mode change 100644 => 100755 jobs/rocoto/aeroanlgenb.sh mode change 100644 => 100755 scripts/exglobal_aero_analysis_generate_bmatrix.py diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX old mode 100644 new mode 100755 diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh old mode 100644 new mode 100755 diff --git a/scripts/exglobal_aero_analysis_generate_bmatrix.py b/scripts/exglobal_aero_analysis_generate_bmatrix.py old mode 100644 new mode 100755 From 50188016c2e732e69c2dada802b873fa4558566b Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 19 Apr 2024 17:02:10 +0000 Subject: [PATCH 10/69] Commit before some testing --- ush/python/pygfs/task/aero_bmatrix.py | 48 +++++++++++++++++++++++++++ ush/python/pygfs/task/bmatrix.py | 22 ++++++++++++ 2 files changed, 70 insertions(+) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 063f1c1f97c..8e73a6352aa 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -4,6 +4,9 @@ from logging import getLogger from typing import List, Dict, Any, Union +from wxflow import (AttrDict, + FileHandler) + logger = getLogger(__name__.split('.')[-1]) @@ -15,9 +18,54 @@ class AerosolBMatrix(BMatrix): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) + _res = int(self.config['CASE'][1:]) + _res_anl = int(self.config['CASE_ANL'][1:]) + + _diagb_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.config.LEVS - 1, + 'npz': self.config.LEVS - 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, + 'npz_anl': self.config['LEVS'] - 1, + 'AERO_WINDOW_BEGIN': _window_begin, + 'AERO_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", + 'aero_bkg_fhr': map(int, str(self.config['aero_bkg_times']).split(',')), + 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", + 'diagb_yaml': _diagb_yaml, + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + @logit(logger) def initialize(self: Analysis) -> None: super().initialize() + # stage fix files + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_list).sync() + + # stage berror files + # copy BUMP files, otherwise it will assume ID matrix + if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: + FileHandler(self.get_berror_dict(self.task_config)).sync() + + # stage backgrounds + FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + + # generate diagb YAML file + logger.debug(f"Generate diagb YAML file: {self.task_config.diagb_yaml}") + save_as_yaml(self.task_config.bmat_config, self.task_config.diagb_yaml) + logger.info(f"Wrote diagb YAML to: {self.task_config.diagb_yaml}") @logit(logger) def finalize(self) -> None: diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index eff147edc02..a4c69d24024 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -21,6 +21,28 @@ def __init__(self, config: Dict[str, Any]) -> None: def initialize(self) -> None: super().initialize() + # all BMatrix tasks need a config + self.task_config.bmat_config = self.get_bmat_config() def finalize(self) -> None: super().finalize() + + @logit(logger) + def get_bmat_config(self) -> Dict[str, Any]: + """Compile a dictionary of B Matrix configuration from BMATYAML template file + + Parameters + ---------- + + Returns + ---------- + bmat_config : Dict + a dictionary containing the fully rendered B matrix yaml configuration + """ + + # generate JEDI YAML file + logger.info(f"Generate B Matrix YAML config: {self.task_config.bmat_yaml}") + bmat_config = parse_j2yaml(self.task_config.BMATYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + logger.debug(f"BMAT config:\n{pformat(bmat_config)}") + + return bmat_config \ No newline at end of file From 0cc3ada43f09a723484574ad46286732b58c8376 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 19 Apr 2024 20:15:28 +0000 Subject: [PATCH 11/69] update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 5b464c6421a..6004fded318 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 5b464c6421a72443e0b9dca4ab28ce07dd3c7ca7 +Subproject commit 6004fded31828c2b03a9670f3608e73affe93ca9 From 6a24e3080e4696ab7480856b48dedf35ba207e48 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 19 Apr 2024 20:16:21 +0000 Subject: [PATCH 12/69] commit after debugging and testing --- jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX | 24 +++- jobs/rocoto/aeroanlgenb.sh | 2 +- parm/config/gfs/config.aeroanl | 1 + parm/config/gfs/config.aeroanlgenb | 2 + ush/python/pygfs/task/aero_bmatrix.py | 144 ++++++++++++++++++-- ush/python/pygfs/task/bmatrix.py | 4 + 6 files changed, 164 insertions(+), 13 deletions(-) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX index 5439d3a4a3b..07b72d16447 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX +++ b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX @@ -2,21 +2,35 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" -export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} +export DATA=${DATA:-${DATAROOT}/${RUN}aerogenb_${cyc}} source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" ############################################## # Set variables used in the script ############################################## +# shellcheck disable=SC2153 +GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +gPDY=${GDATE:0:8} +gcyc=${GDATE:8:2} +GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work ############################################## +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS + +RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ + COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + +mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" + ############################################################### # Run relevant script -EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_generate_bmatrix.py} +EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exglobal_aero_analysis_generate_bmatrix.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" @@ -32,4 +46,10 @@ if [[ -e "${pgmout}" ]] ; then cat "${pgmout}" fi +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" + exit 0 diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh index 7478a0185f6..dba6318e33b 100755 --- a/jobs/rocoto/aeroanlgenb.sh +++ b/jobs/rocoto/aeroanlgenb.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="aeroanlvar" +export job="aeroanlgenb" export jobid="${job}.$$" ############################################################### diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 972f393feb5..337c7c70135 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -19,6 +19,7 @@ export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/fv3jedi_var.x" +export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 9e855fbe6e6..8f397375d5e 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -8,4 +8,6 @@ echo "BEGIN: config.aeroanlgenb" # Get task specific resources source "${EXPDIR}/config.resources" aeroanlgenb +export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" + echo "END: config.aeroanlgenb" diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 8e73a6352aa..deb49cc4603 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -4,8 +4,11 @@ from logging import getLogger from typing import List, Dict, Any, Union -from wxflow import (AttrDict, - FileHandler) +from wxflow import (AttrDict, FileHandler, rm_p, + add_to_datetime, to_fv3time, to_timedelta, + to_fv3time, chdir, Executable, WorkflowException, + parse_j2yaml, save_as_yaml, logit) +from pygfs.task.bmatrix import BMatrix logger = getLogger(__name__.split('.')[-1]) @@ -21,7 +24,7 @@ def __init__(self, config: Dict[str, Any]) -> None: _res = int(self.config['CASE'][1:]) _res_anl = int(self.config['CASE_ANL'][1:]) - _diagb_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") + _bmat_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -33,13 +36,11 @@ def __init__(self, config: Dict[str, Any]) -> None: 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, 'npz_anl': self.config['LEVS'] - 1, - 'AERO_WINDOW_BEGIN': _window_begin, - 'AERO_WINDOW_LENGTH': f"PT{self.config['assim_freq']}H", 'aero_bkg_fhr': map(int, str(self.config['aero_bkg_times']).split(',')), 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", - 'diagb_yaml': _diagb_yaml, + 'bmat_yaml': _bmat_yaml, } ) @@ -47,7 +48,7 @@ def __init__(self, config: Dict[str, Any]) -> None: self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) @logit(logger) - def initialize(self: Analysis) -> None: + def initialize(self: BMatrix) -> None: super().initialize() # stage fix files logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") @@ -63,10 +64,133 @@ def initialize(self: Analysis) -> None: FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() # generate diagb YAML file - logger.debug(f"Generate diagb YAML file: {self.task_config.diagb_yaml}") - save_as_yaml(self.task_config.bmat_config, self.task_config.diagb_yaml) - logger.info(f"Wrote diagb YAML to: {self.task_config.diagb_yaml}") + logger.debug(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") + save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) + logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") + + # create output directory + FileHandler({'mkdir': [os.path.join(self.task_config['DATA'], 'stddev')]}).sync() + + # link executable to run directory + self.link_bmatexe() + + @logit(logger) + def computeVariance(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdasapp_chem_diagb.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.bmat_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + @logit(logger) def finalize(self) -> None: super().finalize() + # copy full YAML from executable to ROTDIR + src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") + yaml_copy = { + 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() + + # copy stddev files to ROTDIR + logger.info('Copying std. dev. files to ROTDIR') + template = f'{to_fv3time(self.task_config.current_cycle)}.stddev.fv_tracer.res.tile{{tilenum}}.nc' + inclist = [] + for itile in range(1, self.task_config.ntiles + 1): + tracer = template.format(tilenum=itile) + src = os.path.join(self.task_config.DATA, 'stddev', tracer) + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) + inclist.append([src, dest]) + FileHandler({'copy': inclist}).sync() + + @logit(logger) + def link_bmatexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.BMATEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + + @logit(logger) + def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: + """Compile a dictionary of model background files to copy + + This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer) + that are needed for global aerosol DA and returns said dictionary for use by the FileHandler class. + + Parameters + ---------- + task_config: Dict + a dictionary containing all of the configuration needed for the task + + Returns + ---------- + bkg_dict: Dict + a dictionary containing the list of model background files to copy for FileHandler + """ + # NOTE for now this is FV3 RESTART files and just assumed to be fh006 + + # get FV3 RESTART files, this will be a lot simpler when using history files + rst_dir = task_config.COM_ATMOS_RESTART_PREV + run_dir = os.path.join(task_config['DATA'], 'bkg') + + # Start accumulating list of background files to copy + bkglist = [] + + # if using IAU, we can use FGAT + bkgtimes = [] + begintime = task_config.previous_cycle + for fcsthr in task_config.aero_bkg_fhr: + bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) + + # now loop over background times + for bkgtime in bkgtimes: + # aerosol DA needs coupler + basename = f'{to_fv3time(bkgtime)}.coupler.res' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + # aerosol DA only needs core/tracer + for ftype in ['core', 'tracer']: + template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + bkg_dict = { + 'mkdir': [run_dir], + 'copy': bkglist, + } + return bkg_dict \ No newline at end of file diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index a4c69d24024..5155acb9919 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -3,6 +3,10 @@ import os from logging import getLogger from typing import List, Dict, Any, Union +from pprint import pformat + +from wxflow import (parse_j2yaml, FileHandler, logit, + Task, Executable, WorkflowException) logger = getLogger(__name__.split('.')[-1]) From 3df6b36a733a9187c8edf54c4e9f1a9eee1be032 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 19 Apr 2024 20:19:54 +0000 Subject: [PATCH 13/69] norm --- ush/python/pygfs/task/aero_bmatrix.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index deb49cc4603..aa9f97a9af3 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -22,7 +22,7 @@ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) _res = int(self.config['CASE'][1:]) - _res_anl = int(self.config['CASE_ANL'][1:]) + _res_anl = int(self.config['CASE_ANL'][1:]) _bmat_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") @@ -94,7 +94,6 @@ def computeVariance(self) -> None: pass - @logit(logger) def finalize(self) -> None: super().finalize() @@ -116,7 +115,7 @@ def finalize(self) -> None: src = os.path.join(self.task_config.DATA, 'stddev', tracer) dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + FileHandler({'copy': inclist}).sync() @logit(logger) def link_bmatexe(self) -> None: @@ -193,4 +192,4 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: 'mkdir': [run_dir], 'copy': bkglist, } - return bkg_dict \ No newline at end of file + return bkg_dict From cee3def4bd7e7bc91495be55a5bab061d492fda5 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 19 Apr 2024 20:22:16 +0000 Subject: [PATCH 14/69] norm was not happy --- ush/python/pygfs/task/aero_bmatrix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index aa9f97a9af3..c33527ddd86 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -193,3 +193,4 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: 'copy': bkglist, } return bkg_dict + From 69e9fa4201775d6f10405b84a20e12e4b758a496 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 19 Apr 2024 20:23:45 +0000 Subject: [PATCH 15/69] Relax Norm, it is Friday afternoon --- ush/python/pygfs/task/aero_bmatrix.py | 1 - ush/python/pygfs/task/bmatrix.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index c33527ddd86..aa9f97a9af3 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -193,4 +193,3 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: 'copy': bkglist, } return bkg_dict - diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index 5155acb9919..0c8696ff1a7 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -49,4 +49,4 @@ def get_bmat_config(self) -> Dict[str, Any]: bmat_config = parse_j2yaml(self.task_config.BMATYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) logger.debug(f"BMAT config:\n{pformat(bmat_config)}") - return bmat_config \ No newline at end of file + return bmat_config From 605fbff8bac2753e97de0c43fe4cab50f5834a6c Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 22 Apr 2024 15:41:50 +0000 Subject: [PATCH 16/69] more updates; looks like it is working --- ush/python/pygfs/task/aero_analysis.py | 13 +++++++++---- ush/python/pygfs/task/aero_bmatrix.py | 13 ++++++++++--- ush/python/pygfs/task/analysis.py | 2 +- workflow/rocoto/gfs_tasks.py | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 8fa2bff6c5b..e898985be86 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -286,19 +286,24 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. b_dir = config.BERROR_DATA_DIR b_datestr = to_fv3time(config.BERROR_DATE) + analysis_dir = config.COM_CHEM_ANALYSIS + cycle_datestr = to_fv3time(config.current_cycle) berror_list = [] + # the stddev is computed every cycle and is available in COM for ftype in ['stddev']: - coupler = f'{b_datestr}.{ftype}.coupler.res' + coupler = f'{cycle_datestr}.{ftype}.coupler.res' berror_list.append([ - os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) + os.path.join(analysis_dir, coupler), os.path.join(config.DATA, 'berror', coupler) ]) - template = f'{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' + template = f'{cycle_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' for itile in range(1, config.ntiles + 1): tracer = template.format(tilenum=itile) berror_list.append([ - os.path.join(b_dir, tracer), os.path.join(config.DATA, 'berror', tracer) + os.path.join(analysis_dir, tracer), os.path.join(config.DATA, 'berror', tracer) ]) + + # the remaining B matrix files are fixed and come from a fix directory radius = 'cor_aero_universe_radius' berror_list.append([ os.path.join(b_dir, radius), os.path.join(config.DATA, 'berror', radius) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index aa9f97a9af3..ac24470aaf7 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -109,13 +109,20 @@ def finalize(self) -> None: # copy stddev files to ROTDIR logger.info('Copying std. dev. files to ROTDIR') template = f'{to_fv3time(self.task_config.current_cycle)}.stddev.fv_tracer.res.tile{{tilenum}}.nc' - inclist = [] + stddevlist = [] + # actual std dev tracer files for itile in range(1, self.task_config.ntiles + 1): tracer = template.format(tilenum=itile) src = os.path.join(self.task_config.DATA, 'stddev', tracer) dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + stddevlist.append([src, dest]) + # coupler file + coupler = f'{to_fv3time(self.task_config.current_cycle)}.stddev.coupler.res' + src = os.path.join(self.task_config.DATA, 'stddev', coupler) + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, coupler) + stddevlist.append([src, dest]) + + FileHandler({'copy': stddevlist}).sync() @logit(logger) def link_bmatexe(self) -> None: diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 2221fb7b340..b20041cc77e 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -207,7 +207,7 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: @logit(logger) def link_jediexe(self) -> None: - """Compile a dictionary of background error files to copy + """ This method links a JEDI executable to the run directory diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 1ba0f9a6106..c2f06203c00 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -486,7 +486,7 @@ def aeroanlgenb(self): def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlgenb'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) From 67f4a2823fec65c06e8f7bb6410ce6153971aef3 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 22 Apr 2024 15:44:21 +0000 Subject: [PATCH 17/69] pynorms --- ush/python/pygfs/task/aero_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index e898985be86..40c76edfa5d 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -302,7 +302,7 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_list.append([ os.path.join(analysis_dir, tracer), os.path.join(config.DATA, 'berror', tracer) ]) - + # the remaining B matrix files are fixed and come from a fix directory radius = 'cor_aero_universe_radius' berror_list.append([ From 406cc4407f0694d809803e277b388e9b443ed3df Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 23 Apr 2024 20:30:09 +0000 Subject: [PATCH 18/69] more fixes --- ush/python/pygfs/task/aero_bmatrix.py | 5 ----- workflow/rocoto/gfs_tasks.py | 5 ++++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index ac24470aaf7..6a559168579 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -55,11 +55,6 @@ def initialize(self: BMatrix) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage berror files - # copy BUMP files, otherwise it will assume ID matrix - if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: - FileHandler(self.get_berror_dict(self.task_config)).sync() - # stage backgrounds FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index c2f06203c00..e3f073cc669 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -511,10 +511,13 @@ def aeroanlvar(self): deps = [] dep_dict = { - 'type': 'task', 'name': f'{self.cdump}aeroanlinit', 'type': 'task', 'name': f'{self.cdump}aeroanlgenb', } deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = { + 'type': 'task', 'name': f'{self.cdump}aeroanlinit', + } + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlvar') From e09656d0e326ddaae4f7f98fc6a2f1eb56ad27e9 Mon Sep 17 00:00:00 2001 From: Andrew Tangborn Date: Tue, 4 Jun 2024 09:38:32 -0500 Subject: [PATCH 19/69] Added tasks to construct aero_diffparm.yaml from template. --- parm/config/gfs/config.aeroanlgenb | 4 ++++ ush/python/pygfs/task/aero_bmatrix.py | 12 +++++++++++- ush/python/pygfs/task/bmatrix.py | 27 +++++++++++++++++++++++++-- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 8f397375d5e..7d7fe4a0e5a 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -9,5 +9,9 @@ echo "BEGIN: config.aeroanlgenb" source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" +export DIFFYAML="${PARMgfs}/gdas/aero/berror/aero_diffparm.yaml.j2" +export iterations=10 +export fixed value=1.0 + echo "END: config.aeroanlgenb" diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 6a559168579..7d45572831a 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -25,6 +25,7 @@ def __init__(self, config: Dict[str, Any]) -> None: _res_anl = int(self.config['CASE_ANL'][1:]) _bmat_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") + _diff_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diff.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -41,6 +42,7 @@ def __init__(self, config: Dict[str, Any]) -> None: 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", 'bmat_yaml': _bmat_yaml, + 'diff_yaml': _diff_yaml, } ) @@ -63,6 +65,11 @@ def initialize(self: BMatrix) -> None: save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") + # generate diffusion parameters YAML file + logger.debug(f"Generate diff YAML file: {self.task_config.diff_yaml}") + save_as_yaml(self.task_config.diff_config, self.task_config.diff_yaml) + logger.info(f"Wrote diff YAML to: {self.task_config.diff_yaml}") + # create output directory FileHandler({'mkdir': [os.path.join(self.task_config['DATA'], 'stddev')]}).sync() @@ -95,9 +102,12 @@ def finalize(self) -> None: # copy full YAML from executable to ROTDIR src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") + src_diff = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diff.yaml") + dest_diff = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diff.yaml") yaml_copy = { 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], - 'copy': [[src, dest]] + 'copy': [[src, dest]], + 'copy': [[src_diff, dest_diff]] } FileHandler(yaml_copy).sync() diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index 0c8696ff1a7..87b0517b706 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -27,6 +27,7 @@ def initialize(self) -> None: super().initialize() # all BMatrix tasks need a config self.task_config.bmat_config = self.get_bmat_config() + self.task_config.diff_config = self.get_diff_config() def finalize(self) -> None: super().finalize() @@ -44,9 +45,31 @@ def get_bmat_config(self) -> Dict[str, Any]: a dictionary containing the fully rendered B matrix yaml configuration """ - # generate JEDI YAML file + # generate JEDI YAML file for stddev logger.info(f"Generate B Matrix YAML config: {self.task_config.bmat_yaml}") bmat_config = parse_j2yaml(self.task_config.BMATYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) logger.debug(f"BMAT config:\n{pformat(bmat_config)}") - return bmat_config + return bmat_config + + def get_diff_config(self) -> Dict[str, Any]: + """Compile a dictionary of diffusion operator configuration from DIFFYAML template file + + Parameters + ---------- + diff_iter + fixed_val + horiz_len + + Returns + ---------- + diff_config : Dict + a dictionary containing the fully rendered diffusion operator yaml configuration + """ + + # generate JEDI YAML file for diffusion parameters + logger.info(f"Generate Diff Parmameter YAML config: {self.task_config.diff_yaml}") + diff_config = parse_j2yaml(self.task_config.DIFFYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + logger.debug(f"DIFF config:\n{pformat(diff_config)}") + + return diff_config From 62457eb6deb07fc25db4cc55099f68c0b85281cc Mon Sep 17 00:00:00 2001 From: Andrew Tangborn Date: Tue, 4 Jun 2024 16:00:14 -0500 Subject: [PATCH 20/69] changed DIFF -> DIFFUSION and diff -> diffusion in files below --- parm/config/gfs/config.aeroanlgenb | 2 +- ush/python/pygfs/task/aero_bmatrix.py | 16 ++++++++-------- ush/python/pygfs/task/bmatrix.py | 20 ++++++++++---------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 7d7fe4a0e5a..c6d07b1dd38 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -9,7 +9,7 @@ echo "BEGIN: config.aeroanlgenb" source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" -export DIFFYAML="${PARMgfs}/gdas/aero/berror/aero_diffparm.yaml.j2" +export DIFFFFYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" export iterations=10 export fixed value=1.0 diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 7d45572831a..436142abc1b 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -25,7 +25,7 @@ def __init__(self, config: Dict[str, Any]) -> None: _res_anl = int(self.config['CASE_ANL'][1:]) _bmat_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") - _diff_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diff.yaml") + _diffusion_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diffusion.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -42,7 +42,7 @@ def __init__(self, config: Dict[str, Any]) -> None: 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", 'bmat_yaml': _bmat_yaml, - 'diff_yaml': _diff_yaml, + 'diffusion_yaml': _diffusion_yaml, } ) @@ -66,9 +66,9 @@ def initialize(self: BMatrix) -> None: logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") # generate diffusion parameters YAML file - logger.debug(f"Generate diff YAML file: {self.task_config.diff_yaml}") - save_as_yaml(self.task_config.diff_config, self.task_config.diff_yaml) - logger.info(f"Wrote diff YAML to: {self.task_config.diff_yaml}") + logger.debug(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") + save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) + logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") # create output directory FileHandler({'mkdir': [os.path.join(self.task_config['DATA'], 'stddev')]}).sync() @@ -102,12 +102,12 @@ def finalize(self) -> None: # copy full YAML from executable to ROTDIR src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") - src_diff = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diff.yaml") - dest_diff = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diff.yaml") + src_diffusion = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diffusion.yaml") + dest_diffusion = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diffusion.yaml") yaml_copy = { 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], 'copy': [[src, dest]], - 'copy': [[src_diff, dest_diff]] + 'copy': [[src_diffusion, dest_diffusion]] } FileHandler(yaml_copy).sync() diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index 87b0517b706..59b4683ff38 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -27,7 +27,7 @@ def initialize(self) -> None: super().initialize() # all BMatrix tasks need a config self.task_config.bmat_config = self.get_bmat_config() - self.task_config.diff_config = self.get_diff_config() + self.task_config.diffusion_config = self.get_diffusion_config() def finalize(self) -> None: super().finalize() @@ -50,26 +50,26 @@ def get_bmat_config(self) -> Dict[str, Any]: bmat_config = parse_j2yaml(self.task_config.BMATYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) logger.debug(f"BMAT config:\n{pformat(bmat_config)}") - return bmat_config + return bmat_config - def get_diff_config(self) -> Dict[str, Any]: - """Compile a dictionary of diffusion operator configuration from DIFFYAML template file + def get_diffusion_config(self) -> Dict[str, Any]: + """Compile a dictionary of diffusion operator configuration from DIFFUSIONYAML template file Parameters ---------- - diff_iter + diffusion_iter fixed_val horiz_len Returns ---------- - diff_config : Dict + diffusion_config : Dict a dictionary containing the fully rendered diffusion operator yaml configuration """ # generate JEDI YAML file for diffusion parameters - logger.info(f"Generate Diff Parmameter YAML config: {self.task_config.diff_yaml}") - diff_config = parse_j2yaml(self.task_config.DIFFYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - logger.debug(f"DIFF config:\n{pformat(diff_config)}") + logger.info(f"Generate Diff Parmameter YAML config: {self.task_config.diffusion_yaml}") + diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) + logger.debug(f"DIFFUSION config:\n{pformat(diffusion_config)}") - return diff_config + return diffusion_config From 17b6dcc3439fe18ca0590d4030d06ff346c13bc0 Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 5 Jun 2024 15:19:46 -0400 Subject: [PATCH 21/69] Update parm/config/gfs/config.aeroanlgenb --- parm/config/gfs/config.aeroanlgenb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index c6d07b1dd38..6a2139ad016 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -9,7 +9,7 @@ echo "BEGIN: config.aeroanlgenb" source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" -export DIFFFFYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" +export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" export iterations=10 export fixed value=1.0 From 5f3297629620bdb9a0782c11d8ad37a89a79321f Mon Sep 17 00:00:00 2001 From: "Andrew.Tangborn" Date: Mon, 10 Jun 2024 17:43:21 +0000 Subject: [PATCH 22/69] Additions for running diffusion operator. --- parm/config/gfs/config.aeroanl | 1 + parm/config/gfs/config.aeroanlgenb | 5 +- parm/ufs/gocart/ExtData.other | 20 +++--- ...exglobal_aero_analysis_generate_bmatrix.py | 3 +- ush/python/pygfs/task/aero_analysis.py | 18 +++--- ush/python/pygfs/task/aero_bmatrix.py | 62 ++++++++++++++++++- 6 files changed, 83 insertions(+), 26 deletions(-) diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 52bc14c9e3d..9fb25fbf41d 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -20,6 +20,7 @@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" +export DIFFUSIONEXE="${EXECgfs}/fv3jedi_error_covariance_toolbox.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 6a2139ad016..90f8c695577 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -10,8 +10,9 @@ source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" -export iterations=10 -export fixed value=1.0 +export diff_iter=10 +export horiz_len=2500e3 +export fixed_val=1.0 echo "END: config.aeroanlgenb" diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 7a0d63d6ca2..5d2ddc51021 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL1 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO4 NA Y Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -63,11 +63,11 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC1 NA Y Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC2 NA Y Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null @@ -88,11 +88,11 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC1 NA Y Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC2 NA Y Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null diff --git a/scripts/exglobal_aero_analysis_generate_bmatrix.py b/scripts/exglobal_aero_analysis_generate_bmatrix.py index cd4a60166e9..19d5454ba02 100755 --- a/scripts/exglobal_aero_analysis_generate_bmatrix.py +++ b/scripts/exglobal_aero_analysis_generate_bmatrix.py @@ -18,8 +18,9 @@ # Take configuration from environment and cast it as python dictionary config = cast_strdict_as_dtypedict(os.environ) - # Instantiate the aerosol variance tasks + # Instantiate the aerosol variance and diffusion correlation tasks AeroB = AerosolBMatrix(config) AeroB.initialize() AeroB.computeVariance() + AeroB.computeDiffusion() AeroB.finalize() diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 430f7f34fc1..e43746017b0 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -305,17 +305,13 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: os.path.join(analysis_dir, tracer), os.path.join(config.DATA, 'berror', tracer) ]) - # the remaining B matrix files are fixed and come from a fix directory - radius = 'cor_aero_universe_radius' - berror_list.append([ - os.path.join(b_dir, radius), os.path.join(config.DATA, 'berror', radius) - ]) - nproc = config.ntiles * config.layout_x * config.layout_y - for nn in range(1, nproc + 1): - berror_list.append([ - os.path.join(b_dir, f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc'), - os.path.join(config.DATA, 'berror', f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc') - ]) + # the diffusion correlation files are computed every cycle and are available in COM + diff_hz = 'diffusion_hz.nc' + diff_vt = 'diffusion_vt.nc' + berror_list.append([ + os.path.join(b_dir, diff_hz), os.path.join(config.DATA, 'berror',diff_hz) + os.path.join(b_dir, diff_vt), os.path.join(config.DATA, 'berror',diff_vt) + berror_dict = { 'mkdir': [os.path.join(config.DATA, 'berror')], 'copy': berror_list, diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 436142abc1b..083e5d51c1d 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -61,20 +61,26 @@ def initialize(self: BMatrix) -> None: FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() # generate diagb YAML file - logger.debug(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") + logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") # generate diffusion parameters YAML file - logger.debug(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") + logger.info(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") # create output directory FileHandler({'mkdir': [os.path.join(self.task_config['DATA'], 'stddev')]}).sync() + # create diffusion output directory + FileHandler({'mkdir': [os.path.join(self.task_config['DATA'], 'diffusion')]}).sync() + # link executable to run directory + logger.info(f'before link_bmatexe') self.link_bmatexe() + logger.info(f'before link_diffusion_exe') + self.link_diffusion_exe() @logit(logger) def computeVariance(self) -> None: @@ -96,6 +102,26 @@ def computeVariance(self) -> None: pass + @logit(logger) + def computeDiffusion(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROGENB) + exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') + exec_cmd_diffusion.add_default_arg(exec_name_diffusion) + exec_cmd_diffusion.add_default_arg(self.task.config.diffusion_yaml) + + try: + logger.debug(f"Executing {exec_cmd_diffusion}") + exec_cmd_diffusion() + except OSError: + raise OSError(f"Failed to execute {exec_cmd_diffusion}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd_diffusion}") + + pass + @logit(logger) def finalize(self) -> None: super().finalize() @@ -128,6 +154,8 @@ def finalize(self) -> None: stddevlist.append([src, dest]) FileHandler({'copy': stddevlist}).sync() + # Diffusion files + @logit(logger) def link_bmatexe(self) -> None: @@ -155,6 +183,36 @@ def link_bmatexe(self) -> None: return + @logit(logger) + def link_diffusion_exe(self) -> None: + """ + + This method links a JEDI (fv3jedi_error_covariance_toolbox.x) + executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + + exe_src_diffusion = self.task_config.DIFFUSIONEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src_diffusion} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest_diffusion = os.path.join(self.task_config.DATA, os.path.basename(exe_src_diffusion)) + if os.path.exists(exe_dest_diffusion): + rm_p(exe_dest_diffusion) + os.symlink(exe_src_diffusion, exe_dest_diffusion) + + return + + + @logit(logger) def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: """Compile a dictionary of model background files to copy From 228ed0ad4df4dc5af686df1ffa247c9e80711e57 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Wed, 12 Jun 2024 18:42:48 +0000 Subject: [PATCH 23/69] link gdas_fv3jedi_error_covariance_toolbox.x --- parm/config/gfs/config.aeroanl | 2 +- sorc/link_workflow.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 9fb25fbf41d..ecbdc485f29 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -20,7 +20,7 @@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" -export DIFFUSIONEXE="${EXECgfs}/fv3jedi_error_covariance_toolbox.x" +export DIFFUSIONEXE="${EXECgfs}/gdas_fv3jedi_error_covariance_toolbox.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 2de15aaa6e7..98c01ad689b 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -342,6 +342,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then declare -a JEDI_EXE=("gdas.x" \ "gdas_soca_gridgen.x" \ "gdas_soca_error_covariance_toolbox.x" \ + "gdas_fv3jedi_error_covariance_toolbox.x" \ "gdas_soca_setcorscales.x" \ "fv3jedi_plot_field.x" \ "gdasapp_chem_diagb.x" \ From a1530f76cd51eb8d4c5ada6d0420f37909afbb34 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Wed, 12 Jun 2024 19:07:06 +0000 Subject: [PATCH 24/69] save diffusion files and aeroanlgenb-related yamls to COM_CHEM_ANALYSIS --- ush/python/pygfs/task/aero_analysis.py | 10 ++++++---- ush/python/pygfs/task/aero_bmatrix.py | 19 ++++++++++++------- workflow/applications/gfs_cycled.py | 7 ++----- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index e43746017b0..f9cddfe76cb 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -286,7 +286,6 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: a dictionary containing the list of background error files to copy for FileHandler """ # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. - b_dir = config.BERROR_DATA_DIR b_datestr = to_fv3time(config.BERROR_DATE) analysis_dir = config.COM_CHEM_ANALYSIS cycle_datestr = to_fv3time(config.current_cycle) @@ -308,9 +307,12 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: # the diffusion correlation files are computed every cycle and are available in COM diff_hz = 'diffusion_hz.nc' diff_vt = 'diffusion_vt.nc' - berror_list.append([ - os.path.join(b_dir, diff_hz), os.path.join(config.DATA, 'berror',diff_hz) - os.path.join(b_dir, diff_vt), os.path.join(config.DATA, 'berror',diff_vt) + berror_list.append([ + os.path.join(config.COM_CHEM_ANALYSIS, diff_hz), os.path.join(config.DATA, 'berror', diff_hz) + ]) + berror_list.append([ + os.path.join(config.COM_CHEM_ANALYSIS, diff_vt), os.path.join(config.DATA, 'berror', diff_vt) + ]) berror_dict = { 'mkdir': [os.path.join(config.DATA, 'berror')], diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 083e5d51c1d..fea9770ae3b 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -110,7 +110,7 @@ def computeDiffusion(self) -> None: exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROGENB) exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') exec_cmd_diffusion.add_default_arg(exec_name_diffusion) - exec_cmd_diffusion.add_default_arg(self.task.config.diffusion_yaml) + exec_cmd_diffusion.add_default_arg(self.task_config.diffusion_yaml) try: logger.debug(f"Executing {exec_cmd_diffusion}") @@ -132,8 +132,7 @@ def finalize(self) -> None: dest_diffusion = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diffusion.yaml") yaml_copy = { 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], - 'copy': [[src, dest]], - 'copy': [[src_diffusion, dest_diffusion]] + 'copy': [[src, dest], [src_diffusion, dest_diffusion]] } FileHandler(yaml_copy).sync() @@ -154,8 +153,16 @@ def finalize(self) -> None: stddevlist.append([src, dest]) FileHandler({'copy': stddevlist}).sync() - # Diffusion files + # Diffusion files + diff_hz = 'diffusion_hz.nc' + diff_vt = 'diffusion_vt.nc' + src_hz = os.path.join(self.task_config.DATA, diff_hz) + dest_hz = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_hz) + src_vt = os.path.join(self.task_config.DATA, diff_vt) + dest_vt = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_vt) + difflist = [[src_hz, dest_hz], [src_vt, dest_vt]] + FileHandler({'copy': difflist}).sync() @logit(logger) def link_bmatexe(self) -> None: @@ -187,7 +194,7 @@ def link_bmatexe(self) -> None: def link_diffusion_exe(self) -> None: """ - This method links a JEDI (fv3jedi_error_covariance_toolbox.x) + This method links a JEDI (fv3jedi_error_covariance_toolbox.x) executable to the run directory Parameters @@ -211,8 +218,6 @@ def link_diffusion_exe(self) -> None: return - - @logit(logger) def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: """Compile a dictionary of model background files to copy diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 65cc4b4b7e5..95b955400aa 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -150,9 +150,6 @@ def get_task_names(self): gdas_gfs_common_tasks_before_fcst += ['sfcanl', 'analcalc'] - if self.do_aero: - gdas_gfs_common_tasks_before_fcst += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] - if self.do_jedisnowda: gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl'] @@ -180,7 +177,7 @@ def get_task_names(self): gdas_tasks += wave_prep_tasks if self.do_aero and 'gdas' in self.aero_anl_cdumps: - gdas_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] gdas_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] @@ -217,7 +214,7 @@ def get_task_names(self): gfs_tasks += wave_prep_tasks if self.do_aero and 'gfs' in self.aero_anl_cdumps: - gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + gfs_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] gfs_tasks += ['atmanlupp', 'atmanlprod', 'fcst'] From b2e250f4fb65dcd85647eb11c0bd473267d67054 Mon Sep 17 00:00:00 2001 From: "Andrew.Tangborn" Date: Tue, 25 Jun 2024 16:22:12 +0000 Subject: [PATCH 25/69] Changes to workflow ush to enable execution of diffusion correlation parameters. --- ush/python/pygfs/task/aero_analysis.py | 2 ++ ush/python/pygfs/task/aero_bmatrix.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index f9cddfe76cb..0c33122f0a5 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -86,6 +86,8 @@ def initialize(self: Analysis) -> None: # copy BUMP files, otherwise it will assume ID matrix if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: FileHandler(self.get_berror_dict(self.task_config)).sync() + if self.task_config.get('STATICB_TYPE', 'identity') in ['diffusion']: + FileHandler(self.get_berror_dict(self.task_config)).sync() # stage backgrounds FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index fea9770ae3b..8f9f6277d56 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -157,9 +157,9 @@ def finalize(self) -> None: # Diffusion files diff_hz = 'diffusion_hz.nc' diff_vt = 'diffusion_vt.nc' - src_hz = os.path.join(self.task_config.DATA, diff_hz) + src_hz = os.path.join(self.task_config.DATA, 'diffusion', diff_hz) dest_hz = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_hz) - src_vt = os.path.join(self.task_config.DATA, diff_vt) + src_vt = os.path.join(self.task_config.DATA, 'diffusion',diff_vt) dest_vt = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_vt) difflist = [[src_hz, dest_hz], [src_vt, dest_vt]] FileHandler({'copy': difflist}).sync() From ff01c53617d4caa1f059daa6029929a823f97658 Mon Sep 17 00:00:00 2001 From: "Andrew.Tangborn" Date: Tue, 25 Jun 2024 21:12:24 +0000 Subject: [PATCH 26/69] Fixed coding norm. --- ush/python/pygfs/task/aero_bmatrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 8f9f6277d56..4a5726ecdfb 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -159,7 +159,7 @@ def finalize(self) -> None: diff_vt = 'diffusion_vt.nc' src_hz = os.path.join(self.task_config.DATA, 'diffusion', diff_hz) dest_hz = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_hz) - src_vt = os.path.join(self.task_config.DATA, 'diffusion',diff_vt) + src_vt = os.path.join(self.task_config.DATA, 'diffusion', diff_vt) dest_vt = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_vt) difflist = [[src_hz, dest_hz], [src_vt, dest_vt]] FileHandler({'copy': difflist}).sync() From 18c86ae0db126e5f07ffa5fdc1f8539db289254c Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Wed, 26 Jun 2024 16:30:07 -0400 Subject: [PATCH 27/69] Update config.aeroanlgenb --- parm/config/gfs/config.aeroanlgenb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 90f8c695577..706b6314c8c 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -10,9 +10,9 @@ source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" -export diff_iter=10 -export horiz_len=2500e3 -export fixed_val=1.0 +export aero_diffusion_iter=10 +export aero_diffusion_horiz_len=2500e3 +export aero_diffusion_fixed_val=1.0 echo "END: config.aeroanlgenb" From 08290228123c747a38044301ff54e84f8ed64589 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 26 Jun 2024 20:52:12 +0000 Subject: [PATCH 28/69] update gdas to develop --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 368c9c5db9b..2daa9fd0514 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 368c9c5db9b5ea62e72937b6d1b0f753adb9be40 +Subproject commit 2daa9fd0514bd164433dcd1b7ef9796329163bc8 From eaf71c9fe720964eff979ed36e73376f99be89fc Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 28 Jun 2024 15:37:12 +0000 Subject: [PATCH 29/69] cleanup thing from merge --- parm/config/gfs/config.resources | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index e42aa4a709b..6ab77f15fd1 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -445,17 +445,17 @@ case ${step} in export npe_aeroanlvar_gfs=$(( layout_x * layout_y * 6 )) export nth_aeroanlvar_gdas=1 export nth_aeroanlvar_gfs=1 - export npe_node_aeroanlvar_gdas=$(( npe_node_max / nth_aeroanlrun_gdas )) - export npe_node_aeroanlrun_gfs=$(( npe_node_max / nth_aeroanlvar_gfs )) + export npe_node_aeroanlvar_gdas=$(( npe_node_max / nth_aeroanlvar_gdas )) + export npe_node_aeroanlvar_gfs=$(( npe_node_max / nth_aeroanlvar_gfs )) export is_exclusive=True - var_npe_node="npe_node_aeroanlrun_${RUN}" - var_nth="nth_aeroanlrun_${RUN}" - var_npe="npe_aeroanlrun_${RUN}" + var_npe_node="npe_node_aeroanlvar_${RUN}" + var_nth="nth_aeroanlvar_${RUN}" + var_npe="npe_aeroanlvar_${RUN}" if [[ -n "${!var_npe_node+0}" ]]; then - declare -x "npe_node_aeroanlrun"="${!var_npe_node}" \ - "nth_aeroanlrun"="${!var_nth}" \ - "npe_aeroanlrun"="${!var_npe}" + declare -x "npe_node_aeroanlvar"="${!var_npe_node}" \ + "nth_aeroanlvar"="${!var_nth}" \ + "npe_aeroanlvar"="${!var_npe}" fi ;; From 93097751c0c10a3a6e983ef06d1ebfe274d96470 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 28 Jun 2024 17:30:12 +0000 Subject: [PATCH 30/69] make default aero DA B option - diffusion --- parm/config/gfs/config.aeroanl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index ecbdc485f29..ec94ace830d 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -7,7 +7,7 @@ echo "BEGIN: config.aeroanl" export CASE_ANL=${CASE} export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" -export STATICB_TYPE='identity' +export STATICB_TYPE='diffusion' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" export BERROR_DATE="20160630.000000" From dcdfee8dc5fe2514f8da9d1d4adcec681750fcd9 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 28 Jun 2024 19:40:24 +0000 Subject: [PATCH 31/69] First friday commit --- jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX | 20 ++++++----------- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 10 +++++---- parm/config/gfs/config.com | 1 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/aero_analysis.py | 24 ++++++++++----------- workflow/applications/gfs_cycled.py | 2 +- workflow/rocoto/gfs_tasks.py | 7 +++--- 7 files changed, 30 insertions(+), 36 deletions(-) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX index 07b72d16447..03120c885dc 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX +++ b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX @@ -1,31 +1,23 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" -export WIPE_DATA="NO" -export DATA=${DATA:-${DATAROOT}/${RUN}aerogenb_${cyc}} source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" ############################################## # Set variables used in the script ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" ############################################## # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +mkdir -p "${COMOUT_CHEM_BMAT}" ############################################################### # Run relevant script @@ -50,6 +42,6 @@ fi # Remove the Temporary working directory ########################################## cd "${DATAROOT}" || exit 1 -[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}" +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" exit 0 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index b2a2893bc04..600c6a19280 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -19,13 +19,15 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + COMIN_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ + COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +mkdir -p "${COMOUT_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index ec867e64ba0..20a84faac6a 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -96,5 +96,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' +declare -rx COM_CHEM_BMAT_TMPL=${COM_BASE}'/bmatrix/chem' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 2daa9fd0514..f479a40a3ae 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 2daa9fd0514bd164433dcd1b7ef9796329163bc8 +Subproject commit f479a40a3ae8002520241bf2218c722f005413b2 diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 0c33122f0a5..9d295cb2bbe 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -84,8 +84,6 @@ def initialize(self: Analysis) -> None: # stage berror files # copy BUMP files, otherwise it will assume ID matrix - if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: - FileHandler(self.get_berror_dict(self.task_config)).sync() if self.task_config.get('STATICB_TYPE', 'identity') in ['diffusion']: FileHandler(self.get_berror_dict(self.task_config)).sync() @@ -142,7 +140,7 @@ def finalize(self: Analysis) -> None: """ # ---- tar up diags # path of output tar statfile - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") # get list of diag files to put in tarball diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) @@ -160,9 +158,9 @@ def finalize(self: Analysis) -> None: # copy full YAML from executable to ROTDIR src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml") + dest = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.aerovar.yaml") yaml_copy = { - 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], + 'mkdir': [self.task_config.COMOUT_CHEM_ANALYSIS], 'copy': [[src, dest]] } FileHandler(yaml_copy).sync() @@ -176,8 +174,8 @@ def finalize(self: Analysis) -> None: bkglist = [] for itile in range(1, self.task_config.ntiles + 1): tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}') + src = os.path.join(self.task_config.COMIN_ATMOS_RESTART_PREV, tracer) + dest = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f'aeroges.{tracer}') bkglist.append([src, dest]) FileHandler({'copy': bkglist}).sync() @@ -192,7 +190,7 @@ def finalize(self: Analysis) -> None: for itile in range(1, self.task_config.ntiles + 1): tracer = template.format(tilenum=itile) src = os.path.join(self.task_config.DATA, 'anl', tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) + dest = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, tracer) inclist.append([src, dest]) FileHandler({'copy': inclist}).sync() @@ -211,7 +209,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, restart_template) + bkg_template = os.path.join(self.task_config.COMIN_ATMOS_RESTART_PREV, restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] @@ -237,7 +235,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: # NOTE for now this is FV3 RESTART files and just assumed to be fh006 # get FV3 RESTART files, this will be a lot simpler when using history files - rst_dir = task_config.COM_ATMOS_RESTART_PREV + rst_dir = task_config.COMIN_ATMOS_RESTART_PREV run_dir = os.path.join(task_config['DATA'], 'bkg') # Start accumulating list of background files to copy @@ -289,7 +287,7 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: """ # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. b_datestr = to_fv3time(config.BERROR_DATE) - analysis_dir = config.COM_CHEM_ANALYSIS + analysis_dir = config.COMIN_CHEM_BMATRIX cycle_datestr = to_fv3time(config.current_cycle) berror_list = [] @@ -310,10 +308,10 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: diff_hz = 'diffusion_hz.nc' diff_vt = 'diffusion_vt.nc' berror_list.append([ - os.path.join(config.COM_CHEM_ANALYSIS, diff_hz), os.path.join(config.DATA, 'berror', diff_hz) + os.path.join(config.COMIN_CHEM_BMATRIX, diff_hz), os.path.join(config.DATA, 'berror', diff_hz) ]) berror_list.append([ - os.path.join(config.COM_CHEM_ANALYSIS, diff_vt), os.path.join(config.DATA, 'berror', diff_vt) + os.path.join(config.COMIN_CHEM_BMATRIX, diff_vt), os.path.join(config.DATA, 'berror', diff_vt) ]) berror_dict = { diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 3d33701fa4b..3e5272149b6 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -218,7 +218,7 @@ def get_task_names(self): gfs_tasks += wave_prep_tasks if self.do_aero and 'gfs' in self.aero_anl_cdumps: - gfs_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gfs_tasks += ['prepobsaero'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index a0e221b197b..d134973ede2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -509,7 +509,7 @@ def prepobsaero(self): def aeroanlgenb(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) @@ -533,7 +533,7 @@ def aeroanlgenb(self): def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlgenb'} + dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_prep_obs_aero: @@ -562,7 +562,8 @@ def aeroanlvar(self): deps = [] dep_dict = { - 'type': 'task', 'name': f'{self.cdump}aeroanlgenb', + 'type': 'task', 'name': f'{self.cdump}aeroanlgenb', + 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])", } deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { From 372bf2ee3a9d5049c8d18919be2edaafc18a632a Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 28 Jun 2024 20:04:49 +0000 Subject: [PATCH 32/69] bugfix --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index d134973ede2..ed866c9fec1 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -563,7 +563,7 @@ def aeroanlvar(self): deps = [] dep_dict = { 'type': 'task', 'name': f'{self.cdump}aeroanlgenb', - 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])", + 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", } deps.append(rocoto.add_dependency(dep_dict)) dep_dict = { From 3741b4f57a70f6780e9251f33c2464df9cabb54d Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 28 Jun 2024 20:37:39 +0000 Subject: [PATCH 33/69] run the diagb in the half cycle --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ed866c9fec1..f348e3a0194 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -519,7 +519,7 @@ def aeroanlgenb(self): 'resources': resources, 'dependency': dependencies, 'envars': self.envars, - 'cycledef': self.cdump.replace('enkf', ''), + 'cycledef': 'gdas_half,gdas', 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', From 40597e1851122a21d6097bacfffbcf6f1aa174d9 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 28 Jun 2024 20:45:51 +0000 Subject: [PATCH 34/69] end of day save --- jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX | 3 +-- sorc/gdas.cd | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX index 03120c885dc..a5eac16aa18 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX +++ b/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX @@ -12,8 +12,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanl ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMIN_OBS:COM_OBS_TMPL \ +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL diff --git a/sorc/gdas.cd b/sorc/gdas.cd index f479a40a3ae..40c2d77e484 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit f479a40a3ae8002520241bf2218c722f005413b2 +Subproject commit 40c2d77e4849c6caa57421cc932296abb217d135 From 1517f23c5ab5c8ab75022b14be0d68fb086cba51 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 5 Jul 2024 17:42:28 +0000 Subject: [PATCH 35/69] saving before testing stage of bkgs --- ...X => JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX} | 2 +- jobs/rocoto/aeroanlgenb.sh | 2 +- parm/config/gfs/config.aeroanlgenb | 1 + parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 29 +++++++++++++++++++ ... exgdas_aero_analysis_generate_bmatrix.py} | 2 +- ush/python/pygfs/__init__.py | 2 ++ ush/python/pygfs/task/aero_bmatrix.py | 25 +++++++++------- 7 files changed, 49 insertions(+), 14 deletions(-) rename jobs/{JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX => JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX} (94%) create mode 100644 parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 rename scripts/{exglobal_aero_analysis_generate_bmatrix.py => exgdas_aero_analysis_generate_bmatrix.py} (94%) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX similarity index 94% rename from jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX rename to jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX index a5eac16aa18..81c89e9155b 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX +++ b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @@ -21,7 +21,7 @@ mkdir -p "${COMOUT_CHEM_BMAT}" ############################################################### # Run relevant script -EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exglobal_aero_analysis_generate_bmatrix.py} +EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh index dba6318e33b..0bffc1ea9bc 100755 --- a/jobs/rocoto/aeroanlgenb.sh +++ b/jobs/rocoto/aeroanlgenb.sh @@ -19,6 +19,6 @@ export PYTHONPATH ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_GENERATE_BMATRIX" +"${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX" status=$? exit "${status}" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 706b6314c8c..335d5110f8a 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -10,6 +10,7 @@ source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" +export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" export aero_diffusion_iter=10 export aero_diffusion_horiz_len=2500e3 export aero_diffusion_fixed_val=1.0 diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 new file mode 100644 index 00000000000..8c09462e540 --- /dev/null +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -0,0 +1,29 @@ +###################################### +# set some variables +###################################### +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +{% set ftype_list = ['fv_core.res', 'fv_tracer.res']} +###################################### +# create working directories +###################################### +mkdir: +- "{{ DATA }}/bkg +copy: +###################################### +# copy deterministic background files +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'ROTDIR':ROTDIR, + 'RUN':RUN, + 'YMD':current_cycle | to_YMD, + 'HH':current_cycle | strftime("%H"), + 'MEMDIR':""} %} + +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ bkg_time }}.coupler.res", "{{ DATA }}/bkg/{{ bkg_time }}.coupler.res"] +{% for ftype in ftype_list %} + {% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ bkg_time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkg_time }}.{{ ftype }}.tile{{ tile }}.nc"] + {% endfor %} +{% endfor %} \ No newline at end of file diff --git a/scripts/exglobal_aero_analysis_generate_bmatrix.py b/scripts/exgdas_aero_analysis_generate_bmatrix.py similarity index 94% rename from scripts/exglobal_aero_analysis_generate_bmatrix.py rename to scripts/exgdas_aero_analysis_generate_bmatrix.py index 19d5454ba02..734b30fea2c 100755 --- a/scripts/exglobal_aero_analysis_generate_bmatrix.py +++ b/scripts/exgdas_aero_analysis_generate_bmatrix.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# exglobal_aero_analysis_generate_bmatrix.py +# exgdas_aero_analysis_generate_bmatrix.py # This script creates an AerosolBMatrix object # and runs the methods needed # to stage files, compute the variance, and write to com diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index fa6b0b373e5..cc61df3ac48 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,8 +2,10 @@ import os from .task.analysis import Analysis +from .task.bmatrix import BMatrix from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis +from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis from .task.atmens_analysis import AtmEnsAnalysis from .task.snow_analysis import SnowAnalysis diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 4a5726ecdfb..526ec6b7cb4 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -24,8 +24,8 @@ def __init__(self, config: Dict[str, Any]) -> None: _res = int(self.config['CASE'][1:]) _res_anl = int(self.config['CASE_ANL'][1:]) - _bmat_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") - _diffusion_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.chem_diffusion.yaml") + _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") + _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -38,16 +38,17 @@ def __init__(self, config: Dict[str, Any]) -> None: 'npy_anl': _res_anl + 1, 'npz_anl': self.config['LEVS'] - 1, 'aero_bkg_fhr': map(int, str(self.config['aero_bkg_times']).split(',')), - 'OPREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'APREFIX': f"{self.runtime_config.CDUMP}.t{self.runtime_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN - 'GPREFIX': f"gdas.t{self.runtime_config.previous_cycle.hour:02d}z.", + 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'bmat_yaml': _bmat_yaml, 'diffusion_yaml': _diffusion_yaml, } ) # task_config is everything that this task should need - self.task_config = AttrDict(**self.config, **self.runtime_config, **local_dict) + self.task_config = AttrDict(. + **self.task_config, **local_dict) @logit(logger) def initialize(self: BMatrix) -> None: @@ -58,7 +59,9 @@ def initialize(self: BMatrix) -> None: FileHandler(jedi_fix_list).sync() # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + logger.info(f"Staging backgrounds prescribed from {self.task_config.AERO_BMATRIX_STAGE_TMPL}") + aero_bmat_stage_list = parse_j2yaml(self.task_config.AERO_BMATRIX_STAGE_TMPL, self.task_config) + FileHandler(aero_bmat_stage_list).sync() # generate diagb YAML file logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") @@ -126,10 +129,10 @@ def computeDiffusion(self) -> None: def finalize(self) -> None: super().finalize() # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diagb.yaml") - src_diffusion = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diffusion.yaml") - dest_diffusion = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.runtime_config['cyc']:02d}z.chem_diffusion.yaml") + src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") + src_diffusion = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") + dest_diffusion = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") yaml_copy = { 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], 'copy': [[src, dest], [src_diffusion, dest_diffusion]] From df9d40ceb3333d075ff7c9251af58ed3e132e0c8 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 5 Jul 2024 18:27:19 +0000 Subject: [PATCH 36/69] intermediate save --- parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 18 +++++++++--------- ush/python/pygfs/task/aero_bmatrix.py | 15 +++++++-------- ush/python/pygfs/task/bmatrix.py | 3 +-- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 index 8c09462e540..f88654f2ad5 100644 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -3,27 +3,27 @@ ###################################### {% set offset_td = "+6H" | to_timedelta %} {% set background_time = current_cycle | add_to_datetime(offset_td) %} -{% set ftype_list = ['fv_core.res', 'fv_tracer.res']} +{% set ftype_list = ['fv_core.res', 'fv_tracer.res'] %} ###################################### # create working directories ###################################### mkdir: -- "{{ DATA }}/bkg +- "{{ DATA }}/bkg" copy: ###################################### # copy deterministic background files ###################################### # define variables # Declare a dict of search and replace terms to run on each template -{% set tmpl_dict = {'ROTDIR':ROTDIR, - 'RUN':RUN, - 'YMD':current_cycle | to_YMD, - 'HH':current_cycle | strftime("%H"), - 'MEMDIR':""} %} +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle | to_YMD, + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':""} %} -- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ bkg_time }}.coupler.res", "{{ DATA }}/bkg/{{ bkg_time }}.coupler.res"] +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.coupler.res"] {% for ftype in ftype_list %} {% for tile in range(1, ntiles+1) %} -- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ bkg_time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkg_time }}.{{ ftype }}.tile{{ tile }}.nc"] +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] {% endfor %} {% endfor %} \ No newline at end of file diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 526ec6b7cb4..565500bcccb 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -21,8 +21,8 @@ class AerosolBMatrix(BMatrix): def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - _res = int(self.config['CASE'][1:]) - _res_anl = int(self.config['CASE_ANL'][1:]) + _res = int(self.task_config['CASE'][1:]) + _res_anl = int(self.task_config['CASE_ANL'][1:]) _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") @@ -32,12 +32,12 @@ def __init__(self, config: Dict[str, Any]) -> None: { 'npx_ges': _res + 1, 'npy_ges': _res + 1, - 'npz_ges': self.config.LEVS - 1, - 'npz': self.config.LEVS - 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, 'npx_anl': _res_anl + 1, 'npy_anl': _res_anl + 1, - 'npz_anl': self.config['LEVS'] - 1, - 'aero_bkg_fhr': map(int, str(self.config['aero_bkg_times']).split(',')), + 'npz_anl': self.task_config['LEVS'] - 1, + 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), 'OPREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'APREFIX': f"{self.task_config.CDUMP}.t{self.task_config.cyc:02d}z.", # TODO: CDUMP is being replaced by RUN 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", @@ -47,8 +47,7 @@ def __init__(self, config: Dict[str, Any]) -> None: ) # task_config is everything that this task should need - self.task_config = AttrDict(. - **self.task_config, **local_dict) + self.task_config = AttrDict(**self.task_config, **local_dict) @logit(logger) def initialize(self: BMatrix) -> None: diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index 59b4683ff38..da4560da37c 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -19,9 +19,8 @@ class BMatrix(Task): """ def __init__(self, config: Dict[str, Any]) -> None: super().__init__(config) - self.config.ntiles = 6 # Store location of GDASApp jinja2 templates - self.gdasapp_j2tmpl_dir = os.path.join(self.config.PARMgfs, 'gdas') + self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') def initialize(self) -> None: super().initialize() From 1424fbd630d003fbf5234f35601751f40e0e5960 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 5 Jul 2024 21:00:50 +0000 Subject: [PATCH 37/69] End of week commit --- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 8 ++-- parm/config/gfs/config.aeroanl | 2 + parm/config/gfs/config.aeroanlgenb | 1 + parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 | 19 ++++++++ parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 2 + parm/gdas/aero_stage_variational.yaml.j2 | 49 +++++++++++++++++++++ ush/python/pygfs/task/aero_analysis.py | 8 ++-- ush/python/pygfs/task/aero_bmatrix.py | 48 ++------------------ 8 files changed, 85 insertions(+), 52 deletions(-) create mode 100644 parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_stage_variational.yaml.j2 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index 600c6a19280..5be8767308c 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -19,13 +19,13 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMIN_OBS:COM_OBS_TMPL \ +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COM_OBS:COM_OBS_TMPL \ COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ + COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL mkdir -p "${COMOUT_CHEM_ANALYSIS}" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index ec94ace830d..d3a2e526b78 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -15,6 +15,8 @@ export BERROR_DATE="20160630.000000" export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" +export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2" + export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 335d5110f8a..28c9cf019d6 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -11,6 +11,7 @@ source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" +export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2" export aero_diffusion_iter=10 export aero_diffusion_horiz_len=2500e3 export aero_diffusion_fixed_val=1.0 diff --git a/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 new file mode 100644 index 00000000000..b33f2809454 --- /dev/null +++ b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 @@ -0,0 +1,19 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set HEAD = RUN + ".t" + cycle_HH + "z." %} +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +copy: +### copy YAMLs used +{% set yaml_list = ['chem_diagb.yaml', 'chem_diffusion.yaml'] %} +{% for fname in yaml_list %} +- ["{{ DATA }}/{{ HEAD }}{{ fname }}", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}{{ fname }}"] +{% endfor %} +### copy stddev files to ROTDIR +{% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.coupler.res", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ DATA }}/diffusion/diffusion_hz.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_hz.nc"] +- ["{{ DATA }}/diffusion/diffusion_vt.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_vt.nc"] diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 index f88654f2ad5..fcef417ffeb 100644 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -9,6 +9,8 @@ ###################################### mkdir: - "{{ DATA }}/bkg" +- "{{ DATA }}/stddev" +- "{{ DATA }}/diffusion" copy: ###################################### # copy deterministic background files diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 new file mode 100644 index 00000000000..18487f5371b --- /dev/null +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -0,0 +1,49 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkg_times = [] %} + {% for fh in range(0, 7, 3) %} + {% set offset = "{{ fh }}H" %} + {% set fcst_timedelta = offset | to_timedelta %} + {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} + {% do bkg_times.append({{ fcst_time | to_fv3time }}) %} + {% endfor %} +{% else %} + {% set bkg_times = ["{{ current_cycle | to_fv3time }}"] %} +{% endif %} +{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} +###################################### +mkdir: +- "{{ DATA }}/anl" +- "{{ DATA }}/diags" +- "{{ DATA }}/berror" +- "{{ DATA }}/bkg" +copy: +###################################### +## copy backgrounds +{% for bkgtime in bkg_times %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime }}.coupler.res"] + {% for fvfile in fvfiles %} + {% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime }}.{{ fvfile }}tile{{ tile }}.nc"] + {% endfor %} + {% endfor %} +{% endfor %} +###################################### +## copy berror files from COMIN_CHEM_BMAT_PREV +## stddev files +{% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMIN_CHEM_BMAT_PREV }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +## coupler file +## diffusion files +### copy stddev files to ROTDIR +{% for tile in range(1, ntiles+1) %} +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/stddev/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/stddev/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] \ No newline at end of file diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index c626eccaaad..ae3a795a623 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -82,10 +82,10 @@ def initialize(self: Analysis) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage berror files - # copy BUMP files, otherwise it will assume ID matrix - if self.task_config.get('STATICB_TYPE', 'identity') in ['diffusion']: - FileHandler(self.get_berror_dict(self.task_config)).sync() + # stage files from COM + logger.info(f"Staging files prescribed from {self.task_config.AERO_STAGE_VARIATIONAL_TMPL}") + aero_var_stage_list = parse_j2yaml(self.task_config.AERO_STAGE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_stage_list).sync() # stage backgrounds FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 565500bcccb..01ab04f9775 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -72,12 +72,6 @@ def initialize(self: BMatrix) -> None: save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") - # create output directory - FileHandler({'mkdir': [os.path.join(self.task_config['DATA'], 'stddev')]}).sync() - - # create diffusion output directory - FileHandler({'mkdir': [os.path.join(self.task_config['DATA'], 'diffusion')]}).sync() - # link executable to run directory logger.info(f'before link_bmatexe') self.link_bmatexe() @@ -127,44 +121,10 @@ def computeDiffusion(self) -> None: @logit(logger) def finalize(self) -> None: super().finalize() - # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") - src_diffusion = os.path.join(self.task_config['DATA'], f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") - dest_diffusion = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['CDUMP']}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") - yaml_copy = { - 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], - 'copy': [[src, dest], [src_diffusion, dest_diffusion]] - } - FileHandler(yaml_copy).sync() - - # copy stddev files to ROTDIR - logger.info('Copying std. dev. files to ROTDIR') - template = f'{to_fv3time(self.task_config.current_cycle)}.stddev.fv_tracer.res.tile{{tilenum}}.nc' - stddevlist = [] - # actual std dev tracer files - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'stddev', tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) - stddevlist.append([src, dest]) - # coupler file - coupler = f'{to_fv3time(self.task_config.current_cycle)}.stddev.coupler.res' - src = os.path.join(self.task_config.DATA, 'stddev', coupler) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, coupler) - stddevlist.append([src, dest]) - - FileHandler({'copy': stddevlist}).sync() - - # Diffusion files - diff_hz = 'diffusion_hz.nc' - diff_vt = 'diffusion_vt.nc' - src_hz = os.path.join(self.task_config.DATA, 'diffusion', diff_hz) - dest_hz = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_hz) - src_vt = os.path.join(self.task_config.DATA, 'diffusion', diff_vt) - dest_vt = os.path.join(self.task_config.COM_CHEM_ANALYSIS, diff_vt) - difflist = [[src_hz, dest_hz], [src_vt, dest_vt]] - FileHandler({'copy': difflist}).sync() + # save files to COMOUT + logger.info(f"Saving files to COMOUT based on {self.task_config.AERO_BMATRIX_FINALIZE_TMPL}") + aero_bmat_finalize_list = parse_j2yaml(self.task_config.AERO_BMATRIX_FINALIZE_TMPL, self.task_config) + FileHandler(aero_bmat_finalize_list).sync() @logit(logger) def link_bmatexe(self) -> None: From a4724201e8b6025f12f77dfcfda7173952945bf9 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 8 Jul 2024 13:23:22 +0000 Subject: [PATCH 38/69] get stage working --- parm/gdas/aero_stage_variational.yaml.j2 | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 index 18487f5371b..81a77b93133 100644 --- a/parm/gdas/aero_stage_variational.yaml.j2 +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -4,13 +4,15 @@ {% if DOIAU == True %} {% set bkg_times = [] %} {% for fh in range(0, 7, 3) %} - {% set offset = "{{ fh }}H" %} + #{% set offset = "+{{ fh }}H" %} + {% set offset = fh | string + "H" %} {% set fcst_timedelta = offset | to_timedelta %} {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} - {% do bkg_times.append({{ fcst_time | to_fv3time }}) %} + {% do bkg_times.append(fcst_time) %} {% endfor %} {% else %} - {% set bkg_times = ["{{ current_cycle | to_fv3time }}"] %} + {% set bkg_times = [] %} + {% do bkg_times.append(current_cycle) %} {% endif %} {% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} ###################################### @@ -23,10 +25,10 @@ copy: ###################################### ## copy backgrounds {% for bkgtime in bkg_times %} -- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime }}.coupler.res"] +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.coupler.res"] {% for fvfile in fvfiles %} {% for tile in range(1,ntiles+1) %} -- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime }}.{{ fvfile }}tile{{ tile }}.nc"] +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] {% endfor %} {% endfor %} {% endfor %} @@ -34,16 +36,10 @@ copy: ## copy berror files from COMIN_CHEM_BMAT_PREV ## stddev files {% for tile in range(1, ntiles+1) %} -- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMIN_CHEM_BMAT_PREV }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} -## coupler file -## diffusion files -### copy stddev files to ROTDIR -{% for tile in range(1, ntiles+1) %} -- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/stddev/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] {% endfor %} ### copy coupler file -- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/stddev/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] ### copy diffusion files - ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] - ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] \ No newline at end of file From 040ff7e0f07600ebeeec8455e3b4279965a34d01 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 8 Jul 2024 13:26:46 +0000 Subject: [PATCH 39/69] appease the pynorm gods --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index ec9e214cd03..09105f7eeca 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -562,7 +562,7 @@ def aeroanlvar(self): deps = [] dep_dict = { - 'type': 'task', 'name': f'{self.cdump}aeroanlgenb', + 'type': 'task', 'name': f'{self.cdump}aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", } deps.append(rocoto.add_dependency(dep_dict)) From 3ec830e8f137cb77f5a5ce2b57dbe59af8fc7263 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 8 Jul 2024 19:38:38 +0000 Subject: [PATCH 40/69] commit before attempt at comparing --- jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 16 ++---- parm/config/gfs/config.aeroanl | 1 + parm/gdas/aero_finalize_variational.yaml.j2 | 23 +++++++++ parm/gdas/aero_stage_variational.yaml.j2 | 1 + ush/forecast_postdet.sh | 12 +++++ ush/python/pygfs/task/aero_analysis.py | 55 ++++----------------- 6 files changed, 50 insertions(+), 58 deletions(-) create mode 100644 parm/gdas/aero_finalize_variational.yaml.j2 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index 455f572da5e..b894b82531f 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -8,25 +8,17 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan ############################################## # Set variables used in the script ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" - ############################################## # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS - -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +mkdir -p "${COMOUT_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index d3a2e526b78..01f9f2e8088 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -16,6 +16,7 @@ export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2" +export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2" export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 new file mode 100644 index 00000000000..b732810fe9c --- /dev/null +++ b/parm/gdas/aero_finalize_variational.yaml.j2 @@ -0,0 +1,23 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkgtime = AERO_WINDOW_BEGIN %} +{% else %} + {% set bkgtime = current_cycle %} +{% endif %} +###################################### +mkdir: +- "{{ COMOUT_CHEM_ANALYSIS }}" +- "{{ COMOUT_ATMOS_RESTART }}" +copy: +## copy variational YAML to ROTDIR +- ["{{ DATA }}/{{ APREFIX }}aerovar.yaml", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aerovar.yaml"] +## copy increments +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_ANALYSIS }}/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +## copy analysis +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} \ No newline at end of file diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 index 81a77b93133..bf25595c31c 100644 --- a/parm/gdas/aero_stage_variational.yaml.j2 +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -29,6 +29,7 @@ copy: {% for fvfile in fvfiles %} {% for tile in range(1,ntiles+1) %} - ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] {% endfor %} {% endfor %} {% endfor %} diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 993331d70b6..1117bfb3001 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -60,6 +60,18 @@ FV3_postdet() { break fi done + # Replace fv_tracer with aeroanl_fv_tracer restart files from current cycle (if found) + local nn + for (( nn = 1; nn <= ntiles; nn++ )); do + if [[ -f "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" ]]; then + rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ + "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + else + echo "'aeroanl_fv_tracer.res.tile1.nc' not found in '${COMOUT_ATMOS_RESTART}', using 'fv_tracer.res.tile1.nc'" + break + fi + done fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index ae3a795a623..f17e7d0e9ed 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -82,27 +82,16 @@ def initialize(self: Analysis) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage files from COM + # stage files from COM and create working directories logger.info(f"Staging files prescribed from {self.task_config.AERO_STAGE_VARIATIONAL_TMPL}") aero_var_stage_list = parse_j2yaml(self.task_config.AERO_STAGE_VARIATIONAL_TMPL, self.task_config) FileHandler(aero_var_stage_list).sync() - # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() - # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config['DATA'], 'anl'), - os.path.join(self.task_config['DATA'], 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - @logit(logger) def variational(self: Analysis) -> None: @@ -140,6 +129,7 @@ def finalize(self: Analysis) -> None: """ # ---- tar up diags # path of output tar statfile + logger.info('Preparing observation space diagnostics for archiving') aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") # get list of diag files to put in tarball @@ -147,6 +137,7 @@ def finalize(self: Analysis) -> None: # gzip the files first for diagfile in diags: + logger.info(f'Adding {diagfile} to tar file') with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) @@ -155,44 +146,16 @@ def finalize(self: Analysis) -> None: for diagfile in diags: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - yaml_copy = { - 'mkdir': [self.task_config.COMOUT_CHEM_ANALYSIS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() - - # ---- copy RESTART fv_tracer files for future reference - if self.task_config.DOIAU: - bkgtime = self.task_config.AERO_WINDOW_BEGIN - else: - bkgtime = self.task_config.current_cycle - template = '{}.fv_tracer.res.tile{}.nc'.format(to_fv3time(bkgtime), '{tilenum}') - bkglist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.COMIN_ATMOS_RESTART_PREV, tracer) - dest = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f'aeroges.{tracer}') - bkglist.append([src, dest]) - FileHandler({'copy': bkglist}).sync() + logger.info(f'Saved diags to {aerostat}') # ---- add increments to RESTART files logger.info('Adding increments to RESTART files') self._add_fms_cube_sphere_increments() - # ---- move increments to ROTDIR - logger.info('Moving increments to ROTDIR') - template = f'aeroinc.{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' - inclist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', tracer) - dest = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, tracer) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + # copy files back to COM + logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}") + aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_final_list).sync() def clean(self): super().clean() @@ -209,7 +172,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.COMIN_ATMOS_RESTART_PREV, restart_template) + bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] From 3d05aa7dc7ceedb7215ee3c523c6be7f00eae3a2 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 8 Jul 2024 19:40:12 +0000 Subject: [PATCH 41/69] norm issue --- ush/python/pygfs/task/aero_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index f17e7d0e9ed..19631601886 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -172,7 +172,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) + bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] From e5f241d8889cd4398cdbd8fdcdafe45b7b6b1f5a Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 8 Jul 2024 19:49:44 +0000 Subject: [PATCH 42/69] add EOLs --- parm/gdas/aero_finalize_variational.yaml.j2 | 2 +- parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 2 +- parm/gdas/aero_stage_variational.yaml.j2 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 index b732810fe9c..b9247bcd620 100644 --- a/parm/gdas/aero_finalize_variational.yaml.j2 +++ b/parm/gdas/aero_finalize_variational.yaml.j2 @@ -20,4 +20,4 @@ copy: ## copy analysis {% for tile in range(1,ntiles+1) %} - ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 index fcef417ffeb..732873bed93 100644 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -28,4 +28,4 @@ copy: {% for tile in range(1, ntiles+1) %} - ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] {% endfor %} -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 index bf25595c31c..7d435076f0c 100644 --- a/parm/gdas/aero_stage_variational.yaml.j2 +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -43,4 +43,4 @@ copy: - ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] ### copy diffusion files - ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] -- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] \ No newline at end of file +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] From e25b90a2cfd77f3f0fd84eb1f2b92a000b999b92 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 8 Jul 2024 21:03:56 +0000 Subject: [PATCH 43/69] placeholders for climatological b --- parm/config/gfs/config.aeroanl | 3 +-- parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 01f9f2e8088..555d59b3558 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -9,8 +9,7 @@ export CASE_ANL=${CASE} export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" export STATICB_TYPE='diffusion' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" -export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" -export BERROR_DATE="20160630.000000" +export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 index 732873bed93..536304fcaf6 100644 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -10,6 +10,7 @@ mkdir: - "{{ DATA }}/bkg" - "{{ DATA }}/stddev" +- "{{ DATA }/clm_stddev"} - "{{ DATA }}/diffusion" copy: ###################################### @@ -29,3 +30,9 @@ copy: - ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] {% endfor %} {% endfor %} +# copy climatological stddev files +###################################### +#{% for tile in range(1, ntiles+1) %} +#- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] +#{% endfor %} +# From 8d57a62f76ec89719f802f4be7bf173cd587bd63 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 9 Jul 2024 14:52:07 +0000 Subject: [PATCH 44/69] add in placeholder for static b + made stubs for berror/bkg dict --- parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 1 + ush/python/pygfs/task/aero_analysis.py | 67 +----------------------- 2 files changed, 3 insertions(+), 65 deletions(-) diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 index 536304fcaf6..0802e310017 100644 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -32,6 +32,7 @@ copy: {% endfor %} # copy climatological stddev files ###################################### +# uncomment out below once files are staged #{% for tile in range(1, ntiles+1) %} #- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] #{% endfor %} diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 19631601886..2a22bd56328 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -195,38 +195,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkg_dict: Dict a dictionary containing the list of model background files to copy for FileHandler """ - # NOTE for now this is FV3 RESTART files and just assumed to be fh006 - - # get FV3 RESTART files, this will be a lot simpler when using history files - rst_dir = task_config.COMIN_ATMOS_RESTART_PREV - run_dir = os.path.join(task_config['DATA'], 'bkg') - - # Start accumulating list of background files to copy - bkglist = [] - - # if using IAU, we can use FGAT - bkgtimes = [] - begintime = task_config.previous_cycle - for fcsthr in task_config.aero_bkg_fhr: - bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) - - # now loop over background times - for bkgtime in bkgtimes: - # aerosol DA needs coupler - basename = f'{to_fv3time(bkgtime)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - # aerosol DA only needs core/tracer - for ftype in ['core', 'tracer']: - template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - bkg_dict = { - 'mkdir': [run_dir], - 'copy': bkglist, - } + bkg_dict = {} return bkg_dict @logit(logger) @@ -248,37 +217,5 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_dict: Dict a dictionary containing the list of background error files to copy for FileHandler """ - # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. - b_datestr = to_fv3time(config.BERROR_DATE) - analysis_dir = config.COMIN_CHEM_BMATRIX - cycle_datestr = to_fv3time(config.current_cycle) - berror_list = [] - - # the stddev is computed every cycle and is available in COM - for ftype in ['stddev']: - coupler = f'{cycle_datestr}.{ftype}.coupler.res' - berror_list.append([ - os.path.join(analysis_dir, coupler), os.path.join(config.DATA, 'berror', coupler) - ]) - template = f'{cycle_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' - for itile in range(1, config.ntiles + 1): - tracer = template.format(tilenum=itile) - berror_list.append([ - os.path.join(analysis_dir, tracer), os.path.join(config.DATA, 'berror', tracer) - ]) - - # the diffusion correlation files are computed every cycle and are available in COM - diff_hz = 'diffusion_hz.nc' - diff_vt = 'diffusion_vt.nc' - berror_list.append([ - os.path.join(config.COMIN_CHEM_BMATRIX, diff_hz), os.path.join(config.DATA, 'berror', diff_hz) - ]) - berror_list.append([ - os.path.join(config.COMIN_CHEM_BMATRIX, diff_vt), os.path.join(config.DATA, 'berror', diff_vt) - ]) - - berror_dict = { - 'mkdir': [os.path.join(config.DATA, 'berror')], - 'copy': berror_list, - } + berror_dict = {} return berror_dict From 7be82c574996019c5870acb41169ba5effb5b6f1 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 9 Jul 2024 20:38:15 +0000 Subject: [PATCH 45/69] end of day --- parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 10 +++++----- sorc/gdas.cd | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 index 0802e310017..887419e2a49 100644 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -10,7 +10,7 @@ mkdir: - "{{ DATA }}/bkg" - "{{ DATA }}/stddev" -- "{{ DATA }/clm_stddev"} +- "{{ DATA }}/clm_stddev" - "{{ DATA }}/diffusion" copy: ###################################### @@ -33,7 +33,7 @@ copy: # copy climatological stddev files ###################################### # uncomment out below once files are staged -#{% for tile in range(1, ntiles+1) %} -#- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] -#{% endfor %} -# +{% for tile in range(1, ntiles+1) %} +- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 40c2d77e484..4fc561a15d5 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 40c2d77e4849c6caa57421cc932296abb217d135 +Subproject commit 4fc561a15d55e0248246fd3787c6919bbb0ebbb0 From 4a167a03d1494f90fb7f1412f145af5bacc1a21a Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 9 Jul 2024 20:38:58 +0000 Subject: [PATCH 46/69] remove comment --- parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 1 - 1 file changed, 1 deletion(-) diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 index 887419e2a49..9005b9ff127 100644 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -32,7 +32,6 @@ copy: {% endfor %} # copy climatological stddev files ###################################### -# uncomment out below once files are staged {% for tile in range(1, ntiles+1) %} - ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] {% endfor %} From e91346efa2f10793f585291ea75453120e5daf42 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 Jul 2024 16:43:45 +0000 Subject: [PATCH 47/69] extend walltime for aeroanlvar --- parm/config/gfs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 6eb36d0dece..896ad27caa8 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -440,7 +440,7 @@ case ${step} in export layout_x export layout_y - export wtime_aeroanlvar="00:30:00" + export wtime_aeroanlvar="00:45:00" export npe_aeroanlvar_gdas=$(( layout_x * layout_y * 6 )) export npe_aeroanlvar_gfs=$(( layout_x * layout_y * 6 )) export nth_aeroanlvar_gdas=1 From 6a7eb9d14e97c0f315be7690785e48715307874a Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 10 Jul 2024 17:03:42 +0000 Subject: [PATCH 48/69] extend walltime even more --- parm/config/gfs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 896ad27caa8..6b53a0419b9 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -440,7 +440,7 @@ case ${step} in export layout_x export layout_y - export wtime_aeroanlvar="00:45:00" + export wtime_aeroanlvar="00:55:00" export npe_aeroanlvar_gdas=$(( layout_x * layout_y * 6 )) export npe_aeroanlvar_gfs=$(( layout_x * layout_y * 6 )) export nth_aeroanlvar_gdas=1 From 54253c7377cd9bfb59299fa87376935b9f925b52 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 11 Jul 2024 19:05:54 +0000 Subject: [PATCH 49/69] save with more changes --- parm/config/gfs/config.aeroanlgenb | 3 + .../exgdas_aero_analysis_generate_bmatrix.py | 1 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/aero_bmatrix.py | 66 ++++++++++++++++++- ush/python/pygfs/task/bmatrix.py | 45 ------------- 5 files changed, 69 insertions(+), 48 deletions(-) diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 28c9cf019d6..b3421f808b8 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -10,11 +10,14 @@ source "${EXPDIR}/config.resources" aeroanlgenb export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" +export INTERPYAML="${PARMgfs}/gdas/aero/berror/aero_interp.yaml.j2" export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2" export aero_diffusion_iter=10 export aero_diffusion_horiz_len=2500e3 export aero_diffusion_fixed_val=1.0 +export npx_clim_b=97 +export npy_clim_b=97 echo "END: config.aeroanlgenb" diff --git a/scripts/exgdas_aero_analysis_generate_bmatrix.py b/scripts/exgdas_aero_analysis_generate_bmatrix.py index 734b30fea2c..0d8389c40d3 100755 --- a/scripts/exgdas_aero_analysis_generate_bmatrix.py +++ b/scripts/exgdas_aero_analysis_generate_bmatrix.py @@ -21,6 +21,7 @@ # Instantiate the aerosol variance and diffusion correlation tasks AeroB = AerosolBMatrix(config) AeroB.initialize() + AeroB.interpBackground() AeroB.computeVariance() AeroB.computeDiffusion() AeroB.finalize() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 4fc561a15d5..375cbb19e50 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 4fc561a15d55e0248246fd3787c6919bbb0ebbb0 +Subproject commit 375cbb19e50470f48774c19d3d54751876ba438f diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 01ab04f9775..538dc68ac7c 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -26,6 +26,7 @@ def __init__(self, config: Dict[str, Any]) -> None: _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") + _convertstate_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.CDUMP}.t{self.task_config['cyc']:02d}z.chem_convertstate.yaml") # Create a local dictionary that is repeatedly used across this class local_dict = AttrDict( @@ -43,6 +44,7 @@ def __init__(self, config: Dict[str, Any]) -> None: 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", 'bmat_yaml': _bmat_yaml, 'diffusion_yaml': _diffusion_yaml, + 'convertstate_yaml': _convertstate_yaml, } ) @@ -62,21 +64,55 @@ def initialize(self: BMatrix) -> None: aero_bmat_stage_list = parse_j2yaml(self.task_config.AERO_BMATRIX_STAGE_TMPL, self.task_config) FileHandler(aero_bmat_stage_list).sync() + # generate convert state YAML file + logger.info(f"Generate convert state YAML file: {self.task_config.convertstate_yaml}") + self.task_config.convertstate_config = parse_j2yaml(self.task_config.INTERPYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.convertstate_config, self.task_config.convertstate_yaml) + logger.info(f"Wrote convert state YAML to: {self.task_config.convertstate_yaml}") + # generate diagb YAML file logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") + self.task_config.bmat_config = parse_j2yaml(self.task_config.BMATYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") # generate diffusion parameters YAML file logger.info(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") + self.task_config.diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") # link executable to run directory - logger.info(f'before link_bmatexe') self.link_bmatexe() - logger.info(f'before link_diffusion_exe') self.link_diffusion_exe() + self.link_jediexe() + + @logit(logger) + def interpBackground(self) -> None: + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('convertstate') + exec_cmd.add_default_arg(self.task_config.convertstate_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass @logit(logger) def computeVariance(self) -> None: @@ -126,6 +162,32 @@ def finalize(self) -> None: aero_bmat_finalize_list = parse_j2yaml(self.task_config.AERO_BMATRIX_FINALIZE_TMPL, self.task_config) FileHandler(aero_bmat_finalize_list).sync() + @logit(logger) + def link_jediexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.JEDIEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return exe_dest + @logit(logger) def link_bmatexe(self) -> None: """ diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index da4560da37c..e49c271d49d 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -24,51 +24,6 @@ def __init__(self, config: Dict[str, Any]) -> None: def initialize(self) -> None: super().initialize() - # all BMatrix tasks need a config - self.task_config.bmat_config = self.get_bmat_config() - self.task_config.diffusion_config = self.get_diffusion_config() def finalize(self) -> None: super().finalize() - - @logit(logger) - def get_bmat_config(self) -> Dict[str, Any]: - """Compile a dictionary of B Matrix configuration from BMATYAML template file - - Parameters - ---------- - - Returns - ---------- - bmat_config : Dict - a dictionary containing the fully rendered B matrix yaml configuration - """ - - # generate JEDI YAML file for stddev - logger.info(f"Generate B Matrix YAML config: {self.task_config.bmat_yaml}") - bmat_config = parse_j2yaml(self.task_config.BMATYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - logger.debug(f"BMAT config:\n{pformat(bmat_config)}") - - return bmat_config - - def get_diffusion_config(self) -> Dict[str, Any]: - """Compile a dictionary of diffusion operator configuration from DIFFUSIONYAML template file - - Parameters - ---------- - diffusion_iter - fixed_val - horiz_len - - Returns - ---------- - diffusion_config : Dict - a dictionary containing the fully rendered diffusion operator yaml configuration - """ - - # generate JEDI YAML file for diffusion parameters - logger.info(f"Generate Diff Parmameter YAML config: {self.task_config.diffusion_yaml}") - diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, self.task_config, searchpath=self.gdasapp_j2tmpl_dir) - logger.debug(f"DIFFUSION config:\n{pformat(diffusion_config)}") - - return diffusion_config From 73b401f4f1626bcc1cce5e3400476a1c58986fa9 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 11 Jul 2024 19:07:52 +0000 Subject: [PATCH 50/69] fix norms --- ush/python/pygfs/task/aero_bmatrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 538dc68ac7c..5e66035c20b 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -67,8 +67,8 @@ def initialize(self: BMatrix) -> None: # generate convert state YAML file logger.info(f"Generate convert state YAML file: {self.task_config.convertstate_yaml}") self.task_config.convertstate_config = parse_j2yaml(self.task_config.INTERPYAML, - self.task_config, - searchpath=self.gdasapp_j2tmpl_dir) + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) save_as_yaml(self.task_config.convertstate_config, self.task_config.convertstate_yaml) logger.info(f"Wrote convert state YAML to: {self.task_config.convertstate_yaml}") From 792716683e30acce5403ea813a4a98e53e1220fb Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 11 Jul 2024 19:19:15 +0000 Subject: [PATCH 51/69] make more options configurable --- parm/config/gfs/config.aeroanlgenb | 8 +++++++- sorc/gdas.cd | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index b3421f808b8..b41b22a5243 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -18,6 +18,12 @@ export aero_diffusion_horiz_len=2500e3 export aero_diffusion_fixed_val=1.0 export npx_clim_b=97 export npy_clim_b=97 - +export aero_diagb_weight=0.9 +export aero_staticb_rescaling_factor=2.0 +export aero_diagb_rescale=20.0 +export aero_diagb_n_halo=4 +export aero_diagb_n_neighbors=16 +export aero_diagb_smooth_horiz_iter=0 +export aero_diagb_smooth_vert_iter=0 echo "END: config.aeroanlgenb" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 375cbb19e50..63fff0dad06 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 375cbb19e50470f48774c19d3d54751876ba438f +Subproject commit 63fff0dad06f2603065ae2c57331a985f598087e From 96b43d082eda5b0394ea92e0bd7a0c07b2cf61b0 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 11 Jul 2024 20:09:50 +0000 Subject: [PATCH 52/69] fix slow aeroanlinit (hopefully) and update gdas hash --- parm/gdas/aero_stage_variational.yaml.j2 | 7 ++++++- sorc/gdas.cd | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 index 7d435076f0c..93d0af5f2e8 100644 --- a/parm/gdas/aero_stage_variational.yaml.j2 +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -29,10 +29,15 @@ copy: {% for fvfile in fvfiles %} {% for tile in range(1,ntiles+1) %} - ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] -- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] {% endfor %} {% endfor %} {% endfor %} +###################################### +## copy backgrounds again for fv_tracer to create analysis files later +{% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + ###################################### ## copy berror files from COMIN_CHEM_BMAT_PREV ## stddev files diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 63fff0dad06..b2ca01e4322 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 63fff0dad06f2603065ae2c57331a985f598087e +Subproject commit b2ca01e43226e6c88673ddf31a16ab901ffed709 From f0e3725b31144abd66a27051d477848c6a8c17de Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 15 Jul 2024 13:53:36 +0000 Subject: [PATCH 53/69] update config files --- parm/config/gfs/config.aeroanl | 17 ++++++++++++++++- parm/config/gfs/config.resources | 24 ++++++++++++------------ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 555d59b3558..5d831193831 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -5,7 +5,22 @@ echo "BEGIN: config.aeroanl" -export CASE_ANL=${CASE} +# define analysis resolution based on deterministic res +case ${CASE} in + "C1152" | "C768" | "C384") + CASE_ANL="C192" + ;; + "C192" | "C96") + CASE_ANL="C96" + ;; + "C48" ) + CASE_ANL="C48" + ;; + *) + echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" + exit 4 +esac +export CASE_ANL export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" export STATICB_TYPE='diffusion' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 6c00e87dbf8..7908cbc65e5 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -387,12 +387,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") - layout_x=8 - layout_y=8 + layout_x=4 + layout_y=4 ;; "C48" ) # this case is for testing only @@ -420,12 +420,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") - layout_x=8 - layout_y=8 + layout_x=4 + layout_y=4 ;; "C48" ) # this case is for testing only @@ -466,12 +466,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") - layout_x=8 - layout_y=8 + layout_x=4 + layout_y=4 ;; "C48" ) # this case is for testing only From 0f28779a6a182b0a101219427409b0748dbe5a99 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 16 Jul 2024 17:43:06 +0000 Subject: [PATCH 54/69] update dependencies for aeroanlinit --- workflow/rocoto/gfs_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index c7562d0cd76..577cff251c7 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -535,6 +535,8 @@ def aeroanlinit(self): deps = [] dep_dict = {'type': 'task', 'name': f'{self.cdump}aeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.cdump}prep'} + deps.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_prep_obs_aero: dep_dict = {'type': 'task', 'name': f'{self.cdump}prepobsaero'} From 9a337465a2ead6cb6697682a0d0bd7cb49752649 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Mon, 29 Jul 2024 19:01:39 +0000 Subject: [PATCH 55/69] Address most comments, still need to look at config --- jobs/rocoto/aeroanlgenb.sh | 5 ----- parm/config/gfs/config.aeroanl | 9 +++------ parm/config/gfs/config.com | 2 +- parm/gdas/aero_stage_variational.yaml.j2 | 1 - sorc/gdas.cd | 2 +- ush/forecast_postdet.sh | 2 +- ush/python/pygfs/task/bmatrix.py | 1 - 7 files changed, 6 insertions(+), 16 deletions(-) diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh index 0bffc1ea9bc..d0bc5dda9b5 100755 --- a/jobs/rocoto/aeroanlgenb.sh +++ b/jobs/rocoto/aeroanlgenb.sh @@ -12,12 +12,7 @@ export job="aeroanlgenb" export jobid="${job}.$$" ############################################################### -# setup python path for workflow utilities and tasks -wxflowPATH="${HOMEgfs}/ush/python:${HOMEgfs}/ush/python/wxflow/src" -PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${wxflowPATH}" -export PYTHONPATH -############################################################### # Execute the JJOB "${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX" status=$? diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 5d831193831..19766062d9c 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -7,14 +7,11 @@ echo "BEGIN: config.aeroanl" # define analysis resolution based on deterministic res case ${CASE} in - "C1152" | "C768" | "C384") + "C1152" | "C768" | "C384" | "C192") CASE_ANL="C192" ;; - "C192" | "C96") - CASE_ANL="C96" - ;; - "C48" ) - CASE_ANL="C48" + "C96" | "C48") + CASE_ANL=${CASE} ;; *) echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 3d49494da16..818ea380179 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -98,6 +98,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' -declare -rx COM_CHEM_BMAT_TMPL=${COM_BASE}'/bmatrix/chem' +declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 index 93d0af5f2e8..afd0e1b9461 100644 --- a/parm/gdas/aero_stage_variational.yaml.j2 +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -4,7 +4,6 @@ {% if DOIAU == True %} {% set bkg_times = [] %} {% for fh in range(0, 7, 3) %} - #{% set offset = "+{{ fh }}H" %} {% set offset = fh | string + "H" %} {% set fcst_timedelta = offset | to_timedelta %} {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 52f41a298b4..5583fbb6884 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 52f41a298b4c6b7bbf6f203b6579516819fbbf36 +Subproject commit 5583fbb6884a9b98fb3df76058ea2f51414299a2 diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 156538e539c..bd15ec8f929 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -68,7 +68,7 @@ FV3_postdet() { ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" else - echo "'aeroanl_fv_tracer.res.tile1.nc' not found in '${COMOUT_ATMOS_RESTART}', using 'fv_tracer.res.tile1.nc'" + echo "WARNING: 'aeroanl_fv_tracer.res.tile1.nc' not found in '${COMOUT_ATMOS_RESTART}', using 'fv_tracer.res.tile1.nc'" break fi done diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py index e49c271d49d..d0edba2358a 100644 --- a/ush/python/pygfs/task/bmatrix.py +++ b/ush/python/pygfs/task/bmatrix.py @@ -3,7 +3,6 @@ import os from logging import getLogger from typing import List, Dict, Any, Union -from pprint import pformat from wxflow import (parse_j2yaml, FileHandler, logit, Task, Executable, WorkflowException) From f448b24c612d75020863127f2009a71e543f0095 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Thu, 1 Aug 2024 18:16:48 +0000 Subject: [PATCH 56/69] address comments --- env/WCOSS2.env | 2 +- workflow/rocoto/gfs_tasks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 645e0cd03a0..89cc51da974 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -73,7 +73,7 @@ elif [[ "${step}" = "aeroanlvar" ]]; then elif [[ "${step}" = "aeroanlgenb" ]]; then export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + export APRUN_AEROANLGENB="${APRUN}" elif [[ "${step}" = "prepobsaero" ]]; then diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index bc903fdb2fb..bc81ded69ab 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -509,12 +509,12 @@ def prepobsaero(self): def aeroanlgenb(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.cdump}fcst'} + dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) resources = self.get_resource('aeroanlgenb') - task_name = f'{self.cdump}aeroanlgenb' + task_name = f'{self.run}aeroanlgenb' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, @@ -564,7 +564,7 @@ def aeroanlvar(self): deps = [] dep_dict = { - 'type': 'task', 'name': f'{self.run}aeroanlgenb', + 'type': 'task', 'name': f'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", } deps.append(rocoto.add_dependency(dep_dict)) From 09b11339fca5a06d52b0a86556fb3f1c81c3f8c7 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 2 Aug 2024 14:03:44 +0000 Subject: [PATCH 57/69] remove CDUMP --- env/HERA.env | 7 +- env/HERCULES.env | 7 +- env/JET.env | 11 +- env/ORION.env | 7 +- env/S4.env | 7 +- env/WCOSS2.env | 7 +- jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX | 46 +++ jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 16 +- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 10 +- ..._RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} | 4 +- jobs/rocoto/aeroanlgenb.sh | 19 ++ jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} | 4 +- parm/config/gfs/config.aeroanl | 24 +- parm/config/gfs/config.aeroanlgenb | 29 ++ parm/config/gfs/config.aeroanlrun | 11 - parm/config/gfs/config.aeroanlvar | 11 + parm/config/gfs/config.com | 1 + parm/config/gfs/config.resources | 54 +++- parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 | 19 ++ parm/gdas/aero_finalize_variational.yaml.j2 | 23 ++ parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 38 +++ parm/gdas/aero_stage_variational.yaml.j2 | 50 +++ parm/ufs/gocart/ExtData.other | 20 +- .../exgdas_aero_analysis_generate_bmatrix.py | 27 ++ ... => exglobal_aero_analysis_variational.py} | 6 +- sorc/gdas.cd | 2 +- sorc/gfs_utils.fd | 2 +- sorc/link_workflow.sh | 2 + ush/forecast_postdet.sh | 12 + ush/python/pygfs/__init__.py | 2 + ush/python/pygfs/task/aero_analysis.py | 129 +------- ush/python/pygfs/task/aero_bmatrix.py | 294 ++++++++++++++++++ ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/bmatrix.py | 28 ++ workflow/applications/gfs_cycled.py | 6 +- workflow/rocoto/gfs_tasks.py | 51 ++- workflow/rocoto/tasks.py | 2 +- 37 files changed, 797 insertions(+), 193 deletions(-) create mode 100755 jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX rename jobs/{JGLOBAL_AERO_ANALYSIS_RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} (83%) create mode 100755 jobs/rocoto/aeroanlgenb.sh rename jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} (83%) create mode 100644 parm/config/gfs/config.aeroanlgenb delete mode 100644 parm/config/gfs/config.aeroanlrun create mode 100644 parm/config/gfs/config.aeroanlvar create mode 100644 parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_finalize_variational.yaml.j2 create mode 100644 parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_stage_variational.yaml.j2 create mode 100755 scripts/exgdas_aero_analysis_generate_bmatrix.py rename scripts/{exglobal_aero_analysis_run.py => exglobal_aero_analysis_variational.py} (84%) create mode 100644 ush/python/pygfs/task/aero_bmatrix.py create mode 100644 ush/python/pygfs/task/bmatrix.py diff --git a/env/HERA.env b/env/HERA.env index 3f0e7c9f369..ae82bfac02a 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -82,13 +82,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index 83fa1aadd18..151a2da2518 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -86,12 +86,17 @@ case ${step} in export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" ;; - "aeroanlrun") + "aeroanlvar") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" + ;; + "aeroanlgenb") + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" ;; "prepobsaero") diff --git a/env/JET.env b/env/JET.env index 810a8cd5017..d93d8438fc8 100755 --- a/env/JET.env +++ b/env/JET.env @@ -70,13 +70,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}" -elif [[ "${step}" = "aeroanlrun" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" +elif [[ "${step}" = "aeroanlvar" ]]; then export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/ORION.env b/env/ORION.env index bbbfb59182b..25fba6cfa64 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -78,13 +78,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/S4.env b/env/S4.env index 840ca658981..5b7432104ce 100755 --- a/env/S4.env +++ b/env/S4.env @@ -70,13 +70,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 18caf1bc037..89cc51da974 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -63,13 +63,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX new file mode 100755 index 00000000000..81c89e9155b --- /dev/null +++ b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @@ -0,0 +1,46 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL + +mkdir -p "${COMOUT_CHEM_BMAT}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index 455f572da5e..b894b82531f 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -8,25 +8,17 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan ############################################## # Set variables used in the script ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" - ############################################## # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS - -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +mkdir -p "${COMOUT_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index b2a2893bc04..5be8767308c 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -19,13 +19,15 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COM_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ + COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +mkdir -p "${COMOUT_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_RUN b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL similarity index 83% rename from jobs/JGLOBAL_AERO_ANALYSIS_RUN rename to jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL index 43749b78c5f..290d7225dd4 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_RUN +++ b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar" ############################################## # Set variables used in the script @@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr ############################################################### # Run relevant script -EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py} +EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh new file mode 100755 index 00000000000..d0bc5dda9b5 --- /dev/null +++ b/jobs/rocoto/aeroanlgenb.sh @@ -0,0 +1,19 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="aeroanlgenb" +export jobid="${job}.$$" + +############################################################### + +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX" +status=$? +exit "${status}" diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlvar.sh similarity index 83% rename from jobs/rocoto/aeroanlrun.sh rename to jobs/rocoto/aeroanlvar.sh index 529bb2d7d1f..7aa7d831f95 100755 --- a/jobs/rocoto/aeroanlrun.sh +++ b/jobs/rocoto/aeroanlvar.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="aeroanlrun" +export job="aeroanlvar" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN" +"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL" status=$? exit "${status}" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 24a5e926448..19766062d9c 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -5,20 +5,36 @@ echo "BEGIN: config.aeroanl" -export CASE_ANL=${CASE} +# define analysis resolution based on deterministic res +case ${CASE} in + "C1152" | "C768" | "C384" | "C192") + CASE_ANL="C192" + ;; + "C96" | "C48") + CASE_ANL=${CASE} + ;; + *) + echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" + exit 4 +esac +export CASE_ANL export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" -export STATICB_TYPE='identity' +export STATICB_TYPE='diffusion' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" -export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" -export BERROR_DATE="20160630.000000" +export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" +export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2" +export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2" + export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" +export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" +export DIFFUSIONEXE="${EXECgfs}/gdas_fv3jedi_error_covariance_toolbox.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb new file mode 100644 index 00000000000..b41b22a5243 --- /dev/null +++ b/parm/config/gfs/config.aeroanlgenb @@ -0,0 +1,29 @@ +#!/bin/bash -x + +########## config.aeroanlgenb ########## +# Aerosol Variance specific + +echo "BEGIN: config.aeroanlgenb" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlgenb + +export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" +export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" +export INTERPYAML="${PARMgfs}/gdas/aero/berror/aero_interp.yaml.j2" +export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" +export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2" +export aero_diffusion_iter=10 +export aero_diffusion_horiz_len=2500e3 +export aero_diffusion_fixed_val=1.0 +export npx_clim_b=97 +export npy_clim_b=97 +export aero_diagb_weight=0.9 +export aero_staticb_rescaling_factor=2.0 +export aero_diagb_rescale=20.0 +export aero_diagb_n_halo=4 +export aero_diagb_n_neighbors=16 +export aero_diagb_smooth_horiz_iter=0 +export aero_diagb_smooth_vert_iter=0 + +echo "END: config.aeroanlgenb" diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun deleted file mode 100644 index 012e5b79f3b..00000000000 --- a/parm/config/gfs/config.aeroanlrun +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -x - -########## config.aeroanlrun ########## -# Aerosol Analysis specific - -echo "BEGIN: config.aeroanlrun" - -# Get task specific resources -source "${EXPDIR}/config.resources" aeroanlrun - -echo "END: config.aeroanlrun" diff --git a/parm/config/gfs/config.aeroanlvar b/parm/config/gfs/config.aeroanlvar new file mode 100644 index 00000000000..4282b6c840b --- /dev/null +++ b/parm/config/gfs/config.aeroanlvar @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlvar ########## +# Aerosol Analysis specific + +echo "BEGIN: config.aeroanlvar" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlvar + +echo "END: config.aeroanlvar" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 222ffdae95d..818ea380179 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -98,5 +98,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' +declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 9ddb85a87ad..9c576aa76b0 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -16,7 +16,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl" - echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -355,12 +355,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") - layout_x=8 - layout_y=8 + layout_x=4 + layout_y=4 ;; "C48" ) # this case is for testing only @@ -381,27 +381,61 @@ case ${step} in memory="3072M" ;; - "aeroanlrun") + "aeroanlvar") case ${CASE} in "C768") layout_x=8 layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") + layout_x=4 + layout_y=4 + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "aeroanlgenb") + case ${CASE} in + "C768") layout_x=8 layout_y=8 ;; + "C384") + layout_x=6 + layout_y=6 + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + ;; "C48" ) # this case is for testing only layout_x=1 layout_y=1 ;; *) - echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 esac @@ -413,8 +447,10 @@ case ${step} in threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True + ;; + "aeroanlfinal") walltime="00:10:00" ntasks=1 diff --git a/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 new file mode 100644 index 00000000000..b33f2809454 --- /dev/null +++ b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 @@ -0,0 +1,19 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set HEAD = RUN + ".t" + cycle_HH + "z." %} +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +copy: +### copy YAMLs used +{% set yaml_list = ['chem_diagb.yaml', 'chem_diffusion.yaml'] %} +{% for fname in yaml_list %} +- ["{{ DATA }}/{{ HEAD }}{{ fname }}", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}{{ fname }}"] +{% endfor %} +### copy stddev files to ROTDIR +{% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.coupler.res", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ DATA }}/diffusion/diffusion_hz.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_hz.nc"] +- ["{{ DATA }}/diffusion/diffusion_vt.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_vt.nc"] diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 new file mode 100644 index 00000000000..b9247bcd620 --- /dev/null +++ b/parm/gdas/aero_finalize_variational.yaml.j2 @@ -0,0 +1,23 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkgtime = AERO_WINDOW_BEGIN %} +{% else %} + {% set bkgtime = current_cycle %} +{% endif %} +###################################### +mkdir: +- "{{ COMOUT_CHEM_ANALYSIS }}" +- "{{ COMOUT_ATMOS_RESTART }}" +copy: +## copy variational YAML to ROTDIR +- ["{{ DATA }}/{{ APREFIX }}aerovar.yaml", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aerovar.yaml"] +## copy increments +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_ANALYSIS }}/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +## copy analysis +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 new file mode 100644 index 00000000000..9005b9ff127 --- /dev/null +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -0,0 +1,38 @@ +###################################### +# set some variables +###################################### +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +{% set ftype_list = ['fv_core.res', 'fv_tracer.res'] %} +###################################### +# create working directories +###################################### +mkdir: +- "{{ DATA }}/bkg" +- "{{ DATA }}/stddev" +- "{{ DATA }}/clm_stddev" +- "{{ DATA }}/diffusion" +copy: +###################################### +# copy deterministic background files +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle | to_YMD, + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':""} %} + +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.coupler.res"] +{% for ftype in ftype_list %} + {% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] + {% endfor %} +{% endfor %} +# copy climatological stddev files +###################################### +{% for tile in range(1, ntiles+1) %} +- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 new file mode 100644 index 00000000000..afd0e1b9461 --- /dev/null +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -0,0 +1,50 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkg_times = [] %} + {% for fh in range(0, 7, 3) %} + {% set offset = fh | string + "H" %} + {% set fcst_timedelta = offset | to_timedelta %} + {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} + {% do bkg_times.append(fcst_time) %} + {% endfor %} +{% else %} + {% set bkg_times = [] %} + {% do bkg_times.append(current_cycle) %} +{% endif %} +{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} +###################################### +mkdir: +- "{{ DATA }}/anl" +- "{{ DATA }}/diags" +- "{{ DATA }}/berror" +- "{{ DATA }}/bkg" +copy: +###################################### +## copy backgrounds +{% for bkgtime in bkg_times %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.coupler.res"] + {% for fvfile in fvfiles %} + {% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] + {% endfor %} + {% endfor %} +{% endfor %} +###################################### +## copy backgrounds again for fv_tracer to create analysis files later +{% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + +###################################### +## copy berror files from COMIN_CHEM_BMAT_PREV +## stddev files +{% for tile in range(1, ntiles+1) %} +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 7a0d63d6ca2..5d2ddc51021 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL1 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO4 NA Y Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -63,11 +63,11 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC1 NA Y Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC2 NA Y Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null @@ -88,11 +88,11 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC1 NA Y Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC2 NA Y Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null diff --git a/scripts/exgdas_aero_analysis_generate_bmatrix.py b/scripts/exgdas_aero_analysis_generate_bmatrix.py new file mode 100755 index 00000000000..0d8389c40d3 --- /dev/null +++ b/scripts/exgdas_aero_analysis_generate_bmatrix.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# exgdas_aero_analysis_generate_bmatrix.py +# This script creates an AerosolBMatrix object +# and runs the methods needed +# to stage files, compute the variance, and write to com +# files needed for the variational solver +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.aero_bmatrix import AerosolBMatrix + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the aerosol variance and diffusion correlation tasks + AeroB = AerosolBMatrix(config) + AeroB.initialize() + AeroB.interpBackground() + AeroB.computeVariance() + AeroB.computeDiffusion() + AeroB.finalize() diff --git a/scripts/exglobal_aero_analysis_run.py b/scripts/exglobal_aero_analysis_variational.py similarity index 84% rename from scripts/exglobal_aero_analysis_run.py rename to scripts/exglobal_aero_analysis_variational.py index 85f4b963a4f..dd5bb4f65a6 100755 --- a/scripts/exglobal_aero_analysis_run.py +++ b/scripts/exglobal_aero_analysis_variational.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# exglobal_aero_analysis_run.py +# exglobal_aero_analysis_variational.py # This script creates an AerosolAnalysis object -# and runs the execute method +# and runs the variational method # which executes the global aerosol variational analysis import os @@ -19,4 +19,4 @@ # Instantiate the aerosol analysis task AeroAnl = AerosolAnalysis(config) - AeroAnl.execute() + AeroAnl.variational() diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 52f41a298b4..5583fbb6884 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 52f41a298b4c6b7bbf6f203b6579516819fbbf36 +Subproject commit 5583fbb6884a9b98fb3df76058ea2f51414299a2 diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 02ce084c244..279bbf2097d 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 02ce084c244823e22661d493a50236b7d5eaf70a +Subproject commit 279bbf2097d87321294436d17bf5b73c4c07ab4a diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 9722f5a2b8e..505e60b7467 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -367,9 +367,11 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then declare -a JEDI_EXE=("gdas.x" \ "gdas_soca_gridgen.x" \ "gdas_soca_error_covariance_toolbox.x" \ + "gdas_fv3jedi_error_covariance_toolbox.x" \ "gdas_soca_setcorscales.x" \ "gdas_soca_diagb.x" \ "fv3jedi_plot_field.x" \ + "gdasapp_chem_diagb.x" \ "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7de31d6235b..bd15ec8f929 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -60,6 +60,18 @@ FV3_postdet() { break fi done + # Replace fv_tracer with aeroanl_fv_tracer restart files from current cycle (if found) + local nn + for (( nn = 1; nn <= ntiles; nn++ )); do + if [[ -f "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" ]]; then + rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ + "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + else + echo "WARNING: 'aeroanl_fv_tracer.res.tile1.nc' not found in '${COMOUT_ATMOS_RESTART}', using 'fv_tracer.res.tile1.nc'" + break + fi + done fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index c0b72bbc352..e1ff7e68624 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,8 +2,10 @@ import os from .task.analysis import Analysis +from .task.bmatrix import BMatrix from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis +from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis from .task.atmens_analysis import AtmEnsAnalysis from .task.marine_bmat import MarineBMat diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 69a992d7d47..2a22bd56328 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -82,29 +82,18 @@ def initialize(self: Analysis) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage berror files - # copy BUMP files, otherwise it will assume ID matrix - if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: - FileHandler(self.get_berror_dict(self.task_config)).sync() - - # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + # stage files from COM and create working directories + logger.info(f"Staging files prescribed from {self.task_config.AERO_STAGE_VARIATIONAL_TMPL}") + aero_var_stage_list = parse_j2yaml(self.task_config.AERO_STAGE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_stage_list).sync() # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config['DATA'], 'anl'), - os.path.join(self.task_config['DATA'], 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self: Analysis) -> None: + def variational(self: Analysis) -> None: chdir(self.task_config.DATA) @@ -140,13 +129,15 @@ def finalize(self: Analysis) -> None: """ # ---- tar up diags # path of output tar statfile - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + logger.info('Preparing observation space diagnostics for archiving') + aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") # get list of diag files to put in tarball diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) # gzip the files first for diagfile in diags: + logger.info(f'Adding {diagfile} to tar file') with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) @@ -155,44 +146,16 @@ def finalize(self: Analysis) -> None: for diagfile in diags: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - yaml_copy = { - 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() - - # ---- copy RESTART fv_tracer files for future reference - if self.task_config.DOIAU: - bkgtime = self.task_config.AERO_WINDOW_BEGIN - else: - bkgtime = self.task_config.current_cycle - template = '{}.fv_tracer.res.tile{}.nc'.format(to_fv3time(bkgtime), '{tilenum}') - bkglist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}') - bkglist.append([src, dest]) - FileHandler({'copy': bkglist}).sync() + logger.info(f'Saved diags to {aerostat}') # ---- add increments to RESTART files logger.info('Adding increments to RESTART files') self._add_fms_cube_sphere_increments() - # ---- move increments to ROTDIR - logger.info('Moving increments to ROTDIR') - template = f'aeroinc.{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' - inclist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + # copy files back to COM + logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}") + aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_final_list).sync() def clean(self): super().clean() @@ -209,7 +172,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, restart_template) + bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] @@ -232,38 +195,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkg_dict: Dict a dictionary containing the list of model background files to copy for FileHandler """ - # NOTE for now this is FV3 RESTART files and just assumed to be fh006 - - # get FV3 RESTART files, this will be a lot simpler when using history files - rst_dir = task_config.COM_ATMOS_RESTART_PREV - run_dir = os.path.join(task_config['DATA'], 'bkg') - - # Start accumulating list of background files to copy - bkglist = [] - - # if using IAU, we can use FGAT - bkgtimes = [] - begintime = task_config.previous_cycle - for fcsthr in task_config.aero_bkg_fhr: - bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) - - # now loop over background times - for bkgtime in bkgtimes: - # aerosol DA needs coupler - basename = f'{to_fv3time(bkgtime)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - # aerosol DA only needs core/tracer - for ftype in ['core', 'tracer']: - template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - bkg_dict = { - 'mkdir': [run_dir], - 'copy': bkglist, - } + bkg_dict = {} return bkg_dict @logit(logger) @@ -285,34 +217,5 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_dict: Dict a dictionary containing the list of background error files to copy for FileHandler """ - # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. - b_dir = config.BERROR_DATA_DIR - b_datestr = to_fv3time(config.BERROR_DATE) - berror_list = [] - - for ftype in ['stddev']: - coupler = f'{b_datestr}.{ftype}.coupler.res' - berror_list.append([ - os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) - ]) - template = f'{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' - for itile in range(1, config.ntiles + 1): - tracer = template.format(tilenum=itile) - berror_list.append([ - os.path.join(b_dir, tracer), os.path.join(config.DATA, 'berror', tracer) - ]) - radius = 'cor_aero_universe_radius' - berror_list.append([ - os.path.join(b_dir, radius), os.path.join(config.DATA, 'berror', radius) - ]) - nproc = config.ntiles * config.layout_x * config.layout_y - for nn in range(1, nproc + 1): - berror_list.append([ - os.path.join(b_dir, f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc'), - os.path.join(config.DATA, 'berror', f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc') - ]) - berror_dict = { - 'mkdir': [os.path.join(config.DATA, 'berror')], - 'copy': berror_list, - } + berror_dict = {} return berror_dict diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py new file mode 100644 index 00000000000..9431e458022 --- /dev/null +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (AttrDict, FileHandler, rm_p, + add_to_datetime, to_fv3time, to_timedelta, + to_fv3time, chdir, Executable, WorkflowException, + parse_j2yaml, save_as_yaml, logit) +from pygfs.task.bmatrix import BMatrix + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolBMatrix(BMatrix): + """ + Class for global aerosol BMatrix tasks + """ + @logit(logger, name="AerosolBMatrix") + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + _res = int(self.task_config['CASE'][1:]) + _res_anl = int(self.task_config['CASE_ANL'][1:]) + + _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") + _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") + _convertstate_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_convertstate.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config['LEVS'] - 1, + 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'bmat_yaml': _bmat_yaml, + 'diffusion_yaml': _diffusion_yaml, + 'convertstate_yaml': _convertstate_yaml, + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.task_config, **local_dict) + + @logit(logger) + def initialize(self: BMatrix) -> None: + super().initialize() + # stage fix files + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_list).sync() + + # stage backgrounds + logger.info(f"Staging backgrounds prescribed from {self.task_config.AERO_BMATRIX_STAGE_TMPL}") + aero_bmat_stage_list = parse_j2yaml(self.task_config.AERO_BMATRIX_STAGE_TMPL, self.task_config) + FileHandler(aero_bmat_stage_list).sync() + + # generate convert state YAML file + logger.info(f"Generate convert state YAML file: {self.task_config.convertstate_yaml}") + self.task_config.convertstate_config = parse_j2yaml(self.task_config.INTERPYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.convertstate_config, self.task_config.convertstate_yaml) + logger.info(f"Wrote convert state YAML to: {self.task_config.convertstate_yaml}") + + # generate diagb YAML file + logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") + self.task_config.bmat_config = parse_j2yaml(self.task_config.BMATYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) + logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") + + # generate diffusion parameters YAML file + logger.info(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") + self.task_config.diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) + logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") + + # link executable to run directory + self.link_bmatexe() + self.link_diffusion_exe() + self.link_jediexe() + + @logit(logger) + def interpBackground(self) -> None: + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('convertstate') + exec_cmd.add_default_arg(self.task_config.convertstate_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeVariance(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdasapp_chem_diagb.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.bmat_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeDiffusion(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROGENB) + exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') + exec_cmd_diffusion.add_default_arg(exec_name_diffusion) + exec_cmd_diffusion.add_default_arg(self.task_config.diffusion_yaml) + + try: + logger.debug(f"Executing {exec_cmd_diffusion}") + exec_cmd_diffusion() + except OSError: + raise OSError(f"Failed to execute {exec_cmd_diffusion}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd_diffusion}") + + pass + + @logit(logger) + def finalize(self) -> None: + super().finalize() + # save files to COMOUT + logger.info(f"Saving files to COMOUT based on {self.task_config.AERO_BMATRIX_FINALIZE_TMPL}") + aero_bmat_finalize_list = parse_j2yaml(self.task_config.AERO_BMATRIX_FINALIZE_TMPL, self.task_config) + FileHandler(aero_bmat_finalize_list).sync() + + @logit(logger) + def link_jediexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.JEDIEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return exe_dest + + @logit(logger) + def link_bmatexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.BMATEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + + @logit(logger) + def link_diffusion_exe(self) -> None: + """ + + This method links a JEDI (fv3jedi_error_covariance_toolbox.x) + executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + + exe_src_diffusion = self.task_config.DIFFUSIONEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src_diffusion} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest_diffusion = os.path.join(self.task_config.DATA, os.path.basename(exe_src_diffusion)) + if os.path.exists(exe_dest_diffusion): + rm_p(exe_dest_diffusion) + os.symlink(exe_src_diffusion, exe_dest_diffusion) + + return + + @logit(logger) + def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: + """Compile a dictionary of model background files to copy + + This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer) + that are needed for global aerosol DA and returns said dictionary for use by the FileHandler class. + + Parameters + ---------- + task_config: Dict + a dictionary containing all of the configuration needed for the task + + Returns + ---------- + bkg_dict: Dict + a dictionary containing the list of model background files to copy for FileHandler + """ + # NOTE for now this is FV3 RESTART files and just assumed to be fh006 + + # get FV3 RESTART files, this will be a lot simpler when using history files + rst_dir = task_config.COM_ATMOS_RESTART_PREV + run_dir = os.path.join(task_config['DATA'], 'bkg') + + # Start accumulating list of background files to copy + bkglist = [] + + # if using IAU, we can use FGAT + bkgtimes = [] + begintime = task_config.previous_cycle + for fcsthr in task_config.aero_bkg_fhr: + bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) + + # now loop over background times + for bkgtime in bkgtimes: + # aerosol DA needs coupler + basename = f'{to_fv3time(bkgtime)}.coupler.res' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + # aerosol DA only needs core/tracer + for ftype in ['core', 'tracer']: + template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + bkg_dict = { + 'mkdir': [run_dir], + 'copy': bkglist, + } + return bkg_dict diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index e407cf17656..0fc07467a05 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -196,7 +196,7 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li @logit(logger) def link_jediexe(self) -> None: - """Compile a dictionary of background error files to copy + """ This method links a JEDI executable to the run directory diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py new file mode 100644 index 00000000000..d0edba2358a --- /dev/null +++ b/ush/python/pygfs/task/bmatrix.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (parse_j2yaml, FileHandler, logit, + Task, Executable, WorkflowException) + +logger = getLogger(__name__.split('.')[-1]) + + +class BMatrix(Task): + """Parent class for GDAS BMatrix tasks + + The BMatrix class is the parent class for all + Global Data Assimilation System (GDAS) BMatrix tasks + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + # Store location of GDASApp jinja2 templates + self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') + + def initialize(self) -> None: + super().initialize() + + def finalize(self) -> None: + super().finalize() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index e049a7d422f..823031ce476 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: configs += ['prepobsaero'] @@ -179,7 +179,7 @@ def get_task_names(self): gdas_tasks += wave_prep_tasks if self.do_aero and 'gdas' in self.aero_anl_runs: - gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gdas_tasks += ['prepobsaero'] @@ -218,7 +218,7 @@ def get_task_names(self): gfs_tasks += wave_prep_tasks if self.do_aero and 'gfs' in self.aero_anl_runs: - gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gfs_tasks += ['prepobsaero'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 1e4fbfc0fae..bc81ded69ab 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -506,13 +506,41 @@ def prepobsaero(self): return task + def aeroanlgenb(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('aeroanlgenb') + task_name = f'{self.run}aeroanlgenb' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': 'gdas_half,gdas', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def aeroanlinit(self): deps = [] + dep_dict = {'type': 'task', 'name': 'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_prep_obs_aero: dep_dict = {'type': 'task', 'name': f'{self.run}prepobsaero'} - deps.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') @@ -532,21 +560,28 @@ def aeroanlinit(self): return task - def aeroanlrun(self): + def aeroanlvar(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlinit'} + dep_dict = { + 'type': 'task', 'name': f'gdasaeroanlgenb', + 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + } deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = { + 'type': 'task', 'name': f'{self.run}aeroanlinit', + } + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('aeroanlrun') - task_name = f'{self.run}aeroanlrun' + resources = self.get_resource('aeroanlvar') + task_name = f'{self.run}aeroanlvar' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -559,7 +594,7 @@ def aeroanlrun(self): def aeroanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlvar'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 72dfba3edf7..0bd9a36282c 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -20,7 +20,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From 5702d3371528faae7c1df233bd6d7792cbbb45f2 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 2 Aug 2024 18:14:53 +0000 Subject: [PATCH 58/69] fix APRUN --- ush/python/pygfs/task/aero_bmatrix.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py index 9431e458022..c652bad558b 100644 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -97,7 +97,7 @@ def initialize(self: BMatrix) -> None: def interpBackground(self) -> None: chdir(self.task_config.DATA) - exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_cmd = Executable(self.task_config.APRUN_AEROANLGENB) exec_name = os.path.join(self.task_config.DATA, 'gdas.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg('fv3jedi') @@ -119,7 +119,7 @@ def computeVariance(self) -> None: chdir(self.task_config.DATA) - exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_cmd = Executable(self.task_config.APRUN_AEROANLGENB) exec_name = os.path.join(self.task_config.DATA, 'gdasapp_chem_diagb.x') exec_cmd.add_default_arg(exec_name) exec_cmd.add_default_arg(self.task_config.bmat_yaml) @@ -139,7 +139,7 @@ def computeDiffusion(self) -> None: chdir(self.task_config.DATA) - exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROGENB) + exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROANLGENB) exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') exec_cmd_diffusion.add_default_arg(exec_name_diffusion) exec_cmd_diffusion.add_default_arg(self.task_config.diffusion_yaml) From d0594ef3505116ca90376872c1faf7d09cee64f9 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 7 Aug 2024 13:43:15 +0000 Subject: [PATCH 59/69] link fix files added in #2806 --- sorc/link_workflow.sh | 2 +- versions/fix.ver | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index dae98bbd652..ca33d21b8fa 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -237,7 +237,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/fix" || exit 1 [[ ! -d gdas ]] && mkdir -p gdas cd gdas || exit 1 - for gdas_sub in fv3jedi gsibec obs soca; do + for gdas_sub in fv3jedi gsibec obs soca aero; do if [[ -d "${gdas_sub}" ]]; then rm -rf "${gdas_sub}" fi diff --git a/versions/fix.ver b/versions/fix.ver index c77b0a22c1f..850214f9006 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -12,6 +12,7 @@ export gdas_fv3jedi_ver=20220805 export gdas_soca_ver=20240624 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 +export gdas_aero_ver=20240806 export glwu_ver=20220805 export gsi_ver=20240208 export lut_ver=20220805 From e8a38d3a7c198ec8d1a6672de4e543f18f9c9f39 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 7 Aug 2024 17:24:38 +0000 Subject: [PATCH 60/69] more robust check of analysis restarts --- ush/forecast_postdet.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index bd15ec8f929..2aff52c06f2 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -37,7 +37,7 @@ FV3_postdet() { fi # Get list of FV3 restart files - local file_list + local file_list file_list=$(FV3_restarts) echo "Copying FV3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'" local fv3_file restart_file @@ -62,16 +62,22 @@ FV3_postdet() { done # Replace fv_tracer with aeroanl_fv_tracer restart files from current cycle (if found) local nn + local use_anl_aero="YES" for (( nn = 1; nn <= ntiles; nn++ )); do - if [[ -f "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" ]]; then - rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" - ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ - "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" - else - echo "WARNING: 'aeroanl_fv_tracer.res.tile1.nc' not found in '${COMOUT_ATMOS_RESTART}', using 'fv_tracer.res.tile1.nc'" + test_tracer_file="${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" + if [[ ! -f "${test_tracer_file}" ]]; then + use_anl_aero="NO" + echo "WARNING: File ${test_tracer_file} does not exist, will not replace any files from the aerosol analysis" break fi done + if [[ use_anl_aero == "YES" ]]; then + for (( nn = 1; nn <= ntiles; nn++ )); do + rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ + "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + done + fi # if [[ use_anl_aero == "YES" ]]; then fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then @@ -262,7 +268,7 @@ FV3_out() { ${NCP} "${DATA}/model_configure" "${COMOUT_CONF}/ufs.model_configure" ${NCP} "${DATA}/ufs.configure" "${COMOUT_CONF}/ufs.ufs.configure" ${NCP} "${DATA}/diag_table" "${COMOUT_CONF}/ufs.diag_table" - + # Determine the dates for restart files to be copied to COM local restart_date restart_dates From f82dc68f585cda20a2ecdbf9cf0841ed57209e94 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 2 Aug 2024 14:03:44 +0000 Subject: [PATCH 61/69] remove CDUMP --- env/HERA.env | 7 +- env/HERCULES.env | 7 +- env/JET.env | 11 +- env/ORION.env | 7 +- env/S4.env | 7 +- env/WCOSS2.env | 7 +- jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX | 46 +++ jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 16 +- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 10 +- ..._RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} | 4 +- jobs/rocoto/aeroanlgenb.sh | 19 ++ jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} | 4 +- parm/config/gfs/config.aeroanl | 24 +- parm/config/gfs/config.aeroanlgenb | 29 ++ parm/config/gfs/config.aeroanlrun | 11 - parm/config/gfs/config.aeroanlvar | 11 + parm/config/gfs/config.com | 1 + parm/config/gfs/config.resources | 54 +++- parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 | 19 ++ parm/gdas/aero_finalize_variational.yaml.j2 | 23 ++ parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 38 +++ parm/gdas/aero_stage_variational.yaml.j2 | 50 +++ parm/ufs/gocart/ExtData.other | 20 +- .../exgdas_aero_analysis_generate_bmatrix.py | 27 ++ ... => exglobal_aero_analysis_variational.py} | 6 +- sorc/link_workflow.sh | 2 + ush/forecast_postdet.sh | 12 + ush/python/pygfs/__init__.py | 2 + ush/python/pygfs/task/aero_analysis.py | 129 +------- ush/python/pygfs/task/aero_bmatrix.py | 294 ++++++++++++++++++ ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/bmatrix.py | 28 ++ workflow/applications/gfs_cycled.py | 6 +- workflow/rocoto/gfs_tasks.py | 51 ++- workflow/rocoto/tasks.py | 2 +- 35 files changed, 795 insertions(+), 191 deletions(-) create mode 100755 jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX rename jobs/{JGLOBAL_AERO_ANALYSIS_RUN => JGLOBAL_AERO_ANALYSIS_VARIATIONAL} (83%) create mode 100755 jobs/rocoto/aeroanlgenb.sh rename jobs/rocoto/{aeroanlrun.sh => aeroanlvar.sh} (83%) create mode 100644 parm/config/gfs/config.aeroanlgenb delete mode 100644 parm/config/gfs/config.aeroanlrun create mode 100644 parm/config/gfs/config.aeroanlvar create mode 100644 parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_finalize_variational.yaml.j2 create mode 100644 parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 create mode 100644 parm/gdas/aero_stage_variational.yaml.j2 create mode 100755 scripts/exgdas_aero_analysis_generate_bmatrix.py rename scripts/{exglobal_aero_analysis_run.py => exglobal_aero_analysis_variational.py} (84%) create mode 100644 ush/python/pygfs/task/aero_bmatrix.py create mode 100644 ush/python/pygfs/task/bmatrix.py diff --git a/env/HERA.env b/env/HERA.env index 3f0e7c9f369..ae82bfac02a 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -82,13 +82,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index 83fa1aadd18..151a2da2518 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -86,12 +86,17 @@ case ${step} in export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" ;; - "aeroanlrun") + "aeroanlvar") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" + ;; + "aeroanlgenb") + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" ;; "prepobsaero") diff --git a/env/JET.env b/env/JET.env index 810a8cd5017..d93d8438fc8 100755 --- a/env/JET.env +++ b/env/JET.env @@ -70,13 +70,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}" -elif [[ "${step}" = "aeroanlrun" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" +elif [[ "${step}" = "aeroanlvar" ]]; then export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/ORION.env b/env/ORION.env index bbbfb59182b..25fba6cfa64 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -78,13 +78,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/S4.env b/env/S4.env index 840ca658981..5b7432104ce 100755 --- a/env/S4.env +++ b/env/S4.env @@ -70,13 +70,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 18caf1bc037..89cc51da974 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -63,13 +63,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX new file mode 100755 index 00000000000..81c89e9155b --- /dev/null +++ b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @@ -0,0 +1,46 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL + +mkdir -p "${COMOUT_CHEM_BMAT}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index 455f572da5e..b894b82531f 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -8,25 +8,17 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan ############################################## # Set variables used in the script ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" - ############################################## # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS - -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +mkdir -p "${COMOUT_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index b2a2893bc04..5be8767308c 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -19,13 +19,15 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COM_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL + COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ + COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +mkdir -p "${COMOUT_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_RUN b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL similarity index 83% rename from jobs/JGLOBAL_AERO_ANALYSIS_RUN rename to jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL index 43749b78c5f..290d7225dd4 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_RUN +++ b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar" ############################################## # Set variables used in the script @@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr ############################################################### # Run relevant script -EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py} +EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh new file mode 100755 index 00000000000..d0bc5dda9b5 --- /dev/null +++ b/jobs/rocoto/aeroanlgenb.sh @@ -0,0 +1,19 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="aeroanlgenb" +export jobid="${job}.$$" + +############################################################### + +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX" +status=$? +exit "${status}" diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlvar.sh similarity index 83% rename from jobs/rocoto/aeroanlrun.sh rename to jobs/rocoto/aeroanlvar.sh index 529bb2d7d1f..7aa7d831f95 100755 --- a/jobs/rocoto/aeroanlrun.sh +++ b/jobs/rocoto/aeroanlvar.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="aeroanlrun" +export job="aeroanlvar" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN" +"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL" status=$? exit "${status}" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 24a5e926448..19766062d9c 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -5,20 +5,36 @@ echo "BEGIN: config.aeroanl" -export CASE_ANL=${CASE} +# define analysis resolution based on deterministic res +case ${CASE} in + "C1152" | "C768" | "C384" | "C192") + CASE_ANL="C192" + ;; + "C96" | "C48") + CASE_ANL=${CASE} + ;; + *) + echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" + exit 4 +esac +export CASE_ANL export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" -export STATICB_TYPE='identity' +export STATICB_TYPE='diffusion' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" -export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" -export BERROR_DATE="20160630.000000" +export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" +export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2" +export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2" + export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" +export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" +export DIFFUSIONEXE="${EXECgfs}/gdas_fv3jedi_error_covariance_toolbox.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb new file mode 100644 index 00000000000..b41b22a5243 --- /dev/null +++ b/parm/config/gfs/config.aeroanlgenb @@ -0,0 +1,29 @@ +#!/bin/bash -x + +########## config.aeroanlgenb ########## +# Aerosol Variance specific + +echo "BEGIN: config.aeroanlgenb" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlgenb + +export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" +export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" +export INTERPYAML="${PARMgfs}/gdas/aero/berror/aero_interp.yaml.j2" +export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" +export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2" +export aero_diffusion_iter=10 +export aero_diffusion_horiz_len=2500e3 +export aero_diffusion_fixed_val=1.0 +export npx_clim_b=97 +export npy_clim_b=97 +export aero_diagb_weight=0.9 +export aero_staticb_rescaling_factor=2.0 +export aero_diagb_rescale=20.0 +export aero_diagb_n_halo=4 +export aero_diagb_n_neighbors=16 +export aero_diagb_smooth_horiz_iter=0 +export aero_diagb_smooth_vert_iter=0 + +echo "END: config.aeroanlgenb" diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun deleted file mode 100644 index 012e5b79f3b..00000000000 --- a/parm/config/gfs/config.aeroanlrun +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -x - -########## config.aeroanlrun ########## -# Aerosol Analysis specific - -echo "BEGIN: config.aeroanlrun" - -# Get task specific resources -source "${EXPDIR}/config.resources" aeroanlrun - -echo "END: config.aeroanlrun" diff --git a/parm/config/gfs/config.aeroanlvar b/parm/config/gfs/config.aeroanlvar new file mode 100644 index 00000000000..4282b6c840b --- /dev/null +++ b/parm/config/gfs/config.aeroanlvar @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlvar ########## +# Aerosol Analysis specific + +echo "BEGIN: config.aeroanlvar" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlvar + +echo "END: config.aeroanlvar" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 222ffdae95d..818ea380179 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -98,5 +98,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' +declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index a596629e76d..b84784e1833 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -16,7 +16,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl" - echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -355,12 +355,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") - layout_x=8 - layout_y=8 + layout_x=4 + layout_y=4 ;; "C48" ) # this case is for testing only @@ -381,27 +381,61 @@ case ${step} in memory="3072M" ;; - "aeroanlrun") + "aeroanlvar") case ${CASE} in "C768") layout_x=8 layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") + layout_x=4 + layout_y=4 + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "aeroanlgenb") + case ${CASE} in + "C768") layout_x=8 layout_y=8 ;; + "C384") + layout_x=6 + layout_y=6 + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + ;; "C48" ) # this case is for testing only layout_x=1 layout_y=1 ;; *) - echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 esac @@ -413,8 +447,10 @@ case ${step} in threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True + ;; + "aeroanlfinal") walltime="00:10:00" ntasks=1 diff --git a/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 new file mode 100644 index 00000000000..b33f2809454 --- /dev/null +++ b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 @@ -0,0 +1,19 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set HEAD = RUN + ".t" + cycle_HH + "z." %} +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +copy: +### copy YAMLs used +{% set yaml_list = ['chem_diagb.yaml', 'chem_diffusion.yaml'] %} +{% for fname in yaml_list %} +- ["{{ DATA }}/{{ HEAD }}{{ fname }}", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}{{ fname }}"] +{% endfor %} +### copy stddev files to ROTDIR +{% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.coupler.res", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ DATA }}/diffusion/diffusion_hz.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_hz.nc"] +- ["{{ DATA }}/diffusion/diffusion_vt.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_vt.nc"] diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 new file mode 100644 index 00000000000..b9247bcd620 --- /dev/null +++ b/parm/gdas/aero_finalize_variational.yaml.j2 @@ -0,0 +1,23 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkgtime = AERO_WINDOW_BEGIN %} +{% else %} + {% set bkgtime = current_cycle %} +{% endif %} +###################################### +mkdir: +- "{{ COMOUT_CHEM_ANALYSIS }}" +- "{{ COMOUT_ATMOS_RESTART }}" +copy: +## copy variational YAML to ROTDIR +- ["{{ DATA }}/{{ APREFIX }}aerovar.yaml", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aerovar.yaml"] +## copy increments +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_ANALYSIS }}/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +## copy analysis +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 new file mode 100644 index 00000000000..9005b9ff127 --- /dev/null +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -0,0 +1,38 @@ +###################################### +# set some variables +###################################### +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +{% set ftype_list = ['fv_core.res', 'fv_tracer.res'] %} +###################################### +# create working directories +###################################### +mkdir: +- "{{ DATA }}/bkg" +- "{{ DATA }}/stddev" +- "{{ DATA }}/clm_stddev" +- "{{ DATA }}/diffusion" +copy: +###################################### +# copy deterministic background files +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle | to_YMD, + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':""} %} + +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.coupler.res"] +{% for ftype in ftype_list %} + {% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] + {% endfor %} +{% endfor %} +# copy climatological stddev files +###################################### +{% for tile in range(1, ntiles+1) %} +- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 new file mode 100644 index 00000000000..afd0e1b9461 --- /dev/null +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -0,0 +1,50 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkg_times = [] %} + {% for fh in range(0, 7, 3) %} + {% set offset = fh | string + "H" %} + {% set fcst_timedelta = offset | to_timedelta %} + {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} + {% do bkg_times.append(fcst_time) %} + {% endfor %} +{% else %} + {% set bkg_times = [] %} + {% do bkg_times.append(current_cycle) %} +{% endif %} +{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} +###################################### +mkdir: +- "{{ DATA }}/anl" +- "{{ DATA }}/diags" +- "{{ DATA }}/berror" +- "{{ DATA }}/bkg" +copy: +###################################### +## copy backgrounds +{% for bkgtime in bkg_times %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.coupler.res"] + {% for fvfile in fvfiles %} + {% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] + {% endfor %} + {% endfor %} +{% endfor %} +###################################### +## copy backgrounds again for fv_tracer to create analysis files later +{% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + +###################################### +## copy berror files from COMIN_CHEM_BMAT_PREV +## stddev files +{% for tile in range(1, ntiles+1) %} +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 7a0d63d6ca2..5d2ddc51021 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL1 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO4 NA Y Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -63,11 +63,11 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC1 NA Y Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC2 NA Y Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null @@ -88,11 +88,11 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC1 NA Y Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC2 NA Y Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null diff --git a/scripts/exgdas_aero_analysis_generate_bmatrix.py b/scripts/exgdas_aero_analysis_generate_bmatrix.py new file mode 100755 index 00000000000..0d8389c40d3 --- /dev/null +++ b/scripts/exgdas_aero_analysis_generate_bmatrix.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# exgdas_aero_analysis_generate_bmatrix.py +# This script creates an AerosolBMatrix object +# and runs the methods needed +# to stage files, compute the variance, and write to com +# files needed for the variational solver +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.aero_bmatrix import AerosolBMatrix + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the aerosol variance and diffusion correlation tasks + AeroB = AerosolBMatrix(config) + AeroB.initialize() + AeroB.interpBackground() + AeroB.computeVariance() + AeroB.computeDiffusion() + AeroB.finalize() diff --git a/scripts/exglobal_aero_analysis_run.py b/scripts/exglobal_aero_analysis_variational.py similarity index 84% rename from scripts/exglobal_aero_analysis_run.py rename to scripts/exglobal_aero_analysis_variational.py index 85f4b963a4f..dd5bb4f65a6 100755 --- a/scripts/exglobal_aero_analysis_run.py +++ b/scripts/exglobal_aero_analysis_variational.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# exglobal_aero_analysis_run.py +# exglobal_aero_analysis_variational.py # This script creates an AerosolAnalysis object -# and runs the execute method +# and runs the variational method # which executes the global aerosol variational analysis import os @@ -19,4 +19,4 @@ # Instantiate the aerosol analysis task AeroAnl = AerosolAnalysis(config) - AeroAnl.execute() + AeroAnl.variational() diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index be912292fee..dae98bbd652 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -367,9 +367,11 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then declare -a JEDI_EXE=("gdas.x" \ "gdas_soca_gridgen.x" \ "gdas_soca_error_covariance_toolbox.x" \ + "gdas_fv3jedi_error_covariance_toolbox.x" \ "gdas_soca_setcorscales.x" \ "gdas_soca_diagb.x" \ "fv3jedi_plot_field.x" \ + "gdasapp_chem_diagb.x" \ "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 7de31d6235b..bd15ec8f929 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -60,6 +60,18 @@ FV3_postdet() { break fi done + # Replace fv_tracer with aeroanl_fv_tracer restart files from current cycle (if found) + local nn + for (( nn = 1; nn <= ntiles; nn++ )); do + if [[ -f "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" ]]; then + rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ + "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + else + echo "WARNING: 'aeroanl_fv_tracer.res.tile1.nc' not found in '${COMOUT_ATMOS_RESTART}', using 'fv_tracer.res.tile1.nc'" + break + fi + done fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index c0b72bbc352..e1ff7e68624 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,8 +2,10 @@ import os from .task.analysis import Analysis +from .task.bmatrix import BMatrix from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis +from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis from .task.atmens_analysis import AtmEnsAnalysis from .task.marine_bmat import MarineBMat diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 69a992d7d47..2a22bd56328 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -82,29 +82,18 @@ def initialize(self: Analysis) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage berror files - # copy BUMP files, otherwise it will assume ID matrix - if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: - FileHandler(self.get_berror_dict(self.task_config)).sync() - - # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + # stage files from COM and create working directories + logger.info(f"Staging files prescribed from {self.task_config.AERO_STAGE_VARIATIONAL_TMPL}") + aero_var_stage_list = parse_j2yaml(self.task_config.AERO_STAGE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_stage_list).sync() # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config['DATA'], 'anl'), - os.path.join(self.task_config['DATA'], 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self: Analysis) -> None: + def variational(self: Analysis) -> None: chdir(self.task_config.DATA) @@ -140,13 +129,15 @@ def finalize(self: Analysis) -> None: """ # ---- tar up diags # path of output tar statfile - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + logger.info('Preparing observation space diagnostics for archiving') + aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") # get list of diag files to put in tarball diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) # gzip the files first for diagfile in diags: + logger.info(f'Adding {diagfile} to tar file') with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) @@ -155,44 +146,16 @@ def finalize(self: Analysis) -> None: for diagfile in diags: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - yaml_copy = { - 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() - - # ---- copy RESTART fv_tracer files for future reference - if self.task_config.DOIAU: - bkgtime = self.task_config.AERO_WINDOW_BEGIN - else: - bkgtime = self.task_config.current_cycle - template = '{}.fv_tracer.res.tile{}.nc'.format(to_fv3time(bkgtime), '{tilenum}') - bkglist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}') - bkglist.append([src, dest]) - FileHandler({'copy': bkglist}).sync() + logger.info(f'Saved diags to {aerostat}') # ---- add increments to RESTART files logger.info('Adding increments to RESTART files') self._add_fms_cube_sphere_increments() - # ---- move increments to ROTDIR - logger.info('Moving increments to ROTDIR') - template = f'aeroinc.{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' - inclist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + # copy files back to COM + logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}") + aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_final_list).sync() def clean(self): super().clean() @@ -209,7 +172,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, restart_template) + bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] @@ -232,38 +195,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkg_dict: Dict a dictionary containing the list of model background files to copy for FileHandler """ - # NOTE for now this is FV3 RESTART files and just assumed to be fh006 - - # get FV3 RESTART files, this will be a lot simpler when using history files - rst_dir = task_config.COM_ATMOS_RESTART_PREV - run_dir = os.path.join(task_config['DATA'], 'bkg') - - # Start accumulating list of background files to copy - bkglist = [] - - # if using IAU, we can use FGAT - bkgtimes = [] - begintime = task_config.previous_cycle - for fcsthr in task_config.aero_bkg_fhr: - bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) - - # now loop over background times - for bkgtime in bkgtimes: - # aerosol DA needs coupler - basename = f'{to_fv3time(bkgtime)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - # aerosol DA only needs core/tracer - for ftype in ['core', 'tracer']: - template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - bkg_dict = { - 'mkdir': [run_dir], - 'copy': bkglist, - } + bkg_dict = {} return bkg_dict @logit(logger) @@ -285,34 +217,5 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_dict: Dict a dictionary containing the list of background error files to copy for FileHandler """ - # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. - b_dir = config.BERROR_DATA_DIR - b_datestr = to_fv3time(config.BERROR_DATE) - berror_list = [] - - for ftype in ['stddev']: - coupler = f'{b_datestr}.{ftype}.coupler.res' - berror_list.append([ - os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) - ]) - template = f'{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' - for itile in range(1, config.ntiles + 1): - tracer = template.format(tilenum=itile) - berror_list.append([ - os.path.join(b_dir, tracer), os.path.join(config.DATA, 'berror', tracer) - ]) - radius = 'cor_aero_universe_radius' - berror_list.append([ - os.path.join(b_dir, radius), os.path.join(config.DATA, 'berror', radius) - ]) - nproc = config.ntiles * config.layout_x * config.layout_y - for nn in range(1, nproc + 1): - berror_list.append([ - os.path.join(b_dir, f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc'), - os.path.join(config.DATA, 'berror', f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc') - ]) - berror_dict = { - 'mkdir': [os.path.join(config.DATA, 'berror')], - 'copy': berror_list, - } + berror_dict = {} return berror_dict diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py new file mode 100644 index 00000000000..9431e458022 --- /dev/null +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (AttrDict, FileHandler, rm_p, + add_to_datetime, to_fv3time, to_timedelta, + to_fv3time, chdir, Executable, WorkflowException, + parse_j2yaml, save_as_yaml, logit) +from pygfs.task.bmatrix import BMatrix + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolBMatrix(BMatrix): + """ + Class for global aerosol BMatrix tasks + """ + @logit(logger, name="AerosolBMatrix") + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + _res = int(self.task_config['CASE'][1:]) + _res_anl = int(self.task_config['CASE_ANL'][1:]) + + _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") + _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") + _convertstate_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_convertstate.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config['LEVS'] - 1, + 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'bmat_yaml': _bmat_yaml, + 'diffusion_yaml': _diffusion_yaml, + 'convertstate_yaml': _convertstate_yaml, + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.task_config, **local_dict) + + @logit(logger) + def initialize(self: BMatrix) -> None: + super().initialize() + # stage fix files + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_list).sync() + + # stage backgrounds + logger.info(f"Staging backgrounds prescribed from {self.task_config.AERO_BMATRIX_STAGE_TMPL}") + aero_bmat_stage_list = parse_j2yaml(self.task_config.AERO_BMATRIX_STAGE_TMPL, self.task_config) + FileHandler(aero_bmat_stage_list).sync() + + # generate convert state YAML file + logger.info(f"Generate convert state YAML file: {self.task_config.convertstate_yaml}") + self.task_config.convertstate_config = parse_j2yaml(self.task_config.INTERPYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.convertstate_config, self.task_config.convertstate_yaml) + logger.info(f"Wrote convert state YAML to: {self.task_config.convertstate_yaml}") + + # generate diagb YAML file + logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") + self.task_config.bmat_config = parse_j2yaml(self.task_config.BMATYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) + logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") + + # generate diffusion parameters YAML file + logger.info(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") + self.task_config.diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) + logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") + + # link executable to run directory + self.link_bmatexe() + self.link_diffusion_exe() + self.link_jediexe() + + @logit(logger) + def interpBackground(self) -> None: + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('convertstate') + exec_cmd.add_default_arg(self.task_config.convertstate_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeVariance(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdasapp_chem_diagb.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.bmat_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeDiffusion(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROGENB) + exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') + exec_cmd_diffusion.add_default_arg(exec_name_diffusion) + exec_cmd_diffusion.add_default_arg(self.task_config.diffusion_yaml) + + try: + logger.debug(f"Executing {exec_cmd_diffusion}") + exec_cmd_diffusion() + except OSError: + raise OSError(f"Failed to execute {exec_cmd_diffusion}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd_diffusion}") + + pass + + @logit(logger) + def finalize(self) -> None: + super().finalize() + # save files to COMOUT + logger.info(f"Saving files to COMOUT based on {self.task_config.AERO_BMATRIX_FINALIZE_TMPL}") + aero_bmat_finalize_list = parse_j2yaml(self.task_config.AERO_BMATRIX_FINALIZE_TMPL, self.task_config) + FileHandler(aero_bmat_finalize_list).sync() + + @logit(logger) + def link_jediexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.JEDIEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return exe_dest + + @logit(logger) + def link_bmatexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.BMATEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + + @logit(logger) + def link_diffusion_exe(self) -> None: + """ + + This method links a JEDI (fv3jedi_error_covariance_toolbox.x) + executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + + exe_src_diffusion = self.task_config.DIFFUSIONEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src_diffusion} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest_diffusion = os.path.join(self.task_config.DATA, os.path.basename(exe_src_diffusion)) + if os.path.exists(exe_dest_diffusion): + rm_p(exe_dest_diffusion) + os.symlink(exe_src_diffusion, exe_dest_diffusion) + + return + + @logit(logger) + def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: + """Compile a dictionary of model background files to copy + + This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer) + that are needed for global aerosol DA and returns said dictionary for use by the FileHandler class. + + Parameters + ---------- + task_config: Dict + a dictionary containing all of the configuration needed for the task + + Returns + ---------- + bkg_dict: Dict + a dictionary containing the list of model background files to copy for FileHandler + """ + # NOTE for now this is FV3 RESTART files and just assumed to be fh006 + + # get FV3 RESTART files, this will be a lot simpler when using history files + rst_dir = task_config.COM_ATMOS_RESTART_PREV + run_dir = os.path.join(task_config['DATA'], 'bkg') + + # Start accumulating list of background files to copy + bkglist = [] + + # if using IAU, we can use FGAT + bkgtimes = [] + begintime = task_config.previous_cycle + for fcsthr in task_config.aero_bkg_fhr: + bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) + + # now loop over background times + for bkgtime in bkgtimes: + # aerosol DA needs coupler + basename = f'{to_fv3time(bkgtime)}.coupler.res' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + # aerosol DA only needs core/tracer + for ftype in ['core', 'tracer']: + template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + bkg_dict = { + 'mkdir': [run_dir], + 'copy': bkglist, + } + return bkg_dict diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index e407cf17656..0fc07467a05 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -196,7 +196,7 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li @logit(logger) def link_jediexe(self) -> None: - """Compile a dictionary of background error files to copy + """ This method links a JEDI executable to the run directory diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py new file mode 100644 index 00000000000..d0edba2358a --- /dev/null +++ b/ush/python/pygfs/task/bmatrix.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (parse_j2yaml, FileHandler, logit, + Task, Executable, WorkflowException) + +logger = getLogger(__name__.split('.')[-1]) + + +class BMatrix(Task): + """Parent class for GDAS BMatrix tasks + + The BMatrix class is the parent class for all + Global Data Assimilation System (GDAS) BMatrix tasks + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + # Store location of GDASApp jinja2 templates + self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') + + def initialize(self) -> None: + super().initialize() + + def finalize(self) -> None: + super().finalize() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index e049a7d422f..823031ce476 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: configs += ['prepobsaero'] @@ -179,7 +179,7 @@ def get_task_names(self): gdas_tasks += wave_prep_tasks if self.do_aero and 'gdas' in self.aero_anl_runs: - gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gdas_tasks += ['prepobsaero'] @@ -218,7 +218,7 @@ def get_task_names(self): gfs_tasks += wave_prep_tasks if self.do_aero and 'gfs' in self.aero_anl_runs: - gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gfs_tasks += ['prepobsaero'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 960a7548abd..72f1f78edfb 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -506,13 +506,41 @@ def prepobsaero(self): return task + def aeroanlgenb(self): + + deps = [] + dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('aeroanlgenb') + task_name = f'{self.run}aeroanlgenb' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': 'gdas_half,gdas', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def aeroanlinit(self): deps = [] + dep_dict = {'type': 'task', 'name': 'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_prep_obs_aero: dep_dict = {'type': 'task', 'name': f'{self.run}prepobsaero'} - deps.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') @@ -532,21 +560,28 @@ def aeroanlinit(self): return task - def aeroanlrun(self): + def aeroanlvar(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlinit'} + dep_dict = { + 'type': 'task', 'name': f'gdasaeroanlgenb', + 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + } deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = { + 'type': 'task', 'name': f'{self.run}aeroanlinit', + } + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('aeroanlrun') - task_name = f'{self.run}aeroanlrun' + resources = self.get_resource('aeroanlvar') + task_name = f'{self.run}aeroanlvar' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -559,7 +594,7 @@ def aeroanlrun(self): def aeroanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlvar'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 353d2aa943c..3b2fbf5420d 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -20,7 +20,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From 4a14a71a4f5385fde85208fe26c773d87e0d226d Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 7 Aug 2024 19:40:57 +0000 Subject: [PATCH 62/69] Revert "remove CDUMP" This reverts commit f82dc68f585cda20a2ecdbf9cf0841ed57209e94. --- env/HERA.env | 7 +- env/HERCULES.env | 7 +- env/JET.env | 11 +- env/ORION.env | 7 +- env/S4.env | 7 +- env/WCOSS2.env | 7 +- jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX | 46 --- jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 16 +- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 10 +- ..._VARIATIONAL => JGLOBAL_AERO_ANALYSIS_RUN} | 4 +- jobs/rocoto/aeroanlgenb.sh | 19 -- jobs/rocoto/{aeroanlvar.sh => aeroanlrun.sh} | 4 +- parm/config/gfs/config.aeroanl | 24 +- parm/config/gfs/config.aeroanlgenb | 29 -- parm/config/gfs/config.aeroanlrun | 11 + parm/config/gfs/config.aeroanlvar | 11 - parm/config/gfs/config.com | 1 - parm/config/gfs/config.resources | 54 +--- parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 | 19 -- parm/gdas/aero_finalize_variational.yaml.j2 | 23 -- parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 | 38 --- parm/gdas/aero_stage_variational.yaml.j2 | 50 --- parm/ufs/gocart/ExtData.other | 20 +- .../exgdas_aero_analysis_generate_bmatrix.py | 27 -- ...ional.py => exglobal_aero_analysis_run.py} | 6 +- sorc/link_workflow.sh | 2 - ush/forecast_postdet.sh | 12 - ush/python/pygfs/__init__.py | 2 - ush/python/pygfs/task/aero_analysis.py | 129 +++++++- ush/python/pygfs/task/aero_bmatrix.py | 294 ------------------ ush/python/pygfs/task/analysis.py | 2 +- ush/python/pygfs/task/bmatrix.py | 28 -- workflow/applications/gfs_cycled.py | 6 +- workflow/rocoto/gfs_tasks.py | 51 +-- workflow/rocoto/tasks.py | 2 +- 35 files changed, 191 insertions(+), 795 deletions(-) delete mode 100755 jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX rename jobs/{JGLOBAL_AERO_ANALYSIS_VARIATIONAL => JGLOBAL_AERO_ANALYSIS_RUN} (83%) delete mode 100755 jobs/rocoto/aeroanlgenb.sh rename jobs/rocoto/{aeroanlvar.sh => aeroanlrun.sh} (83%) delete mode 100644 parm/config/gfs/config.aeroanlgenb create mode 100644 parm/config/gfs/config.aeroanlrun delete mode 100644 parm/config/gfs/config.aeroanlvar delete mode 100644 parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 delete mode 100644 parm/gdas/aero_finalize_variational.yaml.j2 delete mode 100644 parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 delete mode 100644 parm/gdas/aero_stage_variational.yaml.j2 delete mode 100755 scripts/exgdas_aero_analysis_generate_bmatrix.py rename scripts/{exglobal_aero_analysis_variational.py => exglobal_aero_analysis_run.py} (84%) delete mode 100644 ush/python/pygfs/task/aero_bmatrix.py delete mode 100644 ush/python/pygfs/task/bmatrix.py diff --git a/env/HERA.env b/env/HERA.env index ae82bfac02a..3f0e7c9f369 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -82,18 +82,13 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlvar" ]]; then +elif [[ "${step}" = "aeroanlrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" -elif [[ "${step}" = "aeroanlgenb" ]]; then - - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" - elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index 151a2da2518..83fa1aadd18 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -86,17 +86,12 @@ case ${step} in export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" ;; - "aeroanlvar") + "aeroanlrun") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" - ;; - "aeroanlgenb") - - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" ;; "prepobsaero") diff --git a/env/JET.env b/env/JET.env index d93d8438fc8..810a8cd5017 100755 --- a/env/JET.env +++ b/env/JET.env @@ -70,17 +70,12 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}" -elif [[ "${step}" = "aeroanlvar" ]]; then - - export NTHREADS_AEROANL=${NTHREADSmax} - export APRUN_AEROANL="${APRUN}" - -elif [[ "${step}" = "aeroanlgenb" ]]; then +elif [[ "${step}" = "aeroanlrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + export NTHREADS_AEROANL=${NTHREADSmax} + export APRUN_AEROANL="${APRUN}" elif [[ "${step}" = "prepobsaero" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 25fba6cfa64..bbbfb59182b 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -78,18 +78,13 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlvar" ]]; then +elif [[ "${step}" = "aeroanlrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" -elif [[ "${step}" = "aeroanlgenb" ]]; then - - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" - elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/S4.env b/env/S4.env index 5b7432104ce..840ca658981 100755 --- a/env/S4.env +++ b/env/S4.env @@ -70,18 +70,13 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlvar" ]]; then +elif [[ "${step}" = "aeroanlrun" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" -elif [[ "${step}" = "aeroanlgenb" ]]; then - - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" - elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index 89cc51da974..18caf1bc037 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -63,18 +63,13 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlvar" ]]; then +elif [[ "${step}" = "aeroanlrun" ]]; then export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" -elif [[ "${step}" = "aeroanlgenb" ]]; then - - export NTHREADS_AEROANLGENB=${NTHREADSmax} - export APRUN_AEROANLGENB="${APRUN}" - elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX deleted file mode 100755 index 81c89e9155b..00000000000 --- a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX +++ /dev/null @@ -1,46 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" - -############################################## -# Set variables used in the script -############################################## - -############################################## -# Begin JOB SPECIFIC work -############################################## - -# Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ - COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ - COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL - -mkdir -p "${COMOUT_CHEM_BMAT}" - -############################################################### -# Run relevant script - -EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py} -${EXSCRIPT} -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -############################################## -# End JOB SPECIFIC work -############################################## - -############################################## -# Final processing -############################################## -if [[ -e "${pgmout}" ]] ; then - cat "${pgmout}" -fi - -########################################## -# Remove the Temporary working directory -########################################## -cd "${DATAROOT}" || exit 1 -[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" - -exit 0 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index b894b82531f..455f572da5e 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -8,17 +8,25 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan ############################################## # Set variables used in the script ############################################## +# shellcheck disable=SC2153 +GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") +gPDY=${GDATE:0:8} +gcyc=${GDATE:8:2} +GDUMP="gdas" + ############################################## # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ - COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS + +RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ + COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL -mkdir -p "${COMOUT_CHEM_ANALYSIS}" +mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index 5be8767308c..b2a2893bc04 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -19,15 +19,13 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ - COM_OBS:COM_OBS_TMPL \ - COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ - COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL + COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ + COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL -mkdir -p "${COMOUT_CHEM_ANALYSIS}" +mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL b/jobs/JGLOBAL_AERO_ANALYSIS_RUN similarity index 83% rename from jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL rename to jobs/JGLOBAL_AERO_ANALYSIS_RUN index 290d7225dd4..43749b78c5f 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL +++ b/jobs/JGLOBAL_AERO_ANALYSIS_RUN @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun" ############################################## # Set variables used in the script @@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlv ############################################################### # Run relevant script -EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py} +EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh deleted file mode 100755 index d0bc5dda9b5..00000000000 --- a/jobs/rocoto/aeroanlgenb.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env bash - -source "${HOMEgfs}/ush/preamble.sh" - -############################################################### -# Source UFSDA workflow modules -. "${HOMEgfs}/ush/load_ufsda_modules.sh" -status=$? -[[ ${status} -ne 0 ]] && exit "${status}" - -export job="aeroanlgenb" -export jobid="${job}.$$" - -############################################################### - -# Execute the JJOB -"${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX" -status=$? -exit "${status}" diff --git a/jobs/rocoto/aeroanlvar.sh b/jobs/rocoto/aeroanlrun.sh similarity index 83% rename from jobs/rocoto/aeroanlvar.sh rename to jobs/rocoto/aeroanlrun.sh index 7aa7d831f95..529bb2d7d1f 100755 --- a/jobs/rocoto/aeroanlvar.sh +++ b/jobs/rocoto/aeroanlrun.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="aeroanlvar" +export job="aeroanlrun" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL" +"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN" status=$? exit "${status}" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index 19766062d9c..24a5e926448 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -5,36 +5,20 @@ echo "BEGIN: config.aeroanl" -# define analysis resolution based on deterministic res -case ${CASE} in - "C1152" | "C768" | "C384" | "C192") - CASE_ANL="C192" - ;; - "C96" | "C48") - CASE_ANL=${CASE} - ;; - *) - echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" - exit 4 -esac -export CASE_ANL +export CASE_ANL=${CASE} export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" -export STATICB_TYPE='diffusion' +export STATICB_TYPE='identity' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" -export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" +export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" +export BERROR_DATE="20160630.000000" export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" -export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2" -export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2" - export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" -export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" -export DIFFUSIONEXE="${EXECgfs}/gdas_fv3jedi_error_covariance_toolbox.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb deleted file mode 100644 index b41b22a5243..00000000000 --- a/parm/config/gfs/config.aeroanlgenb +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -x - -########## config.aeroanlgenb ########## -# Aerosol Variance specific - -echo "BEGIN: config.aeroanlgenb" - -# Get task specific resources -source "${EXPDIR}/config.resources" aeroanlgenb - -export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" -export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" -export INTERPYAML="${PARMgfs}/gdas/aero/berror/aero_interp.yaml.j2" -export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" -export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2" -export aero_diffusion_iter=10 -export aero_diffusion_horiz_len=2500e3 -export aero_diffusion_fixed_val=1.0 -export npx_clim_b=97 -export npy_clim_b=97 -export aero_diagb_weight=0.9 -export aero_staticb_rescaling_factor=2.0 -export aero_diagb_rescale=20.0 -export aero_diagb_n_halo=4 -export aero_diagb_n_neighbors=16 -export aero_diagb_smooth_horiz_iter=0 -export aero_diagb_smooth_vert_iter=0 - -echo "END: config.aeroanlgenb" diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun new file mode 100644 index 00000000000..012e5b79f3b --- /dev/null +++ b/parm/config/gfs/config.aeroanlrun @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlrun ########## +# Aerosol Analysis specific + +echo "BEGIN: config.aeroanlrun" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlrun + +echo "END: config.aeroanlrun" diff --git a/parm/config/gfs/config.aeroanlvar b/parm/config/gfs/config.aeroanlvar deleted file mode 100644 index 4282b6c840b..00000000000 --- a/parm/config/gfs/config.aeroanlvar +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -x - -########## config.aeroanlvar ########## -# Aerosol Analysis specific - -echo "BEGIN: config.aeroanlvar" - -# Get task specific resources -source "${EXPDIR}/config.resources" aeroanlvar - -echo "END: config.aeroanlvar" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 818ea380179..222ffdae95d 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -98,6 +98,5 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model_data/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' -declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model_data/med/restart' diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index b84784e1833..a596629e76d 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -16,7 +16,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl" - echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -355,12 +355,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=6 - layout_y=6 + layout_x=8 + layout_y=8 ;; "C192" | "C96") - layout_x=4 - layout_y=4 + layout_x=8 + layout_y=8 ;; "C48" ) # this case is for testing only @@ -381,53 +381,19 @@ case ${step} in memory="3072M" ;; - "aeroanlvar") + "aeroanlrun") case ${CASE} in "C768") layout_x=8 layout_y=8 ;; "C384") - layout_x=6 - layout_y=6 - ;; - "C192" | "C96") - layout_x=4 - layout_y=4 - ;; - "C48" ) - # this case is for testing only - layout_x=1 - layout_y=1 - ;; - *) - echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" - exit 4 - esac - - export layout_x - export layout_y - - walltime="00:30:00" - ntasks=$(( layout_x * layout_y * 6 )) - threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) - export is_exclusive=True - ;; - - "aeroanlgenb") - case ${CASE} in - "C768") layout_x=8 layout_y=8 ;; - "C384") - layout_x=6 - layout_y=6 - ;; "C192" | "C96") - layout_x=4 - layout_y=4 + layout_x=8 + layout_y=8 ;; "C48" ) # this case is for testing only @@ -435,7 +401,7 @@ case ${step} in layout_y=1 ;; *) - echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" exit 4 esac @@ -447,10 +413,8 @@ case ${step} in threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True - ;; - "aeroanlfinal") walltime="00:10:00" ntasks=1 diff --git a/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 deleted file mode 100644 index b33f2809454..00000000000 --- a/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 +++ /dev/null @@ -1,19 +0,0 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set HEAD = RUN + ".t" + cycle_HH + "z." %} -{% set offset_td = "+6H" | to_timedelta %} -{% set background_time = current_cycle | add_to_datetime(offset_td) %} -copy: -### copy YAMLs used -{% set yaml_list = ['chem_diagb.yaml', 'chem_diffusion.yaml'] %} -{% for fname in yaml_list %} -- ["{{ DATA }}/{{ HEAD }}{{ fname }}", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}{{ fname }}"] -{% endfor %} -### copy stddev files to ROTDIR -{% for tile in range(1, ntiles+1) %} -- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} -### copy coupler file -- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.coupler.res", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.coupler.res"] -### copy diffusion files -- ["{{ DATA }}/diffusion/diffusion_hz.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_hz.nc"] -- ["{{ DATA }}/diffusion/diffusion_vt.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_vt.nc"] diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 deleted file mode 100644 index b9247bcd620..00000000000 --- a/parm/gdas/aero_finalize_variational.yaml.j2 +++ /dev/null @@ -1,23 +0,0 @@ -###################################### -# set some variables -###################################### -{% if DOIAU == True %} - {% set bkgtime = AERO_WINDOW_BEGIN %} -{% else %} - {% set bkgtime = current_cycle %} -{% endif %} -###################################### -mkdir: -- "{{ COMOUT_CHEM_ANALYSIS }}" -- "{{ COMOUT_ATMOS_RESTART }}" -copy: -## copy variational YAML to ROTDIR -- ["{{ DATA }}/{{ APREFIX }}aerovar.yaml", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aerovar.yaml"] -## copy increments -{% for tile in range(1,ntiles+1) %} -- ["{{ DATA }}/anl/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_ANALYSIS }}/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} -## copy analysis -{% for tile in range(1,ntiles+1) %} -- ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 deleted file mode 100644 index 9005b9ff127..00000000000 --- a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 +++ /dev/null @@ -1,38 +0,0 @@ -###################################### -# set some variables -###################################### -{% set offset_td = "+6H" | to_timedelta %} -{% set background_time = current_cycle | add_to_datetime(offset_td) %} -{% set ftype_list = ['fv_core.res', 'fv_tracer.res'] %} -###################################### -# create working directories -###################################### -mkdir: -- "{{ DATA }}/bkg" -- "{{ DATA }}/stddev" -- "{{ DATA }}/clm_stddev" -- "{{ DATA }}/diffusion" -copy: -###################################### -# copy deterministic background files -###################################### -# define variables -# Declare a dict of search and replace terms to run on each template -{% set tmpl_dict = {'${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':current_cycle | to_YMD, - '${HH}':current_cycle | strftime("%H"), - '${MEMDIR}':""} %} - -- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.coupler.res"] -{% for ftype in ftype_list %} - {% for tile in range(1, ntiles+1) %} -- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] - {% endfor %} -{% endfor %} -# copy climatological stddev files -###################################### -{% for tile in range(1, ntiles+1) %} -- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} - diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 deleted file mode 100644 index afd0e1b9461..00000000000 --- a/parm/gdas/aero_stage_variational.yaml.j2 +++ /dev/null @@ -1,50 +0,0 @@ -###################################### -# set some variables -###################################### -{% if DOIAU == True %} - {% set bkg_times = [] %} - {% for fh in range(0, 7, 3) %} - {% set offset = fh | string + "H" %} - {% set fcst_timedelta = offset | to_timedelta %} - {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} - {% do bkg_times.append(fcst_time) %} - {% endfor %} -{% else %} - {% set bkg_times = [] %} - {% do bkg_times.append(current_cycle) %} -{% endif %} -{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} -###################################### -mkdir: -- "{{ DATA }}/anl" -- "{{ DATA }}/diags" -- "{{ DATA }}/berror" -- "{{ DATA }}/bkg" -copy: -###################################### -## copy backgrounds -{% for bkgtime in bkg_times %} -- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.coupler.res"] - {% for fvfile in fvfiles %} - {% for tile in range(1,ntiles+1) %} -- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] - {% endfor %} - {% endfor %} -{% endfor %} -###################################### -## copy backgrounds again for fv_tracer to create analysis files later -{% for tile in range(1,ntiles+1) %} -- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} - -###################################### -## copy berror files from COMIN_CHEM_BMAT_PREV -## stddev files -{% for tile in range(1, ntiles+1) %} -- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] -{% endfor %} -### copy coupler file -- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] -### copy diffusion files -- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] -- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 5d2ddc51021..7a0d63d6ca2 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_ANTHROL2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Ship emissions -SU_SHIPSO2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_SHIPSO4 NA Y Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -63,11 +63,11 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA Y Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -OC_ANTEOC2 NA Y Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -OC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null @@ -88,11 +88,11 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA Y Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -BC_ANTEBC2 NA Y Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -BC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null diff --git a/scripts/exgdas_aero_analysis_generate_bmatrix.py b/scripts/exgdas_aero_analysis_generate_bmatrix.py deleted file mode 100755 index 0d8389c40d3..00000000000 --- a/scripts/exgdas_aero_analysis_generate_bmatrix.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# exgdas_aero_analysis_generate_bmatrix.py -# This script creates an AerosolBMatrix object -# and runs the methods needed -# to stage files, compute the variance, and write to com -# files needed for the variational solver -import os - -from wxflow import Logger, cast_strdict_as_dtypedict -from pygfs.task.aero_bmatrix import AerosolBMatrix - -# Initialize root logger -logger = Logger(level='DEBUG', colored_log=True) - - -if __name__ == '__main__': - - # Take configuration from environment and cast it as python dictionary - config = cast_strdict_as_dtypedict(os.environ) - - # Instantiate the aerosol variance and diffusion correlation tasks - AeroB = AerosolBMatrix(config) - AeroB.initialize() - AeroB.interpBackground() - AeroB.computeVariance() - AeroB.computeDiffusion() - AeroB.finalize() diff --git a/scripts/exglobal_aero_analysis_variational.py b/scripts/exglobal_aero_analysis_run.py similarity index 84% rename from scripts/exglobal_aero_analysis_variational.py rename to scripts/exglobal_aero_analysis_run.py index dd5bb4f65a6..85f4b963a4f 100755 --- a/scripts/exglobal_aero_analysis_variational.py +++ b/scripts/exglobal_aero_analysis_run.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# exglobal_aero_analysis_variational.py +# exglobal_aero_analysis_run.py # This script creates an AerosolAnalysis object -# and runs the variational method +# and runs the execute method # which executes the global aerosol variational analysis import os @@ -19,4 +19,4 @@ # Instantiate the aerosol analysis task AeroAnl = AerosolAnalysis(config) - AeroAnl.variational() + AeroAnl.execute() diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index dae98bbd652..be912292fee 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -367,11 +367,9 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then declare -a JEDI_EXE=("gdas.x" \ "gdas_soca_gridgen.x" \ "gdas_soca_error_covariance_toolbox.x" \ - "gdas_fv3jedi_error_covariance_toolbox.x" \ "gdas_soca_setcorscales.x" \ "gdas_soca_diagb.x" \ "fv3jedi_plot_field.x" \ - "gdasapp_chem_diagb.x" \ "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index bd15ec8f929..7de31d6235b 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -60,18 +60,6 @@ FV3_postdet() { break fi done - # Replace fv_tracer with aeroanl_fv_tracer restart files from current cycle (if found) - local nn - for (( nn = 1; nn <= ntiles; nn++ )); do - if [[ -f "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" ]]; then - rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" - ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ - "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" - else - echo "WARNING: 'aeroanl_fv_tracer.res.tile1.nc' not found in '${COMOUT_ATMOS_RESTART}', using 'fv_tracer.res.tile1.nc'" - break - fi - done fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index e1ff7e68624..c0b72bbc352 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,10 +2,8 @@ import os from .task.analysis import Analysis -from .task.bmatrix import BMatrix from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis -from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis from .task.atmens_analysis import AtmEnsAnalysis from .task.marine_bmat import MarineBMat diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index 2a22bd56328..69a992d7d47 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -82,18 +82,29 @@ def initialize(self: Analysis) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage files from COM and create working directories - logger.info(f"Staging files prescribed from {self.task_config.AERO_STAGE_VARIATIONAL_TMPL}") - aero_var_stage_list = parse_j2yaml(self.task_config.AERO_STAGE_VARIATIONAL_TMPL, self.task_config) - FileHandler(aero_var_stage_list).sync() + # stage berror files + # copy BUMP files, otherwise it will assume ID matrix + if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: + FileHandler(self.get_berror_dict(self.task_config)).sync() + + # stage backgrounds + FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") + # need output dir for diags and anl + logger.debug("Create empty output [anl, diags] directories to receive output from executable") + newdirs = [ + os.path.join(self.task_config['DATA'], 'anl'), + os.path.join(self.task_config['DATA'], 'diags'), + ] + FileHandler({'mkdir': newdirs}).sync() + @logit(logger) - def variational(self: Analysis) -> None: + def execute(self: Analysis) -> None: chdir(self.task_config.DATA) @@ -129,15 +140,13 @@ def finalize(self: Analysis) -> None: """ # ---- tar up diags # path of output tar statfile - logger.info('Preparing observation space diagnostics for archiving') - aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") # get list of diag files to put in tarball diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) # gzip the files first for diagfile in diags: - logger.info(f'Adding {diagfile} to tar file') with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) @@ -146,16 +155,44 @@ def finalize(self: Analysis) -> None: for diagfile in diags: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - logger.info(f'Saved diags to {aerostat}') + + # copy full YAML from executable to ROTDIR + src = os.path.join(self.task_config['DATA'], f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") + yaml_copy = { + 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], + 'copy': [[src, dest]] + } + FileHandler(yaml_copy).sync() + + # ---- copy RESTART fv_tracer files for future reference + if self.task_config.DOIAU: + bkgtime = self.task_config.AERO_WINDOW_BEGIN + else: + bkgtime = self.task_config.current_cycle + template = '{}.fv_tracer.res.tile{}.nc'.format(to_fv3time(bkgtime), '{tilenum}') + bkglist = [] + for itile in range(1, self.task_config.ntiles + 1): + tracer = template.format(tilenum=itile) + src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer) + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}') + bkglist.append([src, dest]) + FileHandler({'copy': bkglist}).sync() # ---- add increments to RESTART files logger.info('Adding increments to RESTART files') self._add_fms_cube_sphere_increments() - # copy files back to COM - logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}") - aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) - FileHandler(aero_var_final_list).sync() + # ---- move increments to ROTDIR + logger.info('Moving increments to ROTDIR') + template = f'aeroinc.{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' + inclist = [] + for itile in range(1, self.task_config.ntiles + 1): + tracer = template.format(tilenum=itile) + src = os.path.join(self.task_config.DATA, 'anl', tracer) + dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) + inclist.append([src, dest]) + FileHandler({'copy': inclist}).sync() def clean(self): super().clean() @@ -172,7 +209,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) + bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] @@ -195,7 +232,38 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkg_dict: Dict a dictionary containing the list of model background files to copy for FileHandler """ - bkg_dict = {} + # NOTE for now this is FV3 RESTART files and just assumed to be fh006 + + # get FV3 RESTART files, this will be a lot simpler when using history files + rst_dir = task_config.COM_ATMOS_RESTART_PREV + run_dir = os.path.join(task_config['DATA'], 'bkg') + + # Start accumulating list of background files to copy + bkglist = [] + + # if using IAU, we can use FGAT + bkgtimes = [] + begintime = task_config.previous_cycle + for fcsthr in task_config.aero_bkg_fhr: + bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) + + # now loop over background times + for bkgtime in bkgtimes: + # aerosol DA needs coupler + basename = f'{to_fv3time(bkgtime)}.coupler.res' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + # aerosol DA only needs core/tracer + for ftype in ['core', 'tracer']: + template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + bkg_dict = { + 'mkdir': [run_dir], + 'copy': bkglist, + } return bkg_dict @logit(logger) @@ -217,5 +285,34 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_dict: Dict a dictionary containing the list of background error files to copy for FileHandler """ - berror_dict = {} + # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. + b_dir = config.BERROR_DATA_DIR + b_datestr = to_fv3time(config.BERROR_DATE) + berror_list = [] + + for ftype in ['stddev']: + coupler = f'{b_datestr}.{ftype}.coupler.res' + berror_list.append([ + os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) + ]) + template = f'{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' + for itile in range(1, config.ntiles + 1): + tracer = template.format(tilenum=itile) + berror_list.append([ + os.path.join(b_dir, tracer), os.path.join(config.DATA, 'berror', tracer) + ]) + radius = 'cor_aero_universe_radius' + berror_list.append([ + os.path.join(b_dir, radius), os.path.join(config.DATA, 'berror', radius) + ]) + nproc = config.ntiles * config.layout_x * config.layout_y + for nn in range(1, nproc + 1): + berror_list.append([ + os.path.join(b_dir, f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc'), + os.path.join(config.DATA, 'berror', f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc') + ]) + berror_dict = { + 'mkdir': [os.path.join(config.DATA, 'berror')], + 'copy': berror_list, + } return berror_dict diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py deleted file mode 100644 index 9431e458022..00000000000 --- a/ush/python/pygfs/task/aero_bmatrix.py +++ /dev/null @@ -1,294 +0,0 @@ -#!/usr/bin/env python3 - -import os -from logging import getLogger -from typing import List, Dict, Any, Union - -from wxflow import (AttrDict, FileHandler, rm_p, - add_to_datetime, to_fv3time, to_timedelta, - to_fv3time, chdir, Executable, WorkflowException, - parse_j2yaml, save_as_yaml, logit) -from pygfs.task.bmatrix import BMatrix - -logger = getLogger(__name__.split('.')[-1]) - - -class AerosolBMatrix(BMatrix): - """ - Class for global aerosol BMatrix tasks - """ - @logit(logger, name="AerosolBMatrix") - def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) - - _res = int(self.task_config['CASE'][1:]) - _res_anl = int(self.task_config['CASE_ANL'][1:]) - - _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") - _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") - _convertstate_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_convertstate.yaml") - - # Create a local dictionary that is repeatedly used across this class - local_dict = AttrDict( - { - 'npx_ges': _res + 1, - 'npy_ges': _res + 1, - 'npz_ges': self.task_config.LEVS - 1, - 'npz': self.task_config.LEVS - 1, - 'npx_anl': _res_anl + 1, - 'npy_anl': _res_anl + 1, - 'npz_anl': self.task_config['LEVS'] - 1, - 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), - 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", - 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", - 'bmat_yaml': _bmat_yaml, - 'diffusion_yaml': _diffusion_yaml, - 'convertstate_yaml': _convertstate_yaml, - } - ) - - # task_config is everything that this task should need - self.task_config = AttrDict(**self.task_config, **local_dict) - - @logit(logger) - def initialize(self: BMatrix) -> None: - super().initialize() - # stage fix files - logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") - jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) - FileHandler(jedi_fix_list).sync() - - # stage backgrounds - logger.info(f"Staging backgrounds prescribed from {self.task_config.AERO_BMATRIX_STAGE_TMPL}") - aero_bmat_stage_list = parse_j2yaml(self.task_config.AERO_BMATRIX_STAGE_TMPL, self.task_config) - FileHandler(aero_bmat_stage_list).sync() - - # generate convert state YAML file - logger.info(f"Generate convert state YAML file: {self.task_config.convertstate_yaml}") - self.task_config.convertstate_config = parse_j2yaml(self.task_config.INTERPYAML, - self.task_config, - searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(self.task_config.convertstate_config, self.task_config.convertstate_yaml) - logger.info(f"Wrote convert state YAML to: {self.task_config.convertstate_yaml}") - - # generate diagb YAML file - logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") - self.task_config.bmat_config = parse_j2yaml(self.task_config.BMATYAML, - self.task_config, - searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) - logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") - - # generate diffusion parameters YAML file - logger.info(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") - self.task_config.diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, - self.task_config, - searchpath=self.gdasapp_j2tmpl_dir) - save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) - logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") - - # link executable to run directory - self.link_bmatexe() - self.link_diffusion_exe() - self.link_jediexe() - - @logit(logger) - def interpBackground(self) -> None: - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_AEROGENB) - exec_name = os.path.join(self.task_config.DATA, 'gdas.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg('fv3jedi') - exec_cmd.add_default_arg('convertstate') - exec_cmd.add_default_arg(self.task_config.convertstate_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass - - @logit(logger) - def computeVariance(self) -> None: - - chdir(self.task_config.DATA) - - exec_cmd = Executable(self.task_config.APRUN_AEROGENB) - exec_name = os.path.join(self.task_config.DATA, 'gdasapp_chem_diagb.x') - exec_cmd.add_default_arg(exec_name) - exec_cmd.add_default_arg(self.task_config.bmat_yaml) - - try: - logger.debug(f"Executing {exec_cmd}") - exec_cmd() - except OSError: - raise OSError(f"Failed to execute {exec_cmd}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd}") - - pass - - @logit(logger) - def computeDiffusion(self) -> None: - - chdir(self.task_config.DATA) - - exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROGENB) - exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') - exec_cmd_diffusion.add_default_arg(exec_name_diffusion) - exec_cmd_diffusion.add_default_arg(self.task_config.diffusion_yaml) - - try: - logger.debug(f"Executing {exec_cmd_diffusion}") - exec_cmd_diffusion() - except OSError: - raise OSError(f"Failed to execute {exec_cmd_diffusion}") - except Exception: - raise WorkflowException(f"An error occured during execution of {exec_cmd_diffusion}") - - pass - - @logit(logger) - def finalize(self) -> None: - super().finalize() - # save files to COMOUT - logger.info(f"Saving files to COMOUT based on {self.task_config.AERO_BMATRIX_FINALIZE_TMPL}") - aero_bmat_finalize_list = parse_j2yaml(self.task_config.AERO_BMATRIX_FINALIZE_TMPL, self.task_config) - FileHandler(aero_bmat_finalize_list).sync() - - @logit(logger) - def link_jediexe(self) -> None: - """ - - This method links a JEDI executable to the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - exe_src = self.task_config.JEDIEXE - - # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - logger.info(f"Link executable {exe_src} to DATA/") - logger.warn("Linking is not permitted per EE2.") - exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - - return exe_dest - - @logit(logger) - def link_bmatexe(self) -> None: - """ - - This method links a JEDI executable to the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - exe_src = self.task_config.BMATEXE - - # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - logger.info(f"Link executable {exe_src} to DATA/") - logger.warn("Linking is not permitted per EE2.") - exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) - if os.path.exists(exe_dest): - rm_p(exe_dest) - os.symlink(exe_src, exe_dest) - - return - - @logit(logger) - def link_diffusion_exe(self) -> None: - """ - - This method links a JEDI (fv3jedi_error_covariance_toolbox.x) - executable to the run directory - - Parameters - ---------- - Task: GDAS task - - Returns - ---------- - None - """ - - exe_src_diffusion = self.task_config.DIFFUSIONEXE - - # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. - logger.info(f"Link executable {exe_src_diffusion} to DATA/") - logger.warn("Linking is not permitted per EE2.") - exe_dest_diffusion = os.path.join(self.task_config.DATA, os.path.basename(exe_src_diffusion)) - if os.path.exists(exe_dest_diffusion): - rm_p(exe_dest_diffusion) - os.symlink(exe_src_diffusion, exe_dest_diffusion) - - return - - @logit(logger) - def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: - """Compile a dictionary of model background files to copy - - This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer) - that are needed for global aerosol DA and returns said dictionary for use by the FileHandler class. - - Parameters - ---------- - task_config: Dict - a dictionary containing all of the configuration needed for the task - - Returns - ---------- - bkg_dict: Dict - a dictionary containing the list of model background files to copy for FileHandler - """ - # NOTE for now this is FV3 RESTART files and just assumed to be fh006 - - # get FV3 RESTART files, this will be a lot simpler when using history files - rst_dir = task_config.COM_ATMOS_RESTART_PREV - run_dir = os.path.join(task_config['DATA'], 'bkg') - - # Start accumulating list of background files to copy - bkglist = [] - - # if using IAU, we can use FGAT - bkgtimes = [] - begintime = task_config.previous_cycle - for fcsthr in task_config.aero_bkg_fhr: - bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) - - # now loop over background times - for bkgtime in bkgtimes: - # aerosol DA needs coupler - basename = f'{to_fv3time(bkgtime)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - # aerosol DA only needs core/tracer - for ftype in ['core', 'tracer']: - template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - bkg_dict = { - 'mkdir': [run_dir], - 'copy': bkglist, - } - return bkg_dict diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index 0fc07467a05..e407cf17656 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -196,7 +196,7 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li @logit(logger) def link_jediexe(self) -> None: - """ + """Compile a dictionary of background error files to copy This method links a JEDI executable to the run directory diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py deleted file mode 100644 index d0edba2358a..00000000000 --- a/ush/python/pygfs/task/bmatrix.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 - -import os -from logging import getLogger -from typing import List, Dict, Any, Union - -from wxflow import (parse_j2yaml, FileHandler, logit, - Task, Executable, WorkflowException) - -logger = getLogger(__name__.split('.')[-1]) - - -class BMatrix(Task): - """Parent class for GDAS BMatrix tasks - - The BMatrix class is the parent class for all - Global Data Assimilation System (GDAS) BMatrix tasks - """ - def __init__(self, config: Dict[str, Any]) -> None: - super().__init__(config) - # Store location of GDASApp jinja2 templates - self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') - - def initialize(self) -> None: - super().initialize() - - def finalize(self) -> None: - super().finalize() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 823031ce476..e049a7d422f 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] if self.do_prep_obs_aero: configs += ['prepobsaero'] @@ -179,7 +179,7 @@ def get_task_names(self): gdas_tasks += wave_prep_tasks if self.do_aero and 'gdas' in self.aero_anl_runs: - gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] if self.do_prep_obs_aero: gdas_tasks += ['prepobsaero'] @@ -218,7 +218,7 @@ def get_task_names(self): gfs_tasks += wave_prep_tasks if self.do_aero and 'gfs' in self.aero_anl_runs: - gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] + gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] if self.do_prep_obs_aero: gfs_tasks += ['prepobsaero'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 72f1f78edfb..960a7548abd 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -506,41 +506,13 @@ def prepobsaero(self): return task - def aeroanlgenb(self): - - deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) - - resources = self.get_resource('aeroanlgenb') - task_name = f'{self.run}aeroanlgenb' - task_dict = {'task_name': task_name, - 'resources': resources, - 'dependency': dependencies, - 'envars': self.envars, - 'cycledef': 'gdas_half,gdas', - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', - 'job_name': f'{self.pslot}_{task_name}_@H', - 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', - 'maxtries': '&MAXTRIES;' - } - - task = rocoto.create_task(task_dict) - - return task - def aeroanlinit(self): deps = [] - dep_dict = {'type': 'task', 'name': 'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} - deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}prep'} - deps.append(rocoto.add_dependency(dep_dict)) - if self.app_config.do_prep_obs_aero: dep_dict = {'type': 'task', 'name': f'{self.run}prepobsaero'} - deps.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') @@ -560,28 +532,21 @@ def aeroanlinit(self): return task - def aeroanlvar(self): + def aeroanlrun(self): deps = [] - dep_dict = { - 'type': 'task', 'name': f'gdasaeroanlgenb', - 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", - } + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlinit'} deps.append(rocoto.add_dependency(dep_dict)) - dep_dict = { - 'type': 'task', 'name': f'{self.run}aeroanlinit', - } - deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + dependencies = rocoto.create_dependency(dep=deps) - resources = self.get_resource('aeroanlvar') - task_name = f'{self.run}aeroanlvar' + resources = self.get_resource('aeroanlrun') + task_name = f'{self.run}aeroanlrun' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlvar.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlrun.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -594,7 +559,7 @@ def aeroanlvar(self): def aeroanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlvar'} + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlrun'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 3b2fbf5420d..353d2aa943c 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -20,7 +20,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', + 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', 'prepsnowobs', 'snowanl', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp', From 6086f359cc247041e34a5afdc096bce37f7386cd Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 7 Aug 2024 19:45:39 +0000 Subject: [PATCH 63/69] fix shellnorm --- ush/forecast_postdet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 2aff52c06f2..5acf26558e0 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -71,13 +71,13 @@ FV3_postdet() { break fi done - if [[ use_anl_aero == "YES" ]]; then + if [[ ${use_anl_aero} == "YES" ]]; then for (( nn = 1; nn <= ntiles; nn++ )); do rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" done - fi # if [[ use_anl_aero == "YES" ]]; then + fi # if [[ ${use_anl_aero} == "YES" ]]; then fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then From 2af898099d9df433cd5e48d3b4a0384c7f6dd048 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Fri, 9 Aug 2024 17:26:08 +0000 Subject: [PATCH 64/69] update gdas hash and copy gaussian increment --- parm/gdas/aero_finalize_variational.yaml.j2 | 1 + sorc/gdas.cd | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 index b9247bcd620..7dadd362910 100644 --- a/parm/gdas/aero_finalize_variational.yaml.j2 +++ b/parm/gdas/aero_finalize_variational.yaml.j2 @@ -17,6 +17,7 @@ copy: {% for tile in range(1,ntiles+1) %} - ["{{ DATA }}/anl/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_ANALYSIS }}/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] {% endfor %} +- ["{{ DATA }}/anl/aeroinc_gauss.{{ current_cycle | to_isotime }}.gaussian.modelLevels.nc", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aeroinc.nc"] ## copy analysis {% for tile in range(1,ntiles+1) %} - ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] diff --git a/sorc/gdas.cd b/sorc/gdas.cd index f3fa26d4d66..c68ef9d6443 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit f3fa26d4d6693fcf451184d5ecabb86c1b4190ca +Subproject commit c68ef9d64433cc4bf565073a9eae1862cf8571d3 From 7339c0bb7cc6c451b9f5bd31692eb3f60eabc668 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Tue, 13 Aug 2024 17:28:39 +0000 Subject: [PATCH 65/69] remove mkdir lines --- jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE | 2 -- jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE | 2 -- 2 files changed, 4 deletions(-) diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index b894b82531f..9c68d1fed6f 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -18,8 +18,6 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL -mkdir -p "${COMOUT_CHEM_ANALYSIS}" - ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index 5be8767308c..921b1458b25 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -27,8 +27,6 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL -mkdir -p "${COMOUT_CHEM_ANALYSIS}" - ############################################################### # Run relevant script From b20ba7b20105d5d92905ab15e75f45cdc9e42d55 Mon Sep 17 00:00:00 2001 From: ypwang19 Date: Tue, 20 Aug 2024 21:15:38 +0000 Subject: [PATCH 66/69] move archive step to the end of the finalize function --- ush/python/pygfs/task/aero_analysis.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index f98985c45ba..0389e109a1b 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -141,13 +141,6 @@ def finalize(self: Analysis) -> None: with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) - # open tar file for writing - with tarfile.open(aerostat, "w") as archive: - for diagfile in diags: - diaggzip = f"{diagfile}.gz" - archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - logger.info(f'Saved diags to {aerostat}') - # ---- add increments to RESTART files logger.info('Adding increments to RESTART files') self._add_fms_cube_sphere_increments() @@ -157,6 +150,13 @@ def finalize(self: Analysis) -> None: aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) FileHandler(aero_var_final_list).sync() + # open tar file for writing + with tarfile.open(aerostat, "w") as archive: + for diagfile in diags: + diaggzip = f"{diagfile}.gz" + archive.add(diaggzip, arcname=os.path.basename(diaggzip)) + logger.info(f'Saved diags to {aerostat}') + def clean(self): super().clean() From dd1ab121ae8886c0e7be1d926f693c15be3cb720 Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 21 Aug 2024 13:50:21 +0000 Subject: [PATCH 67/69] update gdas hash --- sorc/gdas.cd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/gdas.cd b/sorc/gdas.cd index c68ef9d6443..0431b26650c 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit c68ef9d64433cc4bf565073a9eae1862cf8571d3 +Subproject commit 0431b26650c5e5d4eb741304a05c841d3fda0ddc From 7c0ca0115aac1b1706553553b38566be3581e5aa Mon Sep 17 00:00:00 2001 From: CoryMartin-NOAA Date: Wed, 21 Aug 2024 16:36:53 +0000 Subject: [PATCH 68/69] fix dependency --- workflow/rocoto/gfs_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 7ca8145084d..8753a57ab2b 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -458,7 +458,7 @@ def prepobsaero(self): def aeroanlgenb(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}fcst'} + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep=deps) From 2d5b369f76935418d221a5ed4bd678be5b8425ca Mon Sep 17 00:00:00 2001 From: Cory Martin Date: Tue, 27 Aug 2024 15:34:23 -0400 Subject: [PATCH 69/69] Update config.resources --- parm/config/gfs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index e73e695e26a..045bd09bb6c 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -16,7 +16,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl esnowrecen" - echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu"