Stat Analysis Implementation for global-workflow

.github/CODEOWNERS

@@ -42,6 +42,7 @@ jobs/JGFS_ATMOS_PGRB2_SPEC_NPOESS @WenMeng-NOAA
 jobs/JGFS_ATMOS_POSTSND @BoCui-NOAA
 jobs/JGFS_ATMOS_VERIFICATION
 jobs/JGLOBAL_AERO_ANALYSIS_* @CoryMartin-NOAA
+jobs/JGLOBAL_ANALYSIS_STATS @CoryMartin-NOAA
 jobs/JGLOBAL_*ARCH* @DavidHuber-NOAA
 jobs/JGLOBAL_ATMENS_ANALYSIS_* @RussTreadon-NOAA @CoryMartin-NOAA @DavidNew-NOAA
 jobs/JGLOBAL_ATMOS_ANALYSIS @RussTreadon-NOAA @CatherineThomas-NOAA
@@ -96,6 +97,7 @@ scripts/exgfs_pmgr.sh
 scripts/exgfs_prdgen_manager.sh
 scripts/exgfs_wave_* @JessicaMeixner-NOAA @sbanihash
 scripts/exglobal_aero_analysis_* @CoryMartin-NOAA
+scripts/exglobal_analysis_stats.py @CoryMartin-NOAA
 scripts/exglobal_archive_*.py @DavidHuber-NOAA
 scripts/exglobal_globus_*.py @DavidHuber-NOAA
 scripts/exglobal_atm_analysis_* @RussTreadon-NOAA @DavidNew-NOAA
@@ -190,6 +192,7 @@ ush/python/pygfs/task/aero_bmatrix.py @DavidNew-NOAA @CoryMartin-NOAA
 ush/python/pygfs/task/aero_emissions.py @bbakernoaa
 ush/python/pygfs/task/aero_prepobs.py @CoryMartin-NOAA
 ush/python/pygfs/task/analysis.py @DavidNew-NOAA @RussTreadon-NOAA
+ush/python/pygfs/task/analysis_stats.py @CoryMartin-NOAA
 ush/python/pygfs/task/archive.py @DavidHuber-NOAA
 ush/python/pygfs/task/atm_analysis.py @DavidNew-NOAA @RussTreadon-NOAA
 ush/python/pygfs/task/atmens_analysis.py @DavidNew-NOAA @RussTreadon-NOAA

.gitignore

@@ -52,6 +52,7 @@ parm/gdas/io
 parm/gdas/ioda
 parm/gdas/snow
 parm/gdas/soca
+parm/gdas/stat
 parm/gdas/jcb-gdas
 parm/gdas/jcb-algorithms
 parm/monitor

env/GAEAC6.env

@@ -146,6 +146,11 @@ case ${step} in
     export NTHREADS_OCNANAL=${NTHREADSmax}
     export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
  ;;
+ "anlstat")
+
+    export NTHREADS_ANLSTAT=${NTHREADSmax}
+    export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}"
+ ;;
  "marineanlletkf")
 
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}

env/HERA.env

@@ -109,6 +109,11 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
 
+elif [[ "${step}" = "anlstat" ]]; then
+
+    export NTHREADS_ANLSTAT=${NTHREADSmax}
+    export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

env/HERCULES.env

@@ -157,6 +157,11 @@ case ${step} in
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF="${APRUN_default}"
  ;;
+ "anlstat")
+
+    export NTHREADS_ANLSTAT=${NTHREADSmax}
+    export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" 
+ ;;
  "ecen_fv3jedi")
 
     export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}

env/ORION.env

@@ -158,6 +158,11 @@ elif [[ "${step}" = "marineanlletkf" ]]; then
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF="${APRUN_default}"
 
+elif [[ "${step}" = "anlstat" ]]; then
+
+    export NTHREADS_ANLSTAT=${NTHREADSmax}
+    export APRUN_ANLSTAT="${APRUN_default} --cpus-per-task=${NTHREADS_ANLSTAT}" 
+
 elif [[ "${step}" = "ecen_fv3jedi" ]]; then
 
     export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}

env/WCOSS2.env

@@ -143,6 +143,11 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN_default}"
 
+elif [[ "${step}" = "anlstat" ]]; then
+
+    export NTHREADS_ANLSTAT=${NTHREADSmax}
+    export APRUN_ANLSTAT="${APRUN_default}"
+
 elif [[ "${step}" = "ecen_fv3jedi" ]]; then
 
     export NTHREADS_ECEN_FV3JEDI=${NTHREADSmax}

jobs/JGLOBAL_ANALYSIS_STATS

@@ -0,0 +1,44 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/jjob_header.sh" -e "anlstat" -c "base anlstat"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \
+                                           COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+                                           COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
+                                           COMIN_AERO_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+                                           COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
+                                           COMOUT_CONF:COM_CONF_TMPL \
+                                           COMOUT_ATMOS_ANLMON:COM_ATMOS_ANLMON_TMPL \
+                                           COMOUT_OCEAN_ANLMON:COM_OCEAN_ANLMON_TMPL \
+                                           COMOUT_AERO_ANLMON:COM_CHEM_ANLMON_TMPL \
+                                           COMOUT_SNOW_ANLMON:COM_SNOW_ANLMON_TMPL
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${ANLSTATSPY:-${SCRgfs}/exglobal_analysis_stats.py}
+${EXSCRIPT}
+export err=$?; err_chk
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+exit 0

jobs/rocoto/anlstat.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+set -x
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+err=$?
+if [[ ${err} -ne 0 ]]; then
+    echo "FATAL ERROR Failed to load UFSDA modules!"
+    exit "${err}"
+fi
+
+export job="anlstat"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGLOBAL_ANALYSIS_STATS"
+exit $?

parm/config/gfs/config.anlstat

@@ -0,0 +1,17 @@
+#!/bin/bash -x
+
+########## config.anlstat ##########
+# Analysis Stat
+
+echo "BEGIN: config.anlstat"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" anlstat
+
+export JEDI_CONFIG_YAML="${PARMgfs}/gdas/anlstat_jedi_config.yaml.j2"
+export STAT_BASE_CONFIG_YAML="${PARMgfs}/config/gfs/yaml/stat_base_config.yaml.j2"
+export JCB_BASE_YAML="${PARMgfs}/gdas/stat/aero/jcb-base.yaml.j2"
+export JCB_ALGO_YAML="${PARMgfs}/gdas/jcb-algorithms/anlstat.yaml.j2"
+export JEDIEXE="${HOMEgfs}/sorc/gdas.cd/build/bin/ioda-stats.x"
+
+echo "END: config.anlstat"

parm/config/gfs/config.base.j2

@@ -82,6 +82,7 @@ export DO_GENESIS_FSU="{{ DO_GENESIS_FSU }}" # Cyclone genesis verification (FSU
 export DO_VERFOZN="YES"                  # Ozone data assimilation monitoring
 export DO_VERFRAD="YES"                  # Radiance data assimilation monitoring
 export DO_VMINMON="YES"                  # GSI minimization monitoring
+export DO_ANLSTAT="NO"                   # JEDI-based analysis statistics
 export DO_MOS="NO"                       # GFS Model Output Statistics - Only supported on WCOSS2
 
 # NO for retrospective parallel; YES for real-time parallel
@@ -499,6 +500,11 @@ if [[ "${DO_JEDIATMVAR}" = "YES" ]]; then
    export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
    export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
    export DO_VMINMON="NO"     # GSI minimization monitoring
+   export DO_ANLSTAT="YES"    # JEDI-based analysis statistics
+else
+   if [[ ${DO_AERO} = "YES" || ${DO_JEDIOCNVAR} = "YES" || ${DO_JEDISNOWDA} = "YES " ]]; then
+     export DO_ANLSTAT="YES"  # JEDI-based analysis statistics
+   fi
 fi
 
 # If starting ICs that are not at cycle hour

parm/config/gfs/config.com

@@ -55,6 +55,7 @@ declare -rx COM_ATMOS_INPUT_TMPL=${COM_BASE}'/model/atmos/input'
 declare -rx COM_ATMOS_RESTART_TMPL=${COM_BASE}'/model/atmos/restart'
 declare -rx COM_ATMOS_ANALYSIS_TMPL=${COM_BASE}'/analysis/atmos'
 declare -rx COM_SNOW_ANALYSIS_TMPL=${COM_BASE}'/analysis/snow'
+declare -rx COM_SNOW_ANLMON_TMPL=${COM_BASE}'/products/snow/anlmon'
 declare -rx COM_ATMOS_HISTORY_TMPL=${COM_BASE}'/model/atmos/history'
 declare -rx COM_ATMOS_MASTER_TMPL=${COM_BASE}'/model/atmos/master'
 declare -rx COM_ATMOS_GRIB_TMPL=${COM_BASE}'/products/atmos/grib2'
@@ -68,6 +69,7 @@ declare -rx COM_ATMOS_IMAGERY_TMPL=${COM_BASE}'/products/atmos/imagery'
 declare -rx COM_ATMOS_OZNMON_TMPL=${COM_BASE}'/products/atmos/oznmon'
 declare -rx COM_ATMOS_RADMON_TMPL=${COM_BASE}'/products/atmos/radmon'
 declare -rx COM_ATMOS_MINMON_TMPL=${COM_BASE}'/products/atmos/minmon'
+declare -rx COM_ATMOS_ANLMON_TMPL=${COM_BASE}'/products/atmos/anlmon'
 declare -rx COM_ATMOS_WMO_TMPL=${COM_BASE}'/products/atmos/wmo'
 
 declare -rx COM_WAVE_RESTART_TMPL=${COM_BASE}'/model/wave/restart'
@@ -83,6 +85,7 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history'
 declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart'
 declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input'
 declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean'
+declare -rx COM_OCEAN_ANLMON_TMPL=${COM_BASE}'/products/ocean/anlmon'
 declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf'
 declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean'
 declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf'
@@ -91,6 +94,7 @@ declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}'
 
 declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice'
 declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf'
+declare -rx COM_ICE_ANLMON_TMPL=${COM_BASE}'/products/ice/anlmon'
 declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice'
 declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input'
 declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history'
@@ -102,5 +106,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}'
 declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history'
 declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem'
 declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix'
+declare -rx COM_CHEM_ANLMON_TMPL=${COM_BASE}'/products/chem/anlmon'
 
 declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart'

parm/config/gfs/config.resources

@@ -17,7 +17,7 @@ if (( $# != 1 )); then
     echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal ecen_fv3jedi analcalc_fv3jedi"
     echo "snowanl esnowanl"
     echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb"
-    echo "anal sfcanl analcalc analdiag fcst echgres"
+    echo "anal sfcanl analcalc analdiag anlstat fcst echgres"
     echo "upp atmos_products"
     echo "tracker genesis genesis_fsu"
     echo "verfozn verfrad vminmon fit2obs metp arc_vrfy arc_tars cleanup"
@@ -801,6 +801,14 @@ case ${step} in
     memory="48GB"
     ;;
 
+  "anlstat")
+    walltime="00:30:00"
+    ntasks=20
+    threads_per_task=1
+    tasks_per_node=20
+    memory="90GB"
+    ;;
+
   "sfcanl")
     walltime="00:20:00"
     ntasks=${ntiles:-6}

parm/config/gfs/yaml/stat_base_config.yaml.j2

@@ -0,0 +1,26 @@
+aero:
+  stat_file_path: '{{ COMIN_AERO_ANALYSIS }}'
+  stat_file_name: 'aerostat.tgz'
+  obs spaces:
+  - name: viirs_npp
+    input file: "diag_viirs_npp_aod_{{ current_cycle | to_YMDH }}.nc"
+    output file: "viirs_npp_{{ current_cycle | to_YMDH }}_output_aod.nc"
+  - name: viirs_n20
+    input file: "diag_viirs_n20_aod_{{ current_cycle | to_YMDH }}.nc"
+    output file: "viirs_n20_{{ current_cycle | to_YMDH }}_output_aod.nc"
+
+atmos:
+  stat_file_path: '{{ COMIN_ATMOS_ANALYSIS }}'
+  stat_file_name: 'atmstat'
+  obs spaces:
+  - name: conventional_ps
+    input file: "diag_conventional_ps_{{ current_cycle | to_YMDH }}.nc"
+    output file: "conventional_ps_{{ current_cycle | to_YMDH }}_output_atmos.nc"
+
+snow:
+  stat_file_path: '{{ COMIN_SNOW_ANALYSIS }}'
+  stat_file_name: 'snowstat.tgz'
+  obs spaces:
+  - name: ims_snow
+    input file: "diag_ims_snow_{{ current_cycle | to_YMDH }}.nc"
+    output file: "ims_snow_{{ current_cycle | to_YMDH }}_output_snow.nc"

parm/gdas/anlstat_jedi_config.yaml.j2

@@ -0,0 +1,24 @@
+aero:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ JEDIEXE }}'
+  mpi_cmd: '{{ APRUN_ANLSTAT }}'
+  # jedi_args: None
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/aero/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML }}'
+  jcb_algo: 'anlstat'
+atmos:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ JEDIEXE }}'
+  mpi_cmd: '{{ APRUN_ANLSTAT }}'
+  # jedi_args: None
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/atmos/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML }}'
+  jcb_algo: 'anlstat'
+snow:
+  rundir: '{{ DATA }}'
+  exe_src: '{{ JEDIEXE }}'
+  mpi_cmd: '{{ APRUN_ANLSTAT }}'
+  # jedi_args: None
+  jcb_base_yaml: '{{ PARMgfs }}/gdas/stat/snow/jcb-base.yaml.j2'
+  jcb_algo_yaml: '{{ JCB_ALGO_YAML }}'
+  jcb_algo: 'anlstat'

scripts/exglobal_analysis_stats.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+# exglobal_analysis_stats.py
+# This script creates a AnalysisStats class
+# and runs the initialize, execute, and finalize
+# methods which create and stage the runtime directory
+# and create the YAML configuration
+# to produce summary statistics from the desired analyses
+import os
+
+from wxflow import Logger, cast_strdict_as_dtypedict
+from pygfs.task.analysis_stats import AnalysisStats
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config = cast_strdict_as_dtypedict(os.environ)
+
+    # Instantiate the atm analysis task
+    AnlStats = AnalysisStats(config)
+
+    # Create list based on DA components
+    AnlStats.task_config['STAT_ANALYSES'] = []
+    if AnlStats.task_config.DO_AERO_ANL:
+        AnlStats.task_config['STAT_ANALYSES'].append('aero')
+    if AnlStats.task_config.DO_JEDISNOWDA:
+        AnlStats.task_config['STAT_ANALYSES'].append('snow')
+    if AnlStats.task_config.DO_JEDIATMVAR:
+        AnlStats.task_config['STAT_ANALYSES'].append('atmos')
+
+    # Initialize JEDI variational analysis
+    AnlStats.initialize()
+    for anl in AnlStats.task_config.STAT_ANALYSES:
+        AnlStats.execute(anl)
+        AnlStats.finalize(anl)

sorc/link_workflow.sh

@@ -227,7 +227,7 @@ fi
 #------------------------------
 if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then
   cd "${HOMEgfs}/parm/gdas" || exit 1
-  declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "jcb-gdas" "jcb-algorithms")
+  declare -a gdasapp_comps=("aero" "atm" "io" "ioda" "snow" "soca" "jcb-gdas" "jcb-algorithms" "stat")
   for comp in "${gdasapp_comps[@]}"; do
     if [[ -d "${comp}" ]]; then
         rm -rf "${comp}"