ufs-community · MichaelLueken · Apr 4, 2023 · Feb 27, 2023 · Feb 27, 2023 · Feb 28, 2023
@@ -58,7 +58,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/AQM-utils
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = b066931
+hash = 8e084f9
 local_path = sorc/AQM-utils
 required = True
 

@@ -65,10 +65,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
 export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
 
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8`  # 1 year back
-# The earliest day of training data is August 1 2020
-if [ ${TMP_STDAY} -le 20220810 ]; then 
-  TMP_STDAY=20220810 
-fi
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 #
 #-----------------------------------------------------------------------

@@ -65,10 +65,6 @@ DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
 export PARMaqm_utils="${PARMaqm_utils:-${HOMEdir}/sorc/AQM-utils/parm}"
 
 TMP_STDAY=`${NDATE} -8760 ${PDY}${cyc} | cut -c1-8`  # 1 year back
-# The earliest day of training data is August 1 2020
-if [ ${TMP_STDAY} -le 20220810 ]; then 
-  TMP_STDAY=20220810 
-fi
 export BC_STDAY=${BC_STDAY:-${TMP_STDAY}}
 #
 #-----------------------------------------------------------------------

@@ -315,11 +315,20 @@ FV3_GFS_v15_thompson_mynn_lam3km:
   gfs_physics_nml:
     avg_max_length: 3600.0
     cdmbgwd: [0.88, 0.04]
+    debug: True
     do_deep: False
+    do_gsl_drag_ls_bl: False
+    do_gsl_drag_ss: True
+    do_gsl_drag_tofd: True
     do_mynnsfclay: True
+    do_tofd: False
     do_ugwp: False
+    do_ugwp_v0: False
+    do_ugwp_v0_nst_only: False
+    do_ugwp_v0_orog_only: False
     fhswr: 900.0
     fhlwr: 900.0
+    gwd_opt: 2
     iaer: 1011
     iccn: 2
     icliq_sw: 2
@@ -337,16 +346,20 @@ FV3_GFS_v15_thompson_mynn_lam3km:
     iopt_snf: 4
     iopt_stc: 1
     iopt_tbot: 2
-    iopt_trs: 2
+    iopt_trs: !!python/none
     iovr: 3
     ldiag_ugwp: False
     lgfdlmprad: False
     lsm: 1
+    lsoil: !!python/none
     lsoil_lsm: !!python/none
     ltaerosol: False    
+    print_diff_pgr: True
+    sfclay_compute_flux: !!python/none
     xkzminv: 0.3
     xkzm_m: 1.0
     xkzm_h: 1.0
+  surf_map_nml: !!python/none
 
 FV3_GFS_v16:
   cires_ugwp_nml:
@@ -356,6 +369,8 @@ FV3_GFS_v16:
     agrid_vel_rst: False
     d2_bg_k1: 0.2
     d2_bg_k2: 0.0
+    delt_max: 0.002
+    dz_min: 6
     fv_sg_adj: 450
     hord_dp: -5
     hord_mt: 5
@@ -395,7 +410,6 @@ FV3_GFS_v16:
     lndp_type: !!python/none
     lsoil: 4
     n_var_lndp: !!python/none
-    nstf_name: [2, 1, 0, 0, 0]
     prautco: [0.00015, 0.00015]
     psautco: [0.0008, 0.0005]
     satmedmf: True
@@ -506,7 +520,6 @@ FV3_GFS_v17_p8:
     min_lakeice: 0.15
     min_seaice: 0.15
     nsfullradar_diag: !!python/none
-    nstf_name: [2, 0, 0, 0, 0]
     prautco: [0.00015, 0.00015]
     psautco: [0.0008, 0.0005]
     qdiag3d: False

@@ -1,23 +1,26 @@
-total_member:            1
-PE_MEMBER01:             {{ PE_MEMBER01 }}
 start_year:              {{ start_year }}
 start_month:             {{ start_month }}
 start_day:               {{ start_day }}
 start_hour:              {{ start_hour }}
 start_minute:            0
 start_second:            0
 nhours_fcst:             {{ nhours_fcst }}
+fhrot:                   {{ fhrot }}
 RUN_CONTINUE:            .false.
 ENS_SPS:                 .false.
 dt_atmos:                {{ dt_atmos }}
 calendar:                'julian'
 memuse_verbose:          .false.
-atmos_nthreads:          {{ atmos_nthreads }}
 restart_interval:        {{ restart_interval }}
 output_1st_tstep_rst:    .false.
 write_dopost:            {{ write_dopost }}
 ideflate:                0
 nbits:                   0
+ichunk2d:                -1
+jchunk2d:                -1
+ichunk3d:                -1
+jchunk3d:                -1
+kchunk3d:                -1
 quilting:                {{ quilting }}
 {% if quilting  %}
 #

@@ -2,10 +2,13 @@
 ####  NEMS Run-Time Configuration File  #####
 #############################################
 
-{% if not print_esmf %}
 # ESMF #
+{%- if print_esmf %}
+logKindFlag:            ESMF_LOGKIND_MULTI
+{%- else %}
 logKindFlag:            ESMF_LOGKIND_MULTI_ON_ERROR
-{% endif %}
+{%- endif %}
+globalResourceControl:  true
 
 {% if cpl_aqm  %}
 # EARTH #
@@ -40,9 +43,23 @@ runSeq::
   @
 ::
 {% else %}
+# EARTH #
 EARTH_component_list: ATM
- ATM_model:            fv3
- runSeq::
-   ATM
- ::
+EARTH_attributes::
+  Verbosity = 0
+::
+
+# ATM #
+ATM_model:                      fv3
+ATM_petlist_bounds:             0 {{ pe_member01_m1 }}
+ATM_omp_num_threads:            {{ atm_omp_num_threads }}
+ATM_attributes::
+  Verbosity = 0
+  Diagnostic = 0
+::
+
+# Run Sequence #
+runSeq::
+  ATM
+::
 {% endif %}
@@ -12,7 +12,6 @@ default_aqm_task: &default_aqm
     subcyc: !cycstr "@M"
     LOGDIR: !cycstr "&LOGDIR;"
     SLASH_ENSMEM_SUBDIR: '&SLASH_ENSMEM_SUBDIR;'
-  memory: 2G
   native: '{{ platform.SCHED_NATIVE_CMD }}'
   nnodes: 1
   nodes: '{{ nnodes }}:ppn={{ ppn }}'
@@ -27,6 +26,7 @@ task_nexus_gfs_sfc:
   command: '&LOAD_MODULES_RUN_TASK_FP; "nexus_gfs_sfc" "&JOBSdir;/JREGIONAL_NEXUS_GFS_SFC"'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+  memory: 2G
   dependency:
     or:
       and:
@@ -72,6 +72,7 @@ task_fire_emission:
   <<: *default_aqm
   command: '&LOAD_MODULES_RUN_TASK_FP; "fire_emission" "&JOBSdir;/JREGIONAL_FIRE_EMISSION"'
   join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
+  memory: 2G
 
 task_point_source:
   <<: *default_aqm
@@ -146,7 +147,7 @@ task_aqm_lbcs:
   dependency:
     taskdep:
       attrs:
-        task: make_lbcs
+        task: make_lbcs_mem000
 
 task_pre_post_stat:
   <<: *default_aqm

@@ -27,6 +27,7 @@ rocoto:
     SCRIPTSdir: '{{ user.SCRIPTSdir }}'
     SLASH_ENSMEM_SUBDIR: '{% if global.DO_ENSEMBLE  %}{{ "/mem#mem#" }}{% else %}{{ "/" }}{% endif %}'
     USHdir: '{{ user.USHdir }}'
+    COLDSTART: '{{ workflow.COLDSTART }}'
     WARMSTART_CYCLE_DIR: '{{ workflow.WARMSTART_CYCLE_DIR }}'
     WORKFLOW_ID: '{{ workflow.WORKFLOW_ID }}'
   attrs:

@@ -29,7 +29,7 @@ metatask_run_ens_post:
   metatask_run_post:
     var:
       fhr: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{{ " %03d" % h }}{% endfor %}'
-      cycledef: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h <= workflow.FCST_LEN_HRS %}forecast {% else %}long_forecast {% endif %}{% endfor %}'
+      cycledef: '{% for h in range(0, workflow.LONG_FCST_LEN+1) %}{% if h <= workflow.FCST_LEN_CYCL|min %}forecast {% else %}long_forecast {% endif %}{% endfor %}'
     task_run_post_mem#mem#_f#fhr#:
       <<: *default_task
       dependency:

@@ -60,18 +60,15 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS}
 #
 #-----------------------------------------------------------------------
 #
-set -x
 eval ${PRE_TASK_CMDS}
 
-nprocs=$(( NNODES_AQM_LBCS*PPN_AQM_LBCS ))
-
-if [ -z "${RUN_CMD_UTILS:-}" ] ; then
+if [ -z "${RUN_CMD_AQMLBC:-}" ] ; then
   print_err_msg_exit "\
   Run command was not set in machine file. \
-  Please set RUN_CMD_UTILS for your platform"
+  Please set RUN_CMD_AQM_LBC for your platform"
 else
   print_info_msg "$VERBOSE" "
-  All executables will be submitted with command \'${RUN_CMD_UTILS}\'."
+  All executables will be submitted with command \'${RUN_CMD_AQMLBC}\'."
 fi
 
 #

@@ -53,6 +53,7 @@ This is the ex-script for the task that runs BIAS-CORRECTION-O3.
 export KMP_AFFINITY=${KMP_AFFINITY_BIAS_CORRECTION_O3}
 export OMP_NUM_THREADS=${OMP_NUM_THREADS_BIAS_CORRECTION_O3}
 export OMP_STACKSIZE=${OMP_STACKSIZE_BIAS_CORRECTION_O3}
+export OMP_PLACES=cores
 #
 #-----------------------------------------------------------------------
 #
@@ -84,8 +85,6 @@ rm_vrfy -rf "$DATA"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
 
-set -x
-
 yyyy=${PDY:0:4}
 yyyymm=${PDY:0:6}
 yyyy_m1=${PDYm1:0:4}
@@ -207,7 +206,7 @@ fi
 
 mkdir_vrfy -p ${DATA}/data/sites
 cp_vrfy ${PARMaqm_utils}/bias_correction/config.ozone.bias_corr_${id_domain}.${cyc}z ${DATA}
- 
+
 PREP_STEP
 eval ${RUN_CMD_SERIAL} ${EXECdir}/aqm_bias_correct config.ozone.bias_corr_${id_domain}.${cyc}z ${cyc}z ${BC_STDAY} ${PDY} ${REDIRECT_OUT_ERR} || print_err_msg_exit "Call to executable to run AQM_BIAS_CORRECT returned with nonzero exit code."
 POST_STEP

@@ -53,6 +53,7 @@ This is the ex-script for the task that runs BIAS-CORRECTION-PM25.
 export KMP_AFFINITY=${KMP_AFFINITY_BIAS_CORRECTION_PM25}
 export OMP_NUM_THREADS=${OMP_NUM_THREADS_BIAS_CORRECTION_PM25}
 export OMP_STACKSIZE=${OMP_STACKSIZE_BIAS_CORRECTION_PM25}
+export OMP_PLACES=cores
 #
 #-----------------------------------------------------------------------
 #
@@ -84,8 +85,6 @@ rm_vrfy -rf $DATA
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
 
-set -x
-
 yyyy=${PDY:0:4}
 yyyymm=${PDY:0:6}
 yyyy_m1=${PDYm1:0:4}

@@ -54,7 +54,6 @@ or lateral boundary conditions for the FV3.
 #
 #-----------------------------------------------------------------------
 #
-set -x
 if [ "${ICS_OR_LBCS}" = "ICS" ]; then
   if [ ${TIME_OFFSET_HRS} -eq 0 ] ; then
     file_set="anl"

@@ -244,7 +244,8 @@ mv_vrfy "${raw_orog_fp_orig}" "${raw_orog_fp}"
 #
 #-----------------------------------------------------------------------
 #
-if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ] || [ "${CCPP_PHYS_SUITE}" = "FV3_GFS_v17_p8" ]; then
+suites=( "FV3_HRRR" "FV3_GFS_v15_thompson_mynn_lam3km" "FV3_GFS_v17_p8" )
+if [[ ${suites[@]} =~ "${CCPP_PHYS_SUITE}" ]] ; then
   DATA="${DATA:-${OROG_DIR}/temp_orog_data}"
   mkdir_vrfy -p ${DATA}
   cd_vrfy ${DATA}

@@ -61,19 +61,16 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_NEXUS_EMISSION}
 #
 #-----------------------------------------------------------------------
 #
-set -x
 eval ${PRE_TASK_CMDS}
 
-omp_num_threads_run_aqm="${OMP_NUM_THREADS_NEXUS_EMISSION}"
-
-if [ -z "${RUN_CMD_AQM:-}" ] ; then
+if [ -z "${RUN_CMD_NEXUS:-}" ] ; then
   print_err_msg_exit "\
   Run command was not set in machine file. \
-  Please set RUN_CMD_AQM for your platform"
+  Please set RUN_CMD_NEXUS for your platform"
 else
-  RUN_CMD_AQM=$(eval echo ${RUN_CMD_AQM})
+  RUN_CMD_NEXUS=$(eval echo ${RUN_CMD_NEXUS})
   print_info_msg "$VERBOSE" "
-  All executables will be submitted with command \'${RUN_CMD_AQM}\'."
+  All executables will be submitted with command \'${RUN_CMD_NEXUS}\'."
 fi
 #
 #-----------------------------------------------------------------------
@@ -277,7 +274,7 @@ fi
 #-----------------------------------------------------------------------
 #
 PREP_STEP
-eval ${RUN_CMD_AQM} ${EXECdir}/nexus -c NEXUS_Config.rc -r grid_spec.nc -o NEXUS_Expt_split.nc ${REDIRECT_OUT_ERR} || \
+eval ${RUN_CMD_NEXUS} ${EXECdir}/nexus -c NEXUS_Config.rc -r grid_spec.nc -o NEXUS_Expt_split.nc ${REDIRECT_OUT_ERR} || \
 print_err_msg_exit "\
 Call to execute nexus standalone for the FV3LAM failed."
 POST_STEP

@@ -48,7 +48,6 @@ data files from disk or HPSS.
 #
 #-----------------------------------------------------------------------
 #
-set -x
 DATA="${DATA}/tmp_GFS_SFC"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA

@@ -47,16 +47,6 @@ This is the ex-script for the task that runs POST-STAT-O3.
 #
 #-----------------------------------------------------------------------
 #
-# Set OpenMP variables.
-#
-#-----------------------------------------------------------------------
-#
-export KMP_AFFINITY=${KMP_AFFINITY_POST_STAT_O3}
-export OMP_NUM_THREADS=${OMP_NUM_THREADS_POST_STAT_O3}
-export OMP_STACKSIZE=${OMP_STACKSIZE_POST_STAT_O3}
-#
-#-----------------------------------------------------------------------
-#
 # Set run command.
 #
 #-----------------------------------------------------------------------
@@ -83,8 +73,6 @@ fi
 DATA="${DATA}/tmp_POST_STAT_O3"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
-
-set -x
 #
 #-----------------------------------------------------------------------
 #

@@ -47,16 +47,6 @@ This is the ex-script for the task that runs POST-STAT-PM25.
 #
 #-----------------------------------------------------------------------
 #
-# Set OpenMP variables.
-#
-#-----------------------------------------------------------------------
-#
-export KMP_AFFINITY=${KMP_AFFINITY_POST_STAT_PM25}
-export OMP_NUM_THREADS=${OMP_NUM_THREADS_POST_STAT_PM25}
-export OMP_STACKSIZE=${OMP_STACKSIZE_POST_STAT_PM25}
-#
-#-----------------------------------------------------------------------
-#
 # Set run command.
 #
 #-----------------------------------------------------------------------
@@ -83,9 +73,6 @@ fi
 DATA="${DATA}/tmp_POST_STAT_PM25"
 mkdir_vrfy -p "$DATA"
 cd_vrfy $DATA
-
-set -x
-
 #
 #-----------------------------------------------------------------------
 #