Allow for C1152 ATM-Aero cycled DA to run on WCOSS2

parm/config/gfs/config.aeroanlgenb

@@ -12,17 +12,16 @@ export JEDI_CONFIG_YAML="${PARMgfs}/gdas/aero_bmat_jedi_config.yaml.j2"
 export JCB_BASE_YAML="${PARMgfs}/gdas/aero/jcb-base.yaml.j2"
 export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2"
 export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2"
-export aero_diffusion_iter=10
-export aero_diffusion_horiz_len=2500e3
-export aero_diffusion_fixed_val=1.0
+export aero_diffusion_iter=200
+export aero_diffusion_horiz_len=300e3
+export aero_diffusion_fixed_val=20.0
 export npx_clim_b=97
 export npy_clim_b=97
 export aero_diagb_weight=0.9
 export aero_staticb_rescaling_factor=2.0
-export aero_diagb_rescale=20.0
 export aero_diagb_n_halo=4
 export aero_diagb_n_neighbors=16
-export aero_diagb_smooth_horiz_iter=0
+export aero_diagb_smooth_horiz_iter=200
 export aero_diagb_smooth_vert_iter=0
 
 echo "END: config.aeroanlgenb"

parm/config/gfs/config.resources

@@ -399,19 +399,23 @@ case ${step} in
       "C1152" | "C768")
         layout_x=8
         layout_y=8
+        walltime="00:30:00"
         ;;
       "C384")
         layout_x=6
         layout_y=6
+        walltime="00:20:00"
         ;;
       "C192" | "C96")
         layout_x=4
         layout_y=4
+        walltime="00:10:00"
         ;;
       "C48" )
         # this case is for testing only
         layout_x=1
         layout_y=1
+        walltime="00:10:00"
         ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
@@ -420,7 +424,6 @@ case ${step} in
 
     export layout_x
     export layout_y
-    walltime="00:10:00"
     ntasks=1
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
@@ -432,19 +435,23 @@ case ${step} in
       "C1152" | "C768")
         layout_x=8
         layout_y=8
+        walltime="00:45:00"
         ;;
       "C384")
         layout_x=6
         layout_y=6
+        walltime="00:30:00"
         ;;
       "C192" | "C96")
         layout_x=4
         layout_y=4
+        walltime="00:20:00"
         ;;
       "C48" )
         # this case is for testing only
         layout_x=1
         layout_y=1
+        walltime="00:20:00"
         ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
@@ -454,9 +461,8 @@ case ${step} in
     export layout_x
     export layout_y
 
-    walltime="00:30:00"
     ntasks=$(( layout_x * layout_y * 6 ))
-    threads_per_task=1
+    threads_per_task=5
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     export is_exclusive=True
     ;;
@@ -498,11 +504,36 @@ case ${step} in
 
 
   "aeroanlfinal")
-    walltime="00:10:00"
+    case ${CASE} in
+      "C1152" | "C768")
+        layout_x=8
+        layout_y=8
+        walltime="00:30:00"
+        ;;
+      "C384")
+        layout_x=6
+        layout_y=6
+        walltime="00:20:00"
+        ;;
+      "C192" | "C96")
+        layout_x=4
+        layout_y=4
+        walltime="00:10:00"
+        ;;
+      "C48" )
+        # this case is for testing only
+        layout_x=1
+        layout_y=1
+        walltime="00:10:00"
+        ;;
+      *)
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
+        exit 4
+    esac
     ntasks=1
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
-    memory="3072M"
+    memory="13072M"
     ;;
 
   "marineanlinit")
@@ -888,6 +919,7 @@ case ${step} in
     declare -x "threads_per_task"="${UFS_THREADS}"
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
 
+    # TODO: make walltimes APP dependent
     case "${CASE}" in
       "C48" | "C96" | "C192")
         declare -x "walltime_gdas"="00:20:00"
@@ -903,7 +935,7 @@ case ${step} in
         ;;
       "C768" | "C1152")
         # Not valid resolutions for ensembles
-        declare -x "walltime_gdas"="00:50:00"
+        declare -x "walltime_gdas"="01:20:00"
         declare -x "walltime_gfs"="06:00:00"
         ;;
       *)

parm/config/gfs/config.resources.WCOSS2

@@ -5,6 +5,7 @@
 case ${step} in
   "prep")
     export is_exclusive=True
+    export tasks_per_node=5
     export memory="480GB"
     ;;
 

ush/load_ufsda_modules.sh

@@ -41,7 +41,6 @@ case "${MACHINE_ID}" in
       # TODO: Add path to GDASApp libraries and cray-mpich as temporary patches
       # TODO: Remove LD_LIBRARY_PATH lines as soon as permanent solutions are available	
       export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${HOMEgfs}/sorc/gdas.cd/build/lib"
-      export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/cray/pe/mpich/8.1.19/ofi/intel/19.0/lib"
     fi
     module load "${MODS}/${MACHINE_ID}"
     ncdump=$( command -v ncdump )

ush/module-setup.sh

@@ -52,7 +52,7 @@ elif [[ ${MACHINE_ID} = s4* ]] ; then
 elif [[ ${MACHINE_ID} = wcoss2 ]]; then
     # We are on WCOSS2
     # Ignore default modules of the same version lower in the search path (req'd by spack-stack)
-    #export LMOD_TMOD_FIND_FIRST=yes #TODO: Uncomment this when using spack-stack
+    #export LMOD_TMOD_FIND_FIRST=yes #TODO: Uncomment this when using spack-stack for the entire workflow
     module reset
 
 elif [[ ${MACHINE_ID} = cheyenne* ]] ; then