From e4259c07c75de01ecd53f1bda4828623b6d4a0a8 Mon Sep 17 00:00:00 2001 From: "jessica.meixner" Date: Tue, 11 Mar 2025 11:13:28 -0400 Subject: [PATCH 1/2] updates to run C1152 on gaea, also used more task/node for upp but not committing that change as it's likely a gaea only need --- env/GAEAC6.env | 10 +++++++++- parm/config/gfs/config.resources.GAEAC6 | 2 +- parm/config/gfs/config.ufs | 12 ++++-------- workflow/hosts/gaeac6.yaml | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 8819d1a1a74..e4383c0308b 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -227,7 +227,15 @@ case ${step} in ;; "fcst" | "efcs") - export OMP_STACKSIZE=512M + export OMP_NUM_THREADS=1 + export OMP_STACKSIZE=1024M + + export MPICH_COLL_SYNC=MPI_Bcast + export FI_VERBS_PREFER_XRC=0 + export FI_CXI_RX_MATCH_MODE=hybrid + export COMEX_EAGER_THRESHOLD=65536 + export FI_CXI_RDZV_THRESHOLD=65536 + export FI_CXI_DEFAULT_CQ_SIZE=1048576 (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) (( ufs_ntasks = nnodes*tasks_per_node )) diff --git a/parm/config/gfs/config.resources.GAEAC6 b/parm/config/gfs/config.resources.GAEAC6 index 719f7a9a4c4..97fed78f8a1 100644 --- a/parm/config/gfs/config.resources.GAEAC6 +++ b/parm/config/gfs/config.resources.GAEAC6 @@ -13,7 +13,7 @@ case ${step} in export tasks_per_node=144 ;; "C1152") - #TODO set tasks_per_node after investigating a safe threshold + export tasks_per_node=144 ;; *) # Nothing to do for other resolutions diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 839567419fa..2f326455c7f 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -299,8 +299,8 @@ case "${fv3_res}" in export DELTIM=150 export layout_x_gdas=8 export layout_y_gdas=16 - export layout_x_gfs=8 - export layout_y_gfs=16 + export layout_x_gfs=16 + export layout_y_gfs=24 export nthreads_fv3_gdas=4 export nthreads_fv3_gfs=4 export nthreads_ufs_gdas=4 @@ -605,7 +605,7 @@ if [[ "${skip_ww3}" == "false" ]]; then export cplwav=".true." export cplwav2atm=".true." model_list="${model_list}.wave" - nthreads_ww3=2 + nthreads_ww3=1 case "${ww3_res}" in "gnh_10m;aoc_9km;gsh_15m") ntasks_ww3=140 @@ -618,26 +618,22 @@ if [[ "${skip_ww3}" == "false" ]]; then ;; "glo_100") ntasks_ww3=20 - nthreads_ww3=1 ;; "glo_200") ntasks_ww3=30 - nthreads_ww3=1 ;; "glo_500") ntasks_ww3=12 - nthreads_ww3=1 ;; "mx025") ntasks_ww3=80 ;; "uglo_15km") ntasks_ww3=1000 - nthreads_ww3=1 + nthreads_ww3=2 ;; "uglo_100km") ntasks_ww3=40 - nthreads_ww3=1 ;; *) echo "FATAL ERROR: Unsupported WW3 resolution = ${ww3_res}, ABORT!" diff --git a/workflow/hosts/gaeac6.yaml b/workflow/hosts/gaeac6.yaml index d3544e47203..71ede484422 100644 --- a/workflow/hosts/gaeac6.yaml +++ b/workflow/hosts/gaeac6.yaml @@ -32,6 +32,6 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'YES' DO_GENESIS: 'YES' DO_METP: 'NO' -SUPPORTED_RESOLUTIONS: ['C768', 'C384', 'C192', 'C96', 'C48'] +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] AERO_INPUTS_DIR: '/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/gocart_emissions' USE_SCRONTAB: 'YES' From ca741099d5ca47c034bc5a76b1df47c196ec5c3b Mon Sep 17 00:00:00 2001 From: "jessica.meixner" Date: Tue, 11 Mar 2025 13:25:35 -0400 Subject: [PATCH 2/2] update based on review comments --- env/GAEAC6.env | 1 - 1 file changed, 1 deletion(-) diff --git a/env/GAEAC6.env b/env/GAEAC6.env index e4383c0308b..1da92e3cd16 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -227,7 +227,6 @@ case ${step} in ;; "fcst" | "efcs") - export OMP_NUM_THREADS=1 export OMP_STACKSIZE=1024M export MPICH_COLL_SYNC=MPI_Bcast