From 00cb943ea12cba91af76638f18576bcb1ef883fa Mon Sep 17 00:00:00 2001
From: MinsukJi-NOAA <Minsuk.Ji@noaa.gov>
Date: Thu, 29 Jul 2021 18:16:01 -0500
Subject: [PATCH 1/3] Update stampede related files

---
 modulefiles/ufs_stampede.intel       | 15 ++++++++-------
 tests/default_vars.sh                |  4 ++--
 tests/fv3_conf/fv3_slurm.IN_stampede |  4 ++--
 3 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/modulefiles/ufs_stampede.intel b/modulefiles/ufs_stampede.intel
index 458b91933f..ef80cfd18e 100644
--- a/modulefiles/ufs_stampede.intel
+++ b/modulefiles/ufs_stampede.intel
@@ -14,7 +14,7 @@ module-whatis "loads NEMS FV3 prerequisites for Stampede/Intel"
 ##
 ## NCEP libraries
 ##
-module use /work/08048/hanglei/NCEP/install/hpc-stack/modulefiles/stack
+module use /work2/08048/hanglei/stampede2/NCEP/install/hpc-stack/modulefiles
 module load hpc/1.1.0
 module load hpc-intel/18.0.2
 module load hpc-impi/18.0.2
@@ -22,14 +22,14 @@ module load python3/3.7.0
 
 module load bacio/2.4.1
 module load crtm/2.3.0
-module load g2/3.4.1
-module load g2tmpl/1.9.1
+module load g2/3.4.3
+module load g2tmpl/1.10.0
 module load ip/3.3.3
 module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
-module load upp/10.0.0
+module load upp/10.0.8
 
 module load gfsio/1.4.1
 module load sfcio/1.4.1
@@ -41,13 +41,14 @@ module load zlib/1.2.11
 
 module load hdf5/1.10.6
 module load netcdf/4.7.4
-module load pio/2.5.2
-module load esmf/8_1_0_beta_snapshot_47
+module load pio/2.5.3
+module load fms/2020.04.03
+module load esmf/8_1_1
 
 ##
 ## load cmake
 ##
-module load cmake/3.16.1
+module load cmake/3.20.1
 setenv CC mpiicc
 setenv CXX mpiicpc
 setenv FC mpiifort
diff --git a/tests/default_vars.sh b/tests/default_vars.sh
index 261ab446e1..7dd3c0c57f 100755
--- a/tests/default_vars.sh
+++ b/tests/default_vars.sh
@@ -358,7 +358,7 @@ elif [[ $MACHINE_ID = stampede.* ]]; then
   OPB_cpl_dflt_wwav="150 179"; IPB_cpl_dflt_wwav="180 191"; WPB_cpl_dflt_wwav="192 383"
 
   TASKS_cpl_thrd=120; TPN_cpl_thrd=24; INPES_cpl_thrd=3; JNPES_cpl_thrd=4
-  THRD_cpl_thrd=2; WPG_cpl_thrd=6;  MPB_cpl_thrd="0 77";  APB_cpl_thrd="0 77"
+  THRD_cpl_thrd=2; WPG_cpl_thrd=6;  MPB_cpl_thrd="0 71";  APB_cpl_thrd="0 77"
   OPB_cpl_thrd="78 107";  IPB_cpl_thrd="108 119"
 
   TASKS_cpl_bmrk=480; TPN_cpl_bmrk=48; INPES_cpl_bmrk=6; JNPES_cpl_bmrk=8
@@ -369,7 +369,7 @@ elif [[ $MACHINE_ID = stampede.* ]]; then
   THRD_cpl_wwav=1; WPG_cpl_wwav=24; MPB_cpl_wwav="0 287"; APB_cpl_wwav="0 311"
   OPB_cpl_wwav="312 431"; IPB_cpl_wwav="432 479"; WPB_cpl_wwav="480 527"
 
-  TASKS_cpl_c192=288; TPN_cpl_c192=40; INPES_cpl_c192=4; JNPES_cpl_c192=8
+  TASKS_cpl_c192=288; TPN_cpl_c192=48; INPES_cpl_c192=4; JNPES_cpl_c192=8
   THRD_cpl_c192=1; WPG_cpl_c192=12;  MPB_cpl_c192="0 191"; APB_cpl_c192="0 203"
   OPB_cpl_c192="204 263"; IPB_cpl_c192="264 287"
 
diff --git a/tests/fv3_conf/fv3_slurm.IN_stampede b/tests/fv3_conf/fv3_slurm.IN_stampede
index 0384da5966..c8383e9f86 100644
--- a/tests/fv3_conf/fv3_slurm.IN_stampede
+++ b/tests/fv3_conf/fv3_slurm.IN_stampede
@@ -2,9 +2,9 @@
 #SBATCH -o out
 #SBATCH -e err
 #SBATCH -N @[NODES]
+#SBATCH --ntasks-per-node=@[TPN]
 #SBATCH -A @[ACCNR]
 #SBATCH -p @[QUEUE]
-#SBATCH -n @[TASKS]
 #SBATCH -t 00:@[WLCLK]:00
 
 set -eux
@@ -30,7 +30,7 @@ export LD_BIND_NOW=1
 #sync && sleep 1
 
 #mpirun -prepend-rank -np $SBATCH_NP ./fv3.exe
-ibrun   ./fv3.exe
+ibrun -n @[TASKS] ./fv3.exe
 
 echo "Model ended:    " `date`
 echo -n " $( date +%s )," >> job_timestamp.txt

From d7877e0f8b7a0748b8481b213d8b29f43b9fe241 Mon Sep 17 00:00:00 2001
From: MinsukJi-NOAA <Minsuk.Ji@noaa.gov>
Date: Fri, 30 Jul 2021 07:11:03 -0500
Subject: [PATCH 2/3] Change work to work2

---
 tests/rt.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/rt.sh b/tests/rt.sh
index 98b7956eee..64277e36f1 100755
--- a/tests/rt.sh
+++ b/tests/rt.sh
@@ -332,7 +332,7 @@ elif [[ $MACHINE_ID = stampede.* ]]; then
   PARTITION=
   ACCNR=TG-EES200015
   dprefix=$SCRATCH/ufs-weather-model/run
-  DISKNM=/work/07736/minsukji/stampede2/ufs-weather-model/RT
+  DISKNM=/work2/07736/minsukji/stampede2/ufs-weather-model/RT
   STMP=$dprefix
   PTMP=$dprefix
   SCHEDULER=slurm

From 16668e3049a6bee2206c2f907cd5543bdb4b4991 Mon Sep 17 00:00:00 2001
From: MinsukJi-NOAA <Minsuk.Ji@noaa.gov>
Date: Fri, 30 Jul 2021 07:32:59 -0500
Subject: [PATCH 3/3] Fix module path

---
 modulefiles/ufs_stampede.intel | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modulefiles/ufs_stampede.intel b/modulefiles/ufs_stampede.intel
index ef80cfd18e..31d27016b6 100644
--- a/modulefiles/ufs_stampede.intel
+++ b/modulefiles/ufs_stampede.intel
@@ -14,7 +14,7 @@ module-whatis "loads NEMS FV3 prerequisites for Stampede/Intel"
 ##
 ## NCEP libraries
 ##
-module use /work2/08048/hanglei/stampede2/NCEP/install/hpc-stack/modulefiles
+module use /work2/08048/hanglei/stampede2/NCEP/install/hpc-stack/modulefiles/stack
 module load hpc/1.1.0
 module load hpc-intel/18.0.2
 module load hpc-impi/18.0.2