diff --git a/regtests/bin/matrix_cmake_ncep b/regtests/bin/matrix_cmake_ncep index 9ac8346602..4c4efd4b3d 100755 --- a/regtests/bin/matrix_cmake_ncep +++ b/regtests/bin/matrix_cmake_ncep @@ -22,7 +22,7 @@ usage () { cat 2>&1 << EOF - Usage: $myname model_dir + Usage: $myname model_dir Required: model_dir : path to model dir of WW3 source EOF @@ -41,38 +41,38 @@ EOF # Convert main_dir to absolute path main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" -# Module Versions from HPC-Stack that are common for all platforms +# Module Versions from HPC-Stack that are common for all platforms modnetcdf='netcdf/4.7.4' modjasper='jasper/2.0.25' modzlib='zlib/1.2.11' - modpng='png/1.6.35' + modpng='libpng/1.6.37' modhdf5='hdf5/1.10.6' modbacio='bacio/2.4.1' - modg2='g2/3.4.2' + modg2='g2/3.4.5' modw3emc='w3emc/2.9.2' - modesmf='esmf/8_2_0' + modesmf='esmf/8.3.0b09' -# Set batchq queue, choose modules and other custom variables to fit system and +# Set batchq queue, choose modules and other custom variables to fit system and # to define headers etc (default to original version if empty) ishera=`hostname | grep hfe` isorion=`hostname | grep Orion` if [ $ishera ] then - # If no other h, assuming Hera + # If no other h, assuming Hera batchq='slurm' hpcstackpath='/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack' - hpcstackversion='hpc/1.1.0' - modcomp='hpc-intel/18.0.5.274' - modmpi='hpc-impi/2018.0.4' + hpcstackversion='hpc/1.2.0' + modcomp='hpc-intel/2022.1.2' + modmpi='hpc-impi/2022.1.2' metispath='/scratch2/COASTAL/coastal/save/Ali.Abdolali/hpc-stack/parmetis-4.0.3' modcmake='cmake/3.20.1' elif [ $isorion ] then batchq='slurm' - hpcstackpath='/apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack' - hpcstackversion='hpc/1.1.0' - modcomp='hpc-intel/2018.4' - modmpi='hpc-impi/2018.4' + hpcstackpath='/work/noaa/epic-ps/hpc-stack/libs/intel/2022.1.2/modulefiles/stack' + hpcstackversion='hpc/1.2.0' + modcomp='hpc-intel/2022.1.2' + modmpi='hpc-impi/2022.1.2' metispath='/work/noaa/marine/ali.abdolali/Source/hpc-stack/parmetis-4.0.3' modcmake='cmake/3.22.1' else @@ -80,7 +80,7 @@ EOF fi # 1. Set up - + export np='24' #number of mpi tasks export npl='140' #number of mpi tasks for ufs applications and large setups export npl1='100' #number of mpi tasks for ufs/large setups (b4b check) @@ -102,7 +102,12 @@ EOF echo '#SBATCH -J ww3_regtest' >> matrix.head echo '#SBATCH -o matrix.out' >> matrix.head echo '#SBATCH -p orion' >> matrix.head + echo '#SBATCH --exclusive' >> matrix.head echo 'ulimit -s unlimited' >> matrix.head + echo 'ulimit -c 0' >> matrix.head + echo 'export KMP_STACKSIZE=2G' >> matrix.head + echo 'export FI_OFI_RXM_BUFFER_SIZE=128000' >> matrix.head + echo 'export FI_OFI_RXM_RX_SIZE=64000' >> matrix.head elif [ $batchq = "slurm" ] then echo "#SBATCH -n ${np}" >> matrix.head @@ -145,7 +150,7 @@ EOF echo " export METIS_PATH=${metispath}" >> matrix.head echo " export path_build_root=$(dirname $main_dir)/regtests/buildmatrix" >> matrix.head echo ' [[ -d ${path_build_root} ]] && rm -rf ${path_build_root}' >> matrix.head - echo ' ' + echo ' ' if [ "$batchq" = 'slurm' ] then @@ -186,7 +191,7 @@ EOF export pdlib='y' # unstr with pdlib for domain decomposition and implicit solver export smcgr='y' # SMC grid test export rtd='y' # Rotated pole test - export mudice='y' # Mud/Ice and wave interaction tests + export mudice='y' # Mud/Ice and wave interaction tests export infgrv='y' # Second harmonic generation tests export uost='y' # ww3_ts4 Unresolved Obstacles Source Term (UOST) export assim='y' # Restart spectra update @@ -212,28 +217,28 @@ EOF export nth_b4b='y' # Thread Reproducibility export esmf='n' # ESMF coupling # export filter='PR3 ST2 UQ' - # The filter does a set of consecutive greps on the + # The filter does a set of consecutive greps on the # command lines generated by filter.base with the above # selected options. # --------------------------------------------------------------------------- # # 2. Execute matrix.base ... # # --------------------------------------------------------------------------- # - + $main_dir/../regtests/bin/matrix.base - $main_dir/../regtests/bin/matrix_divider_cmake.sh + $main_dir/../regtests/bin/matrix_divider_cmake.sh echo "#submit all of the diveded matrix files" > msuball.sh - if [ $batchq = "slurm" ] + if [ $batchq = "slurm" ] then files=`ls matrix??` - for file in $files + for file in $files do echo "sbatch < $file" >> msuball.sh done - fi + fi # --------------------------------------------------------------------------- # # End to the matrix #