Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 28 additions & 23 deletions regtests/bin/matrix_cmake_ncep
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ usage ()
{
cat 2>&1 << EOF

Usage: $myname model_dir
Usage: $myname model_dir
Required:
model_dir : path to model dir of WW3 source
EOF
Expand All @@ -41,46 +41,46 @@ EOF
# Convert main_dir to absolute path
main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`"

# Module Versions from HPC-Stack that are common for all platforms
# Module Versions from HPC-Stack that are common for all platforms
modnetcdf='netcdf/4.7.4'
modjasper='jasper/2.0.25'
modzlib='zlib/1.2.11'
modpng='png/1.6.35'
modpng='libpng/1.6.37'
modhdf5='hdf5/1.10.6'
modbacio='bacio/2.4.1'
modg2='g2/3.4.2'
modg2='g2/3.4.5'
modw3emc='w3emc/2.9.2'
modesmf='esmf/8_2_0'
modesmf='esmf/8.3.0b09'

# Set batchq queue, choose modules and other custom variables to fit system and
# Set batchq queue, choose modules and other custom variables to fit system and
# to define headers etc (default to original version if empty)
ishera=`hostname | grep hfe`
isorion=`hostname | grep Orion`
if [ $ishera ]
then
# If no other h, assuming Hera
# If no other h, assuming Hera
batchq='slurm'
hpcstackpath='/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack'
hpcstackversion='hpc/1.1.0'
modcomp='hpc-intel/18.0.5.274'
modmpi='hpc-impi/2018.0.4'
hpcstackversion='hpc/1.2.0'
modcomp='hpc-intel/2022.1.2'
modmpi='hpc-impi/2022.1.2'
metispath='/scratch2/COASTAL/coastal/save/Ali.Abdolali/hpc-stack/parmetis-4.0.3'
modcmake='cmake/3.20.1'
elif [ $isorion ]
then
batchq='slurm'
hpcstackpath='/apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack'
hpcstackversion='hpc/1.1.0'
modcomp='hpc-intel/2018.4'
modmpi='hpc-impi/2018.4'
hpcstackpath='/work/noaa/epic-ps/hpc-stack/libs/intel/2022.1.2/modulefiles/stack'
hpcstackversion='hpc/1.2.0'
modcomp='hpc-intel/2022.1.2'
modmpi='hpc-impi/2022.1.2'
metispath='/work/noaa/marine/ali.abdolali/Source/hpc-stack/parmetis-4.0.3'
modcmake='cmake/3.22.1'
else
batchq=
fi

# 1. Set up

export np='24' #number of mpi tasks
export npl='140' #number of mpi tasks for ufs applications and large setups
export npl1='100' #number of mpi tasks for ufs/large setups (b4b check)
Expand All @@ -102,7 +102,12 @@ EOF
echo '#SBATCH -J ww3_regtest' >> matrix.head
echo '#SBATCH -o matrix.out' >> matrix.head
echo '#SBATCH -p orion' >> matrix.head
echo '#SBATCH --exclusive' >> matrix.head
echo 'ulimit -s unlimited' >> matrix.head
echo 'ulimit -c 0' >> matrix.head
echo 'export KMP_STACKSIZE=2G' >> matrix.head
echo 'export FI_OFI_RXM_BUFFER_SIZE=128000' >> matrix.head
echo 'export FI_OFI_RXM_RX_SIZE=64000' >> matrix.head
elif [ $batchq = "slurm" ]
then
echo "#SBATCH -n ${np}" >> matrix.head
Expand Down Expand Up @@ -145,7 +150,7 @@ EOF
echo " export METIS_PATH=${metispath}" >> matrix.head
echo " export path_build_root=$(dirname $main_dir)/regtests/buildmatrix" >> matrix.head
echo ' [[ -d ${path_build_root} ]] && rm -rf ${path_build_root}' >> matrix.head
echo ' '
echo ' '

if [ "$batchq" = 'slurm' ]
then
Expand Down Expand Up @@ -186,7 +191,7 @@ EOF
export pdlib='y' # unstr with pdlib for domain decomposition and implicit solver
export smcgr='y' # SMC grid test
export rtd='y' # Rotated pole test
export mudice='y' # Mud/Ice and wave interaction tests
export mudice='y' # Mud/Ice and wave interaction tests
export infgrv='y' # Second harmonic generation tests
export uost='y' # ww3_ts4 Unresolved Obstacles Source Term (UOST)
export assim='y' # Restart spectra update
Expand All @@ -212,28 +217,28 @@ EOF
export nth_b4b='y' # Thread Reproducibility
export esmf='n' # ESMF coupling
# export filter='PR3 ST2 UQ'
# The filter does a set of consecutive greps on the
# The filter does a set of consecutive greps on the
# command lines generated by filter.base with the above
# selected options.

# --------------------------------------------------------------------------- #
# 2. Execute matrix.base ... #
# --------------------------------------------------------------------------- #

$main_dir/../regtests/bin/matrix.base

$main_dir/../regtests/bin/matrix_divider_cmake.sh
$main_dir/../regtests/bin/matrix_divider_cmake.sh


echo "#submit all of the diveded matrix files" > msuball.sh
if [ $batchq = "slurm" ]
if [ $batchq = "slurm" ]
then
files=`ls matrix??`
for file in $files
for file in $files
do
echo "sbatch < $file" >> msuball.sh
done
fi
fi

# --------------------------------------------------------------------------- #
# End to the matrix #
Expand Down