Fb matrix divider

.gitignore

@@ -45,6 +45,10 @@ manual/*.toc
 manual/*.out
 manual/*.dvi
 manual/*.pdf
+model?
+model??
+regtests/list*
+regtests/before
 regtests/matrix*
 regtests/*/work*
 regtests/*/input*/*.nc

regtests/bin/matrix_divider.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+# --------------------------------------------------------------------------- #
+# matrix_divider simply divides the main matrix into subsets with a given     #
+# number of test (i.e. maxlist=100)                                           #
+#                                                                             #
+# Remarks:                                                                    #
+# - Once the matrix is generated, this script can be execute. The user should #
+#   define the maxlist and this script divide the matrix into matrix? and     #
+#   submit the job using sbatch.                                              #
+#                                                                             #
+#                                                      Ali Abdolali           #
+#                                                      August 2018            #
+#                                                      March 2021             #
+#                                                                             #
+#    Copyright 2013 National Weather Service (NWS),                           #
+#       National Oceanic and Atmospheric Administration.  All rights          #
+#       reserved.  WAVEWATCH III is a trademark of the NWS.                   #
+#       No unauthorized use without permission.                               #
+#                                                                             #
+# --------------------------------------------------------------------------- #
+# --------------------------------------------------------------------------- #
+# 1.  clean up and definitions                                                #
+# --------------------------------------------------------------------------- #
+rm -r ../model?
+rm -r ../model??
+#Put the job requirement/spec in "before"
+sed -e "/run_test/,\$d" matrix > before
+#Put the list of tests in "list"
+command egrep 'run_test|ww3_tp2.14' matrix | cat >> list
+#Count the number of lines in "list"
+listn=$(wc -l < list)
+count=0
+countT=0
+# Define the number of test in each subset "maxlist"
+maxlist=100
+matrixno=1
+# --------------------------------------------------------------------------- #
+# 2.  Divide and dump in subsets                                              #
+# --------------------------------------------------------------------------- #
+while read line
+do
+if [[ count -eq 0 ]]
+then 
+cat before >> matrix$matrixno
+fi
+  if [[ count -le maxlist ]]
+     then
+     echo $line >> matrix$matrixno
+     (( count = count + 1 ))
+     (( countT = countT + 1 ))
+#     echo $count
+#     echo $countT
+ else
+ count=0
+ (( matrixno = matrixno + 1 ))
+  fi
+done < list
+
+  for i in `seq 1 1 "$((matrixno))"`; do
+#Replace matrix.out > matrix?.out, model > model?
+  cp -r ../model ../model$i
+  sed -i 's/'matrix.out'/'matrix${i}.out'/gI' matrix$i
+  sed -i 's/'model'/'model${i}'/gI' matrix$i
+  echo "  echo ' '"                                                                     >> matrix$i
+  echo "  echo '     **************************************************************'"   >> matrix$i
+  echo "  echo '     *  end of WAVEWATCH III matrix$i of regression tests         *'"   >> matrix$i
+  echo "  echo '     **************************************************************'"   >> matrix$i
+  echo "  echo ' '"                                                                     >> matrix$i
+done
+
+rm before
+rm list
+
+  echo "file matrix is divided into $i subsets ...."
+
+# --------------------------------------------------------------------------- #
+# End to matrix_divider                                                       #
+# --------------------------------------------------------------------------- #

regtests/bin/matrix_divider_p.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+# --------------------------------------------------------------------------- #
+# matrix_divider simply divides the main matrix into subsets with a given     #
+# number of test (i.e. maxlist1 for mpi tests and maxlist2 for serial tests)  #
+#                                                                             #
+# Remarks:                                                                    #
+# - Once the matrix is generated, this script can be execute. The user should #
+#   define the maxlist and this script divide the matrix into matrix? and     #
+#   submit the job using sbatch.                                              #
+#                                                                             #
+#                                                      Ali Abdolali           #
+#                                                      August 2018            #
+#                                                      March 2021             #
+#                                                                             #
+#    Copyright 2013 National Weather Service (NWS),                           #
+#       National Oceanic and Atmospheric Administration.  All rights          #
+#       reserved.  WAVEWATCH III is a trademark of the NWS.                   #
+#       No unauthorized use without permission.                               #
+#                                                                             #
+# --------------------------------------------------------------------------- #
+# --------------------------------------------------------------------------- #
+# 1.  clean up and definitions                                                #
+# --------------------------------------------------------------------------- #
+rm -r ../model?
+rm -r ../model??
+cp matrix matrix.tmp
+
+maxlist1=47
+maxlist2=91
+
+#Put the job requirement/spec in "before"
+sed -e "/run_test/,\$d" matrix.tmp > before
+#Put the list of tests in "list"
+command egrep 'ww3_tp2.14|ww3_tp2.17' matrix.tmp | cat >> list_heavy
+awk '!/ww3_tp2.14/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp
+awk '!/ww3_tp2.17/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp
+command egrep 'mpirun|mpiexec|MPI_LAUNCH' matrix.tmp | cat >> list_mpi
+awk '!/mpirun|mpiexec|MPI_LAUNCH/' matrix.tmp > tmpfile && mv tmpfile matrix.tmp
+split -dl $maxlist1 list_mpi list_mpi_
+rm list_mpi
+matrixno1=$(ls list_mpi_* | wc -l)
+echo "Total nummber of matrix with parallel tests = $(($matrixno1 + 1)); each includes $maxlist1 tests"
+command egrep 'run_test' matrix.tmp | cat >> list_serial
+split -dl $maxlist2 list_serial list_serial_
+rm list_serial
+matrixno2=$(ls list_serial_* | wc -l)
+echo "Total nummber of matrix with serial test = $matrixno2; each includes $maxlist2 tests"
+rm matrix.tmp
+
+# --------------------------------------------------------------------------- #
+# 2.  Divide and dump in subsets                                              #
+# --------------------------------------------------------------------------- #
+# parallel jobs
+count=0
+  for i in `seq -f '%02g' 0 "$((matrixno1 - 1))"`; do
+#echo $i
+#Replace matrix.out > matrix?.out, model > model?
+  (( count = count + 1 ))
+  cat before >> matrix$count
+  cat list_mpi_$i >> matrix$count
+  cp -r ../model ../model$count
+  sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix$count
+  sed -i 's/'model'/'model${count}'/gI' matrix$count
+  echo "  echo ' '"                                                                     >> matrix$count
+  echo "  echo '     **************************************************************'"   >> matrix$count
+  echo "  echo '     *  end of WAVEWATCH III matrix$count of regression tests     *'"   >> matrix$count
+  echo "  echo '     **************************************************************'"   >> matrix$count
+  echo "  echo ' '"                                                                     >> matrix$count
+  echo "rm -r ../model$count"                                                              >> matrix$count
+  echo " matrix$count prepared"
+ done
+
+#serial jobs
+ for i in `seq -f '%02g' 0 "$((matrixno2 - 1))"`; do
+#echo $i
+#Replace matrix.out > matrix?.out, model > model?
+  (( count = count + 1 ))
+  cat before >> matrix$count
+  cat list_serial_$i >> matrix$count
+  cp -r ../model ../model$count
+  sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix$count
+  sed -i 's/'model'/'model${count}'/gI' matrix$count
+  echo "  echo ' '"                                                                     >> matrix$count
+  echo "  echo '     **************************************************************'"   >> matrix$count
+  echo "  echo '     *  end of WAVEWATCH III matrix$count of regression tests     *'"   >> matrix$count
+  echo "  echo '     **************************************************************'"   >> matrix$count
+  echo "  echo ' '"                                                                     >> matrix$count
+  echo "rm -r ../model$count"                                                              >> matrix$count
+  echo " matrix$count prepared"
+ done
+
+#ww3_tp2.14 is separated, as it has dependency. 
+#ww3_tp2.17 is separated, as it takes a long time to finish
+  (( count = count + 1 ))
+  cat before >> matrix$count
+  cat list_heavy >> matrix$count
+  cp -r ../model ../model$count
+  sed -i 's/'matrix.out'/'matrix${count}.out'/gI' matrix$count
+  sed -i 's/'model'/'model${count}'/gI' matrix$count
+  echo "  echo ' '"                                                                     >> matrix$count
+  echo "  echo '     **************************************************************'"   >> matrix$count
+  echo "  echo '     *  end of WAVEWATCH III matrix$count of regression tests     *'"   >> matrix$count
+  echo "  echo '     **************************************************************'"   >> matrix$count
+  echo "  echo ' '"                                                                     >> matrix$count
+  echo "rm -r ../model$count"                                                              >> matrix$count
+  echo " matrix$count prepared"
+
+
+
+rm before
+rm list*
+
+  echo "file matrix is divided into $count subsets ...."
+
+# --------------------------------------------------------------------------- #
+# End to matrix_divider                                                       #
+# --------------------------------------------------------------------------- #
+