diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake new file mode 100644 index 000000000..bf43069c5 --- /dev/null +++ b/cmake/LibMPI.cmake @@ -0,0 +1,217 @@ +# This file contains CMake code related to MPI. (Taken from the +# ParallelIO project.) + +# Jim Edwards +include (CMakeParseArguments) + +# Find Valgrind to perform memory leak check +if (PIO_VALGRIND_CHECK) + find_program (VALGRIND_COMMAND NAMES valgrind) + if (VALGRIND_COMMAND) + set (VALGRIND_COMMAND_OPTIONS --leak-check=full --show-reachable=yes) + else () + message (WARNING "Valgrind not found: memory leak check could not be performed") + set (VALGRIND_COMMAND "") + endif () +endif () + +# +# - Functions for parallel testing with CTest +# + +#============================================================================== +# - Get the machine platform-specific +# +# Syntax: platform_name (RETURN_VARIABLE) +# +function (platform_name RETURN_VARIABLE) + + # Determine platform name from site name... + site_name (SITENAME) + + # hera + if (SITENAME MATCHES "^hfe01" OR + SITENAME MATCHES "^hfe02" OR + SITENAME MATCHES "^hfe03" OR + SITENAME MATCHES "^hfe04" OR + SITENAME MATCHES "^hfe05" OR + SITENAME MATCHES "^hfe06" OR + SITENAME MATCHES "^hfe07" OR + SITENAME MATCHES "^hfe08" OR + SITENAME MATCHES "^hfe09" OR + SITENAME MATCHES "^hfe10" OR + SITENAME MATCHES "^hfe11" OR + SITENAME MATCHES "^hfe12") + + set (${RETURN_VARIABLE} "hera" PARENT_SCOPE) + + # wcoss_cray (Luna) + elseif (SITENAME MATCHES "^llogin1" OR + SITENAME MATCHES "^llogin2" OR + SITENAME MATCHES "^llogin3") + + set (${RETURN_VARIABLE} "wcoss_cray" PARENT_SCOPE) + + # wcoss_cray (Surge) + elseif (SITENAME MATCHES "^slogin1" OR + SITENAME MATCHES "^slogin2" OR + SITENAME MATCHES "^slogin3") + + set (${RETURN_VARIABLE} "wcoss_cray" PARENT_SCOPE) + + # wcoss_dell_p3 (Venus) + elseif (SITENAME MATCHES "^v71a1.ncep.noaa.gov" OR + SITENAME MATCHES "^v71a2.ncep.noaa.gov" OR + SITENAME MATCHES "^v71a3.ncep.noaa.gov" OR + SITENAME MATCHES "^v72a1.ncep.noaa.gov" OR + SITENAME MATCHES "^v72a2.ncep.noaa.gov" OR + SITENAME MATCHES "^v72a3.ncep.noaa.gov") + + set (${RETURN_VARIABLE} "wcoss_dell_p3" PARENT_SCOPE) + + # wcoss_dell_p3 (Mars) + elseif (SITENAME MATCHES "^m71a1.ncep.noaa.gov" OR + SITENAME MATCHES "^m71a2.ncep.noaa.gov" OR + SITENAME MATCHES "^m71a3.ncep.noaa.gov" OR + SITENAME MATCHES "^m72a1.ncep.noaa.gov" OR + SITENAME MATCHES "^m72a2.ncep.noaa.gov" OR + SITENAME MATCHES "^m72a3.ncep.noaa.gov") + + set (${RETURN_VARIABLE} "wcoss_dell_p3" PARENT_SCOPE) + + # wcoss2 + elseif (SITENAME MATCHES "^along01" OR + SITENAME MATCHES "^alogin02") + + set (${RETURN_VARIABLE} "wcoss2" PARENT_SCOPE) + + # gaea + elseif (SITENAME MATCHES "^gaea9" OR + SITENAME MATCHES "^gaea10" OR + SITENAME MATCHES "^gaea11" OR + SITENAME MATCHES "^gaea12" OR + SITENAME MATCHES "^gaea13" OR + SITENAME MATCHES "^gaea14" OR + SITENAME MATCHES "^gaea15" OR + SITENAME MATCHES "^gaea16" OR + SITENAME MATCHES "^gaea9.ncrc.gov" OR + SITENAME MATCHES "^gaea10.ncrc.gov" OR + SITENAME MATCHES "^gaea11.ncrc.gov" OR + SITENAME MATCHES "^gaea12.ncrc.gov" OR + SITENAME MATCHES "^gaea13.ncrc.gov" OR + SITENAME MATCHES "^gaea14.ncrc.gov" OR + SITENAME MATCHES "^gaea15.ncrc.gov" OR + SITENAME MATCHES "^gaea16.ncrc.gov") + + set (${RETURN_VARIABLE} "gaea" PARENT_SCOPE) + + # jet + elseif (SITENAME MATCHES "^fe1" OR + SITENAME MATCHES "^fe2" OR + SITENAME MATCHES "^fe3" OR + SITENAME MATCHES "^fe4" OR + SITENAME MATCHES "^fe5" OR + SITENAME MATCHES "^fe6" OR + SITENAME MATCHES "^fe7" OR + SITENAME MATCHES "^fe8" OR + SITENAME MATCHES "^tfe1" OR + SITENAME MATCHES "^tfe2") + + set (${RETURN_VARIABLE} "jet" PARENT_SCOPE) + + elseif (SITENAME MATCHES "^Orion-login-1.HPC.MsState.Edu" OR + SITENAME MATCHES "^Orion-login-2.HPC.MsState.Edu" OR + SITENAME MATCHES "^Orion-login-3.HPC.MsState.Edu" OR + SITENAME MATCHES "^Orion-login-4.HPC.MsState.Edu") + + set (${RETURN_VARIABLE} "orion" PARENT_SCOPE) + + elseif (SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne2.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne3.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne4.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne5.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne6.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne1.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne2.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne3.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne4.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne5.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^cheyenne6.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^chadmin1.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^chadmin2.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^chadmin3.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^chadmin4.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^chadmin5.ib0.cheyenne.ucar.edu" OR + SITENAME MATCHES "^chadmin6.ib0.cheyenne.ucar.edu") + + set (${RETURN_VARIABLE} "cheyenne" PARENT_SCOPE) + elseif (SITENAME MATCHES "^login1.stampede2.tacc.utexas.edu" OR + SITENAME MATCHES "^login2.stampede2.tacc.utexas.edu" OR + SITENAME MATCHES "^login3.stampede2.tacc.utexas.edu" OR + SITENAME MATCHES "^login4.stampede2.tacc.utexas.edu") + + + set (${RETURN_VARIABLE} "stampede" PARENT_SCOPE) + + else () + + set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE) + + endif () +endfunction () + +#============================================================================== +# - Add a new parallel test +# +# Syntax: add_mpi_test ( +# EXECUTABLE +# ARGUMENTS ... +# NUMPROCS +# TIMEOUT ) +function (add_mpi_test TESTNAME) + + # Parse the input arguments + set (options) + set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE) + set (multiValueArgs ARGUMENTS) + cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + # Store parsed arguments for convenience + set (exec_file ${${TESTNAME}_EXECUTABLE}) + set (exec_args ${${TESTNAME}_ARGUMENTS}) + set (num_procs ${${TESTNAME}_NUMPROCS}) + set (timeout ${${TESTNAME}_TIMEOUT}) + + # Get the platform name + platform_name (PLATFORM) + + get_property(WITH_MPIEXEC GLOBAL PROPERTY WITH_MPIEXEC) + if (WITH_MPIEXEC) + set(MPIEXEC "${WITH_MPIEXEC}") + endif () + + # Default ("unknown" platform) execution + if (PLATFORM STREQUAL "unknown") + + # Run tests directly from the command line + set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} + ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} + ${MPIEXEC_POSTFLAGS} ${exec_args}) + + else () + + # Run tests from the platform-specific executable + set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} + ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args}) + + endif () + + # Add the test to CTest + add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD}) + + # Adjust the test timeout + set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout}) + +endfunction() diff --git a/cmake/mpiexec.hera b/cmake/mpiexec.hera new file mode 100755 index 000000000..332b33e29 --- /dev/null +++ b/cmake/mpiexec.hera @@ -0,0 +1,15 @@ +#!/bin/bash +# +# Arguments: +# +# $1 - Number of MPI Tasks +# $2+ - Executable and its arguments +# + +ACCOUNT= +QOS=debug + +NP=$1 +shift + +srun -A $ACCOUNT -q $QOS -n $NP $@ diff --git a/cmake/mpiexec.jet b/cmake/mpiexec.jet new file mode 100755 index 000000000..332b33e29 --- /dev/null +++ b/cmake/mpiexec.jet @@ -0,0 +1,15 @@ +#!/bin/bash +# +# Arguments: +# +# $1 - Number of MPI Tasks +# $2+ - Executable and its arguments +# + +ACCOUNT= +QOS=debug + +NP=$1 +shift + +srun -A $ACCOUNT -q $QOS -n $NP $@ diff --git a/cmake/mpiexec.orion b/cmake/mpiexec.orion new file mode 100755 index 000000000..332b33e29 --- /dev/null +++ b/cmake/mpiexec.orion @@ -0,0 +1,15 @@ +#!/bin/bash +# +# Arguments: +# +# $1 - Number of MPI Tasks +# $2+ - Executable and its arguments +# + +ACCOUNT= +QOS=debug + +NP=$1 +shift + +srun -A $ACCOUNT -q $QOS -n $NP $@ diff --git a/tests/chres_cube/CMakeLists.txt b/tests/chres_cube/CMakeLists.txt index 0d3232d68..893b0e914 100644 --- a/tests/chres_cube/CMakeLists.txt +++ b/tests/chres_cube/CMakeLists.txt @@ -3,6 +3,9 @@ # # George Gayno, Lin Gan, Ed Hartnett +# Include cmake to allow parallel I/O tests. +include (LibMPI) + if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -convert big_endian -assume byterecl") elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$") @@ -51,7 +54,6 @@ add_executable(ftst_program_setup "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/input "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/search_util.F90" "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/utils.F90" ftst_program_setup.F90) -add_test(NAME ftst_program_setup COMMAND ftst_program_setup) target_link_libraries( ftst_program_setup nemsio::nemsio @@ -69,5 +71,10 @@ if(OpenMP_Fortran_FOUND) target_link_libraries(ftst_program_setup OpenMP::OpenMP_Fortran) endif() +# Cause test to be run with MPI. +add_mpi_test(ftst_program_setup + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ftst_program_setup + NUMPROCS 4 + TIMEOUT 60) diff --git a/tests/chres_cube/ftst_program_setup.F90 b/tests/chres_cube/ftst_program_setup.F90 index ddb6064ae..759324a38 100644 --- a/tests/chres_cube/ftst_program_setup.F90 +++ b/tests/chres_cube/ftst_program_setup.F90 @@ -3,15 +3,21 @@ ! Ed Hartnett 2/16/21 program ftst_program_setup + use mpi use esmf use netcdf use program_setup implicit none integer :: is + integer :: my_rank, nprocs + integer :: ierr - print*, "Starting test of program_setup." + call mpi_init(ierr) + call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr) + call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr) - print*, "testing read_setup_namelist with file fort.41..." + if (my_rank .eq. 0) print*, "Starting test of program_setup." + if (my_rank .eq. 0) print*, "testing read_setup_namelist with file fort.41..." call read_setup_namelist() if (cycle_mon .ne. 7 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 12) stop 4 if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 5 @@ -43,9 +49,9 @@ program ftst_program_setup tracers(is) = "NULL" tracers_input(is) = "NULL" enddo - print*, "OK" + if (my_rank .eq. 0) print*, "OK" - print*, "testing read_setup_namelist with config_fv3_tiled..." + if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_fv3_tiled..." call read_setup_namelist("config_fv3_tiled.nml") if (cycle_mon .ne. 10 .or. cycle_day .ne. 3 .or. cycle_hour .ne. 0) stop 34 if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 35 @@ -73,14 +79,14 @@ program ftst_program_setup tracers(is) = "NULL" tracers_input(is) = "NULL" enddo - print*, "OK" + if (my_rank .eq. 0) print*, "OK" ! Reading this namelist fails for some reason. ! print*, "testing read_setup_namelist with config_fv3_tiled_warm_restart..." ! call read_setup_namelist("config_fv3_tiled_warm_restart.nml") ! print*, "OK" - print*, "testing read_setup_namelist with config_gaussian_nemsio..." + if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_gaussian_nemsio..." call read_setup_namelist("config_gaussian_nemsio.nml") if (cycle_mon .ne. 7 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 12) stop 74 if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 75 @@ -98,7 +104,6 @@ program ftst_program_setup if (atm_weight_file .ne. "NULL") stop 88 if (trim(input_type) .ne. "gaussian_nemsio") stop 89 if (trim(external_model) .ne. "GFS") stop 90 - print *,num_tracers if (num_tracers .ne. 7) stop 21 if (tracers(1) .ne. "sphum" .or. tracers(2) .ne. "liq_wat" .or. tracers(3) .ne. "o3mr" .or. & tracers(4) .ne. "ice_wat" .or. tracers(5) .ne. "rainwat" .or. tracers(6) .ne. "snowwat" .or. & @@ -113,9 +118,9 @@ program ftst_program_setup tracers(is) = "NULL" tracers_input(is) = "NULL" enddo - print*, "OK" + if (my_rank .eq. 0) print*, "OK" - print*, "testing read_setup_namelist with config_spectral_sigio..." + if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_spectral_sigio..." call read_setup_namelist("config_spectral_sigio.nml") if (cycle_mon .ne. 7 .or. cycle_day .ne. 17 .or. cycle_hour .ne. 0) stop 114 if (.not. convert_atm .or. .not. convert_sfc .or. convert_nst) stop 115 @@ -143,9 +148,9 @@ program ftst_program_setup tracers(is) = "NULL" tracers_input(is) = "NULL" enddo - print*, "OK" + if (my_rank .eq. 0) print*, "OK" - print*, "testing read_setup_namelist with config_gfs_grib2..." + if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_gfs_grib2..." call read_setup_namelist("config_gfs_grib2.nml") if (cycle_mon .ne. 11 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 0) stop 94 if (.not. convert_atm .or. .not. convert_sfc .or. convert_nst) stop 95 @@ -177,7 +182,9 @@ program ftst_program_setup tracers(is) = "NULL" tracers_input(is) = "NULL" enddo - print*, "OK" + if (my_rank .eq. 0) print*, "OK" - print*, "SUCCESS!" + if (my_rank .eq. 0) print*, "SUCCESS!" + + call mpi_finalize(ierr) end program ftst_program_setup