From b9988dfbea8ffc35012d766966d87885a1c5bf9c Mon Sep 17 00:00:00 2001 From: jcschaff Date: Sun, 28 Jul 2024 11:42:24 -0400 Subject: [PATCH] smoke test for NFsim runs with bash and python3 on all platforms --- .github/workflows/cd.yml | 6 +- .gitignore | 3 + CMakeLists.txt | 8 +- Dockerfile | 78 +++++++++---------- NFsim_v1.11/CMakeLists.txt | 19 ++++- .../smoke/SimID_273069657_0_.gdat.expected | 12 +++ .../tests/smoke/SimID_273069657_0_.nfsimInput | 1 + .../smoke/SimID_273069657_0_.species.expected | 5 ++ NFsim_v1.11/tests/smoke/smoke.py | 69 ++++++++++++++++ NFsim_v1.11/tests/smoke/smoke.sh | 62 +++++++++++++++ .../include/VCELL/SimulationMessaging.h | 4 +- VCellMessaging/src/SimulationMessaging.cpp | 5 +- 12 files changed, 227 insertions(+), 45 deletions(-) create mode 100644 NFsim_v1.11/tests/smoke/SimID_273069657_0_.gdat.expected create mode 100644 NFsim_v1.11/tests/smoke/SimID_273069657_0_.nfsimInput create mode 100644 NFsim_v1.11/tests/smoke/SimID_273069657_0_.species.expected create mode 100755 NFsim_v1.11/tests/smoke/smoke.py create mode 100755 NFsim_v1.11/tests/smoke/smoke.sh diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 478669a89..ec460a97b 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -64,6 +64,7 @@ jobs: brew install llvm echo 'export PATH="/usr/local/opt/llvm/bin:$PATH"' >> /Users/runner/.bash_profile source /Users/runner/.bash_profile + ln -s $(which gfortran-14) /usr/local/bin/gfortran gcc --version gfortran --version @@ -218,7 +219,7 @@ jobs: mkdir build cd build - PATH="/c/Program\ Files/LLVM/bin:$PATH" + export PATH="/c/Program\ Files/LLVM/bin:$PATH" cmake \ -G Ninja \ @@ -248,6 +249,9 @@ jobs: echo "working dir is $PWD" cd build + + export PATH="/d/a/_temp/msys64/clang64/bin:$PATH" + pacman -Sy --noconfirm diffutils ctest -VV diff --git a/.gitignore b/.gitignore index 2c0fede2f..f006e5125 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ /all_solvers/* .DS_Store + +NFsim_v1.11/tests/smoke/SimID_273069657_0_.gdat +NFsim_v1.11/tests/smoke/SimID_273069657_0_.species diff --git a/CMakeLists.txt b/CMakeLists.txt index 22c78cfb6..d2291161c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,7 +127,13 @@ if (${OPTION_TARGET_DOCS}) endif() if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + if (APPLE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + elseif (MINGW) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + else () + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++") + endif() set(CMAKE_OSX_DEPLOYMENT_TARGET "10.7" CACHE STRING "Choose minimum deploy target for Macos machines") endif() diff --git a/Dockerfile b/Dockerfile index a3dee25b0..4e495e5c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,22 @@ -FROM ubuntu:20.04 as build +FROM ubuntu:22.04 as build RUN apt-get -y update && apt-get install -y apt-utils && \ apt-get install -y -qq -o=Dpkg::Use-Pty=0 build-essential gfortran zlib1g-dev \ - libhdf5-dev libcurl4-openssl-dev libboost-dev cmake wget python - -# -# build PETSc with mpich for static linking -# -RUN mkdir /usr/local/petsc && \ - cd /usr/local/petsc && \ - wget http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.7.7.tar.gz && \ - tar xzf petsc-3.7.7.tar.gz && \ - cd petsc-3.7.7 && \ - ./configure --with-shared-libraries=0 --download-fblaslapack=1 --with-debugging=1 --download-mpich && \ - make PETSC_DIR=/usr/local/petsc/petsc-3.7.7 PETSC_ARCH=arch-linux2-c-debug all - -ENV PETSC_DIR=/usr/local/petsc/petsc-3.7.7 \ - PETSC_ARCH=arch-linux2-c-debug + libhdf5-dev libcurl4-openssl-dev libboost-dev cmake wget python3 + +## +## build PETSc with mpich for static linking +## +#RUN mkdir /usr/local/petsc && \ +# cd /usr/local/petsc && \ +# wget http://ftp.mcs.anl.gov/pub/petsc/release-snapshots/petsc-3.7.7.tar.gz && \ +# tar xzf petsc-3.7.7.tar.gz && \ +# cd petsc-3.7.7 && \ +# ./configure --with-shared-libraries=0 --download-fblaslapack=1 --with-debugging=1 --download-mpich && \ +# make PETSC_DIR=/usr/local/petsc/petsc-3.7.7 PETSC_ARCH=arch-linux2-c-debug all + +#ENV PETSC_DIR=/usr/local/petsc/petsc-3.7.7 \ +# PETSC_ARCH=arch-linux2-c-debug COPY . /vcellroot @@ -41,30 +41,30 @@ RUN /usr/bin/cmake \ -DOPTION_TARGET_HY3S_SOLVERS=OFF \ .. && \ make && \ - ctest + ctest -VV +## +## build FiniteVolume with PETSc (FiniteVolume_PETSc_x64) +## +#RUN mkdir -p /vcellroot/build_PETSc/bin +#WORKDIR /vcellroot/build_PETSc # -# build FiniteVolume with PETSc (FiniteVolume_PETSc_x64) -# -RUN mkdir -p /vcellroot/build_PETSc/bin -WORKDIR /vcellroot/build_PETSc - -RUN /usr/bin/cmake \ - -DOPTION_TARGET_MESSAGING=ON \ - -DOPTION_TARGET_PARALLEL=OFF \ - -DOPTION_TARGET_PETSC=ON \ - -DOPTION_TARGET_CHOMBO2D_SOLVER=OFF \ - -DOPTION_TARGET_CHOMBO3D_SOLVER=OFF \ - -DOPTION_TARGET_SMOLDYN_SOLVER=OFF \ - -DOPTION_TARGET_FV_SOLVER=ON \ - -DOPTION_TARGET_STOCHASTIC_SOLVER=OFF \ - -DOPTION_TARGET_NFSIM_SOLVER=OFF \ - -DOPTION_TARGET_MOVINGBOUNDARY_SOLVER=OFF \ - -DOPTION_TARGET_SUNDIALS_SOLVER=OFF \ - -DOPTION_TARGET_HY3S_SOLVERS=OFF \ - .. && \ - make && \ - ctest +#RUN /usr/bin/cmake \ +# -DOPTION_TARGET_MESSAGING=ON \ +# -DOPTION_TARGET_PARALLEL=OFF \ +# -DOPTION_TARGET_PETSC=ON \ +# -DOPTION_TARGET_CHOMBO2D_SOLVER=OFF \ +# -DOPTION_TARGET_CHOMBO3D_SOLVER=OFF \ +# -DOPTION_TARGET_SMOLDYN_SOLVER=OFF \ +# -DOPTION_TARGET_FV_SOLVER=ON \ +# -DOPTION_TARGET_STOCHASTIC_SOLVER=OFF \ +# -DOPTION_TARGET_NFSIM_SOLVER=OFF \ +# -DOPTION_TARGET_MOVINGBOUNDARY_SOLVER=OFF \ +# -DOPTION_TARGET_SUNDIALS_SOLVER=OFF \ +# -DOPTION_TARGET_HY3S_SOLVERS=OFF \ +# .. && \ +# make && \ +# ctest FROM eclipse-temurin:17 as jre-build @@ -96,7 +96,7 @@ RUN apt-get install -qq -y -o=Dpkg::Use-Pty=0 gcc gfortran zlib1g \ libhdf5-103 libhdf5-cpp-103 libcurl4-openssl-dev zip COPY --from=build /vcellroot/build/bin /vcellbin -COPY --from=build /vcellroot/build_PETSc/bin/FiniteVolume_PETSc_x64 /vcellbin/ +#COPY --from=build /vcellroot/build_PETSc/bin/FiniteVolume_PETSc_x64 /vcellbin/ WORKDIR /vcellbin ENV PATH=/vcellbin:$PATH diff --git a/NFsim_v1.11/CMakeLists.txt b/NFsim_v1.11/CMakeLists.txt index 1b449ef44..68771ed82 100755 --- a/NFsim_v1.11/CMakeLists.txt +++ b/NFsim_v1.11/CMakeLists.txt @@ -87,10 +87,27 @@ endif (UNIX) set(exe_target NFsim) if (ARCH_64bit) set(exe_target ${exe_target}_x64) -endif (ARCH_64bit) +endif (ARCH_64bit) + +enable_testing() add_executable(${exe_target} ${SRC_FILES}) target_link_libraries(${exe_target} vcommons vcellmessaging) +if (MINGW) + set(test_nfsim_exe ${CMAKE_BINARY_DIR}/bin/${exe_target}.exe) + set(python_cmd py) +else (MINGW) + set(test_nfsim_exe ${CMAKE_BINARY_DIR}/bin/${exe_target}) + set(python_cmd python3) +endif (MINGW) +set(test_dir ${CMAKE_CURRENT_SOURCE_DIR}/tests/smoke) + +# smoke test as a python script +add_test(NAME ${exe_target}_smoke_python COMMAND ${python_cmd} ${test_dir}/smoke.py ${test_nfsim_exe} WORKING_DIRECTORY ${test_dir}) + +# smoke test as a bash script +add_test(NAME ${exe_target}_smoke_bash COMMAND bash -c "${test_dir}/smoke.sh ${test_nfsim_exe}" WORKING_DIRECTORY ${test_dir}) + install(TARGETS ${exe_target} RUNTIME DESTINATION ${OPTION_EXE_DIRECTORY}) diff --git a/NFsim_v1.11/tests/smoke/SimID_273069657_0_.gdat.expected b/NFsim_v1.11/tests/smoke/SimID_273069657_0_.gdat.expected new file mode 100644 index 000000000..1d3c606d5 --- /dev/null +++ b/NFsim_v1.11/tests/smoke/SimID_273069657_0_.gdat.expected @@ -0,0 +1,12 @@ +# time O0_MT0_tot_Count O0_MT1_tot_Count O0_MT2_tot_Count O0_MT3_tot_Count O0_MT4_tot_Count s0_Count s1_Count s2_Count s3_Count s4_Count + 0.00000000e+00 6.02000000e+02 9.03000000e+02 1.20400000e+03 0.00000000e+00 0.00000000e+00 6.02000000e+02 9.03000000e+02 1.20400000e+03 0.00000000e+00 0.00000000e+00 + 1.00000000e-01 5.37000000e+02 9.68000000e+02 1.13900000e+03 0.00000000e+00 0.00000000e+00 5.37000000e+02 9.68000000e+02 1.13900000e+03 0.00000000e+00 0.00000000e+00 + 2.00000000e-01 5.24000000e+02 9.81000000e+02 1.12600000e+03 0.00000000e+00 0.00000000e+00 5.24000000e+02 9.81000000e+02 1.12600000e+03 0.00000000e+00 0.00000000e+00 + 3.00000000e-01 5.23000000e+02 9.82000000e+02 1.12500000e+03 0.00000000e+00 0.00000000e+00 5.23000000e+02 9.82000000e+02 1.12500000e+03 0.00000000e+00 0.00000000e+00 + 4.00000000e-01 5.30000000e+02 9.75000000e+02 1.13200000e+03 0.00000000e+00 0.00000000e+00 5.30000000e+02 9.75000000e+02 1.13200000e+03 0.00000000e+00 0.00000000e+00 + 5.00000000e-01 5.23000000e+02 9.82000000e+02 1.12500000e+03 0.00000000e+00 0.00000000e+00 5.23000000e+02 9.82000000e+02 1.12500000e+03 0.00000000e+00 0.00000000e+00 + 6.00000000e-01 5.21000000e+02 9.84000000e+02 1.12300000e+03 0.00000000e+00 0.00000000e+00 5.21000000e+02 9.84000000e+02 1.12300000e+03 0.00000000e+00 0.00000000e+00 + 7.00000000e-01 5.20000000e+02 9.85000000e+02 1.12200000e+03 0.00000000e+00 0.00000000e+00 5.20000000e+02 9.85000000e+02 1.12200000e+03 0.00000000e+00 0.00000000e+00 + 8.00000000e-01 5.13000000e+02 9.92000000e+02 1.11500000e+03 0.00000000e+00 0.00000000e+00 5.13000000e+02 9.92000000e+02 1.11500000e+03 0.00000000e+00 0.00000000e+00 + 9.00000000e-01 5.13000000e+02 9.92000000e+02 1.11500000e+03 0.00000000e+00 0.00000000e+00 5.13000000e+02 9.92000000e+02 1.11500000e+03 0.00000000e+00 0.00000000e+00 + 1.00000000e+00 5.33000000e+02 9.72000000e+02 1.13500000e+03 0.00000000e+00 0.00000000e+00 5.33000000e+02 9.72000000e+02 1.13500000e+03 0.00000000e+00 0.00000000e+00 diff --git a/NFsim_v1.11/tests/smoke/SimID_273069657_0_.nfsimInput b/NFsim_v1.11/tests/smoke/SimID_273069657_0_.nfsimInput new file mode 100644 index 000000000..310f94fda --- /dev/null +++ b/NFsim_v1.11/tests/smoke/SimID_273069657_0_.nfsimInput @@ -0,0 +1 @@ +((O0_MT0_tot_Count * UnitFactor_uM_um3_molecules_neg_1) / Size_c0)((O0_MT1_tot_Count * UnitFactor_uM_um3_molecules_neg_1) / Size_c0)((O0_MT2_tot_Count * UnitFactor_uM_um3_molecules_neg_1) / Size_c0)((O0_MT3_tot_Count * UnitFactor_uM_um3_molecules_neg_1) / Size_c0)((O0_MT4_tot_Count * UnitFactor_uM_um3_molecules_neg_1) / Size_c0)((UnitFactor_uM_um3_molecules_neg_1 * s0_Count) / Size_c0)((UnitFactor_uM_um3_molecules_neg_1 * s1_Count) / Size_c0)((UnitFactor_uM_um3_molecules_neg_1 * s2_Count) / Size_c0)((UnitFactor_uM_um3_molecules_neg_1 * s3_Count) / Size_c0)((UnitFactor_uM_um3_molecules_neg_1 * s4_Count) / Size_c0)k8s-wn-01.cam.uchc.edu:30163clientUserdummyworkerEventserviceControlschaff2730696570 \ No newline at end of file diff --git a/NFsim_v1.11/tests/smoke/SimID_273069657_0_.species.expected b/NFsim_v1.11/tests/smoke/SimID_273069657_0_.species.expected new file mode 100644 index 000000000..c80fe267d --- /dev/null +++ b/NFsim_v1.11/tests/smoke/SimID_273069657_0_.species.expected @@ -0,0 +1,5 @@ +# nfsim generated species list for system: 'nameless' +# warning! this feature is not yet fully tested! +MT0(AAA~c0,AAB~0) 532 +MT1(AAA~c0,AAB~0) 973 +MT2(AAA~c0,AAB~0) 1134 diff --git a/NFsim_v1.11/tests/smoke/smoke.py b/NFsim_v1.11/tests/smoke/smoke.py new file mode 100755 index 000000000..ac9570926 --- /dev/null +++ b/NFsim_v1.11/tests/smoke/smoke.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import os +import posixpath +import subprocess +import sys + +# get the directory of this script +test_dir = os.path.dirname(os.path.realpath(__file__)) +# in the path replace \ with /, D:\ with /d/ +test_dir = test_dir.replace("\\", "/") +# tell os.path.join to use / as the path separator +os.path.sep = "/" +exe = sys.argv[1] + +print(f"test_dir: {test_dir}") +print(f"exe: {exe}") + +input_file = posixpath.join(test_dir, "SimID_273069657_0_.nfsimInput") +output = posixpath.join(test_dir, "SimID_273069657_0_.gdat") +expected_output = posixpath.join(test_dir, "SimID_273069657_0_.gdat.expected") +species = posixpath.join(test_dir, "SimID_273069657_0_.species") +expected_species = posixpath.join(test_dir, "SimID_273069657_0_.species.expected") + +if not posixpath.exists(exe): + print(f"NFsim executable {exe} not found. Exiting...") + sys.exit(1) + +if not posixpath.exists(input_file): + print(f"Input file {input_file} not found. Exiting...") + sys.exit(1) + +if not posixpath.exists(expected_output): + print(f"Expected output file {expected_output} not found. Exiting...") + sys.exit(1) + +if not posixpath.exists(expected_species): + print(f"Expected species file {expected_species} not found. Exiting...") + sys.exit(1) + +command = [exe, "-seed", "505790288", "-vcell", "-xml", input_file, "-o", output, "-sim", "1.0", "-ss", species, "-oStep", "20", "-notf", "-utl", "1000", "-cb", "-pcmatch", "-tid", "0"] +print(" ".join(command)) + +try: + subprocess.check_call(command) +except subprocess.CalledProcessError: + print("NFsim failed to run. Exiting...") + sys.exit(1) + +# verify that the output files exist +if not os.path.isfile(output): + print(f"Output file {output} not found. Exiting...") + sys.exit(1) + +if not os.path.isfile(species): + print(f"Species file {species} not found. Exiting...") + sys.exit(1) + +# verify that the output files match the expected output files +if open(output).read() != open(expected_output).read(): + print(f"Output file {output} does not match expected output {expected_output}. Exiting...") + sys.exit(1) + +if open(species).read() != open(expected_species).read(): + print(f"Species file {species} does not match expected species {expected_species}. Exiting...") + sys.exit(1) + +print("NFsim solver completed and solution matched expected output. Exiting...") +sys.exit(0) \ No newline at end of file diff --git a/NFsim_v1.11/tests/smoke/smoke.sh b/NFsim_v1.11/tests/smoke/smoke.sh new file mode 100755 index 000000000..d58eb0506 --- /dev/null +++ b/NFsim_v1.11/tests/smoke/smoke.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e +EXE=$1 + +echo "Running NFsim solver with test2.sh $EXE" + +# get the directory of this script +TEST_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +INPUT="${TEST_DIR}/SimID_273069657_0_.nfsimInput" +OUTPUT="${TEST_DIR}/SimID_273069657_0_.gdat" +EXPECTED_OUTPUT="${TEST_DIR}/SimID_273069657_0_.gdat.expected" +SPECIES="${TEST_DIR}/SimID_273069657_0_.species" +EXPECTED_SPECIES="${TEST_DIR}/SimID_273069657_0_.species.expected" + +if [ ! -f $EXE ]; then + echo "NFsim executable $EXE not found. Exiting..." + exit 1 +fi +if [ ! -f $INPUT ]; then + echo "Input file $INPUT not found. Exiting..." + exit 1 +fi +if [ ! -f $EXPECTED_OUTPUT ]; then + echo "Expected output file $EXPECTED_OUTPUT not found. Exiting..." + exit 1 +fi +if [ ! -f $EXPECTED_SPECIES ]; then + echo "Expected species file $EXPECTED_SPECIES not found. Exiting..." + exit 1 +fi + + +command="$EXE -seed 505790288 -vcell -xml $INPUT -o $OUTPUT -sim 1.0 -ss $SPECIES -oStep 20 -notf -utl 1000 -cb -pcmatch -tid 0" +echo $command +if ! $command; then + echo "NFsim failed to run. Exiting..." + exit 1 +fi + +# verify that the output files exist +if [ ! -f $OUTPUT ]; then + echo "Output file $OUTPUT not found. Exiting..." + exit 1 +fi +if [ ! -f $SPECIES ]; then + echo "Species file $SPECIES not found. Exiting..." + exit 1 +fi + +# verify that the output files match the expected output files +if ! diff $OUTPUT $EXPECTED_OUTPUT; then + echo "Output file $OUTPUT does not match expected output $EXPECTED_OUTPUT. Exiting..." + exit 1 +fi +if ! diff $SPECIES $EXPECTED_SPECIES; then + echo "Species file $SPECIES does not match expected species $EXPECTED_SPECIES. Exiting..." + exit 1 +fi + +echo "NFsim solver complted and solution matched expected output. Exiting..." +exit 0 diff --git a/VCellMessaging/include/VCELL/SimulationMessaging.h b/VCellMessaging/include/VCELL/SimulationMessaging.h index 20979cced..3180e2118 100644 --- a/VCellMessaging/include/VCELL/SimulationMessaging.h +++ b/VCellMessaging/include/VCELL/SimulationMessaging.h @@ -1,7 +1,7 @@ #ifndef _SIMULATIONMESSAGING_H_ #define _SIMULATIONMESSAGING_H_ -#include +#include #ifdef USE_MESSAGING #include #include @@ -145,7 +145,7 @@ class SimulationMessaging private: SimulationMessaging(); static SimulationMessaging *m_inst; - std::deque events; + std::vector events; int workerEventOutputMode; void sendStatus(); diff --git a/VCellMessaging/src/SimulationMessaging.cpp b/VCellMessaging/src/SimulationMessaging.cpp index 4747cb85d..5537ff495 100644 --- a/VCellMessaging/src/SimulationMessaging.cpp +++ b/VCellMessaging/src/SimulationMessaging.cpp @@ -155,7 +155,10 @@ void SimulationMessaging::sendStatus() { WorkerEventLocker locker(*this); if (events.size( ) > 0 ) { workerEvent = events.front( ); - events.pop_front( ); + if (!events.empty()) + { + events.erase(events.begin()); + } } else { return;