diff --git a/res/cmake/dep/samrai.cmake b/res/cmake/dep/samrai.cmake index 0db7cb838..166afb03a 100644 --- a/res/cmake/dep/samrai.cmake +++ b/res/cmake/dep/samrai.cmake @@ -28,8 +28,9 @@ else() set(BUILD_SHARED_LIBS ON CACHE BOOL "Make shared libs" FORCE) # default as of 25/JAN/2026 is static libs endif(SAMRAI_BUILD_SHARED_LIBS) - option(ENABLE_TESTS "Enable Samrai Test" OFF ) # disable SAMRAI Test so that we can use the googletest pulled after - option(ENABLE_SAMRAI_TESTS "Enable Samrai Test" OFF ) # disable SAMRAI Test so that we can use the googletest pulled after + option(ENABLE_TESTS "Enable Samrai Test" OFF) # disable SAMRAI Test so that we can use the googletest pulled after + option(ENABLE_SAMRAI_TESTS "Enable Samrai Test" OFF) # disable SAMRAI Test so that we can use the googletest pulled after + option(ENABLE_TIMERS "NO TIMERS!" OFF) add_subdirectory(${SAMRAI_SRCDIR}) unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY CACHE) # undoes what samrai does, so ctest can continue to work diff --git a/src/amr/samrai.cpp b/src/amr/samrai.cpp index c78b52441..efb6a72d0 100644 --- a/src/amr/samrai.cpp +++ b/src/amr/samrai.cpp @@ -1,12 +1,28 @@ #include "samrai.hpp" +#include "H5pubconf.h" // may define H5_HAVE_SUBFILING_VFD + +#if !defined(H5_HAVE_SUBFILING_VFD) +#define H5_HAVE_SUBFILING_VFD 0 +#endif namespace PHARE { SamraiLifeCycle::SamraiLifeCycle(int argc, char** argv) { +#if H5_HAVE_SUBFILING_VFD + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + if (provided < MPI_THREAD_MULTIPLE) + throw std::runtime_error( + "MPI_THREAD_MULTIPLE required for HDF5 subfiling but not provided"); + SAMRAI::tbox::SAMRAI_MPI::init(MPI_COMM_WORLD); + +#else // normal way SAMRAI::tbox::SAMRAI_MPI::init(&argc, &argv); +#endif // H5_HAVE_SUBFILING_VFD + SAMRAI::tbox::SAMRAIManager::initialize(); SAMRAI::tbox::SAMRAIManager::startup(); // uncomment next line for debugging samrai issues @@ -27,7 +43,11 @@ SamraiLifeCycle::~SamraiLifeCycle() PHARE_WITH_PHLOP(phlop::ScopeTimerMan::reset()); SAMRAI::tbox::SAMRAIManager::shutdown(); SAMRAI::tbox::SAMRAIManager::finalize(); + SAMRAI::tbox::SAMRAI_MPI::finalize(); +#if H5_HAVE_SUBFILING_VFD + MPI_Finalize(); +#endif // H5_HAVE_SUBFILING_VFD } void SamraiLifeCycle::reset() diff --git a/src/hdf5/detail/h5/h5_file.hpp b/src/hdf5/detail/h5/h5_file.hpp index f8208d75c..aeed052eb 100644 --- a/src/hdf5/detail/h5/h5_file.hpp +++ b/src/hdf5/detail/h5/h5_file.hpp @@ -7,17 +7,32 @@ #include "core/utilities/mpi_utils.hpp" #include "core/utilities/meta/meta_utilities.hpp" +#include "H5pubconf.h" // may define H5_HAVE_SUBFILING_VFD + +#if !defined(H5_HAVE_SUBFILING_VFD) +#define H5_HAVE_SUBFILING_VFD 0 +#endif #include "highfive/H5File.hpp" #include "highfive/H5Easy.hpp" - +#if H5_HAVE_SUBFILING_VFD +#include "H5FDsubfiling.h" +#include "H5FDioc.h" +#endif namespace PHARE::hdf5::h5::detail { - +int inline determine_subfile_count() +{ +#if H5_HAVE_SUBFILING_VFD + return core::get_env_as("PHARE_H5_SUB_FILES", int{20}); +#endif + return 0; +} // https://support.hdfgroup.org/documentation/hdf5/latest/hdf5_chunking.html static inline auto const CHUNK_SIZE = core::get_env_as("PHARE_H5_CHUNK_SIZE", std::size_t{1024}); +static inline auto const SUB_FILES = determine_subfile_count(); } // namespace PHARE::hdf5::h5::detail @@ -38,31 +53,83 @@ NO_DISCARD auto vector_for_dim() return std::vector>>(); } + +struct SubFiler +{ + friend HighFive::FileAccessProps; + +public: + SubFiler([[maybe_unused]] hid_t const plist_id, + [[maybe_unused]] int const subfiles = detail::SUB_FILES) + { +#if H5_HAVE_SUBFILING_VFD + if (H5Pget_fapl_subfiling(plist_id, &vfd_config) < 0) + throw std::runtime_error("Failed to get subfiling config!"); + if (H5Pget_fapl_ioc(plist_id, &ioc_config) < 0) + throw std::runtime_error("Failed to get IOC config!"); + + // review https://github.com/HDFGroup/hdf5-examples/blob/master/C/H5PAR/ph5_subfiling.c#L217 + vfd_config.shared_cfg.stripe_count = subfiles; + + if (H5Pset_fapl_ioc(vfd_config.ioc_fapl_id, &ioc_config) < 0) + throw std::runtime_error("Subfiling ioc failed!"); +#else + throw std::runtime_error("Subfiling not available!"); +#endif + } + + ~SubFiler() + { +#if H5_HAVE_SUBFILING_VFD + H5Pclose(vfd_config.ioc_fapl_id); +#endif + } + +private: + void apply(hid_t const plist_id) const + { +#if H5_HAVE_SUBFILING_VFD + if (H5Pset_fapl_subfiling(plist_id, &vfd_config) < 0) + throw std::runtime_error("Subfiling failed!"); +#endif + } + +private: +#if H5_HAVE_SUBFILING_VFD + H5FD_subfiling_config_t vfd_config; + H5FD_ioc_config_t ioc_config; +#endif +}; + class HighFiveFile { public: - template - static auto createHighFiveFile(std::string const path, FileOp flags, bool para, - FileAccessProps& fapl) + static auto createHighFiveFile(std::string const path, FileOp flags, bool para) { + auto fapl = HighFive::FileAccessProps::Empty(); if (para) { #if defined(H5_HAVE_PARALLEL) fapl.add(HighFive::MPIOFileAccess{MPI_COMM_WORLD, MPI_INFO_NULL}); -#else + +#if H5_HAVE_SUBFILING_VFD + if (detail::SUB_FILES > 0) // run time disabled + fapl.add(SubFiler{fapl.getId()}); + +#endif // H5_HAVE_SUBFILING_VFD + +#else // NOT H5_HAVE_PARALLEL std::cout << "WARNING: PARALLEL HDF5 not available" << std::endl; if (core::mpi::size() > 1) - { throw std::runtime_error("HDF5 NOT PARALLEL!"); - } -#endif +#endif // defined(H5_HAVE_PARALLEL) } + return HiFile{path, flags, fapl}; } HighFiveFile(std::string const path, FileOp flags = HiFile::ReadWrite, bool para = true) - : fapl_{} - , h5file_{createHighFiveFile(path, flags, para, fapl_)} + : h5file_{createHighFiveFile(path, flags, para)} { } @@ -287,7 +354,6 @@ class HighFiveFile HighFiveFile& operator=(HighFiveFile&&) = delete; private: - HighFive::FileAccessProps fapl_; HiFile h5file_; diff --git a/tools/config/config.py b/tools/config/config.py index bcf381315..1c29b5f84 100644 --- a/tools/config/config.py +++ b/tools/config/config.py @@ -25,6 +25,7 @@ class SystemSettings: cxx_compiler_version: str hdf5_version: str hdf5_is_parallel: str + hdf5_has_subfiling: str mpi_version: str python_binary: str python_version: str @@ -34,6 +35,7 @@ class SystemSettings: SYSTEM_CPP_ = """ +#include #include #include @@ -46,6 +48,7 @@ class SystemSettings: constexpr static std::string_view CXX_COMPILER_VERSION = R"({})"; constexpr static std::string_view HDF5_VERSION = R"({})"; constexpr static std::string_view HDF5_IS_PARALLEL = R"({})"; + constexpr static std::string_view HDF5_HAS_SUBFILING = R"({})"; constexpr static std::string_view _MPI_VERSION_ = R"({})"; constexpr static std::string_view PYTHON_BINARY = R"({})"; constexpr static std::string_view PYTHON_VERSION = R"({})"; @@ -55,7 +58,7 @@ class SystemSettings: }}; -std::unordered_map build_config(){{ +std::unordered_map inline build_config(){{ return {{ {{"CMAKE_BINARY", std::string{{SystemConfig::CMAKE_BINARY}}}}, {{"CMAKE_VERSION", std::string{{SystemConfig::CMAKE_VERSION}}}}, @@ -63,6 +66,7 @@ class SystemSettings: {{"CXX_COMPILER_VERSION", std::string{{SystemConfig::CXX_COMPILER_VERSION}}}}, {{"HDF5_VERSION", std::string{{SystemConfig::HDF5_VERSION}}}}, {{"HDF5_IS_PARALLEL", std::string{{SystemConfig::HDF5_IS_PARALLEL}}}}, + {{"HDF5_HAS_SUBFILING", std::string{{SystemConfig::HDF5_HAS_SUBFILING}}}}, {{"MPI_VERSION", std::string{{SystemConfig::_MPI_VERSION_}}}}, {{"PYTHON_BINARY", std::string{{SystemConfig::PYTHON_BINARY}}}}, {{"PYTHON_VERSION", std::string{{SystemConfig::PYTHON_VERSION}}}}, @@ -197,6 +201,9 @@ def gen_system_file(): hdf5_is_parallel=file_string_or( "PHARE_HDF5_is_parallel.txt", "HDF5 is parallel check failed" ), + hdf5_has_subfiling=file_string_or( + "PHARE_HDF5_has_subfiling.txt", "HDF5 has subfiling check failed" + ), mpi_version=file_string_or( "PHARE_MPI_Get_library_version.txt", "MPI version failed" ), diff --git a/tools/config/main.cpp b/tools/config/main.cpp index 506374d84..d7621e037 100644 --- a/tools/config/main.cpp +++ b/tools/config/main.cpp @@ -9,10 +9,19 @@ #if __has_include("hdf5.h") #include "hdf5.h" + +#include "H5pubconf.h" // may define H5_HAVE_SUBFILING_VFD + +#if !defined(H5_HAVE_SUBFILING_VFD) +#define H5_HAVE_SUBFILING_VFD 0 +#endif + constexpr static std::string_view hdf5_version = H5_VERSION; +constexpr static bool hdf5_subfiling = H5_HAVE_SUBFILING_VFD; #else -constexpr static std::string_view hdf5_version = "HDF5 was not found!"; -constexpr static std::string_view H5_HAVE_PARALLEL = false; +constexpr static std::string_view hdf5_version = "HDF5 was not found!"; +constexpr static bool H5_HAVE_PARALLEL = false; +constexpr static bool hdf5_subfiling = false; #endif constexpr std::string_view hdf5_is_parallel() { @@ -40,10 +49,16 @@ void write_hdf5_is_parallel() write_string_to_file(std::string{hdf5_is_parallel()}, "PHARE_HDF5_is_parallel.txt"); } +void write_hdf5_has_subfiling() +{ + write_string_to_file(std::string{hdf5_subfiling}, "PHARE_HDF5_has_subfiling.txt"); +} + void write_hdf5_info() { write_hdf5_version(); write_hdf5_is_parallel(); + write_hdf5_has_subfiling(); } void config_mpi() @@ -72,7 +87,7 @@ int main(int argc, char** argv) config_mpi(); write_hdf5_info(); } - catch (const std::exception& e) + catch (std::exception const& e) { std::cerr << e.what() << std::endl; MPI_Abort(MPI_COMM_WORLD, 1);