hdf5 use subfiling if available

res/cmake/dep/samrai.cmake

@@ -28,8 +28,9 @@ else()
     set(BUILD_SHARED_LIBS ON CACHE BOOL "Make shared libs" FORCE) # default as of 25/JAN/2026 is static libs
   endif(SAMRAI_BUILD_SHARED_LIBS)
 
-  option(ENABLE_TESTS "Enable Samrai Test" OFF ) # disable SAMRAI Test so that we can use the googletest pulled after
-  option(ENABLE_SAMRAI_TESTS "Enable Samrai Test" OFF ) # disable SAMRAI Test so that we can use the googletest pulled after
+  option(ENABLE_TESTS "Enable Samrai Test" OFF) # disable SAMRAI Test so that we can use the googletest pulled after
+  option(ENABLE_SAMRAI_TESTS "Enable Samrai Test" OFF) # disable SAMRAI Test so that we can use the googletest pulled after
+  option(ENABLE_TIMERS "NO TIMERS!" OFF)
 
   add_subdirectory(${SAMRAI_SRCDIR})
   unset(CMAKE_RUNTIME_OUTPUT_DIRECTORY CACHE) # undoes what samrai does, so ctest can continue to work

src/amr/samrai.cpp

@@ -1,12 +1,28 @@
 #include "samrai.hpp"
 
+#include "H5pubconf.h" // may define H5_HAVE_SUBFILING_VFD
+
+#if !defined(H5_HAVE_SUBFILING_VFD)
+#define H5_HAVE_SUBFILING_VFD 0
+#endif
 
 namespace PHARE
 {
 
 SamraiLifeCycle::SamraiLifeCycle(int argc, char** argv)
 {
+#if H5_HAVE_SUBFILING_VFD
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
+    if (provided < MPI_THREAD_MULTIPLE)
+        throw std::runtime_error(
+            "MPI_THREAD_MULTIPLE required for HDF5 subfiling but not provided");
+    SAMRAI::tbox::SAMRAI_MPI::init(MPI_COMM_WORLD);
+
+#else  // normal way
     SAMRAI::tbox::SAMRAI_MPI::init(&argc, &argv);
+#endif // H5_HAVE_SUBFILING_VFD
+
     SAMRAI::tbox::SAMRAIManager::initialize();
     SAMRAI::tbox::SAMRAIManager::startup();
     // uncomment next line for debugging samrai issues
@@ -27,7 +43,11 @@ SamraiLifeCycle::~SamraiLifeCycle()
     PHARE_WITH_PHLOP(phlop::ScopeTimerMan::reset());
     SAMRAI::tbox::SAMRAIManager::shutdown();
     SAMRAI::tbox::SAMRAIManager::finalize();
+
     SAMRAI::tbox::SAMRAI_MPI::finalize();
+#if H5_HAVE_SUBFILING_VFD
+    MPI_Finalize();
+#endif // H5_HAVE_SUBFILING_VFD
 }
 
 void SamraiLifeCycle::reset()

src/hdf5/detail/h5/h5_file.hpp

@@ -7,17 +7,32 @@
 #include "core/utilities/mpi_utils.hpp"
 #include "core/utilities/meta/meta_utilities.hpp"
 
+#include "H5pubconf.h" // may define H5_HAVE_SUBFILING_VFD
+
+#if !defined(H5_HAVE_SUBFILING_VFD)
+#define H5_HAVE_SUBFILING_VFD 0
+#endif
 
 #include "highfive/H5File.hpp"
 #include "highfive/H5Easy.hpp"
 
-
+#if H5_HAVE_SUBFILING_VFD
+#include "H5FDsubfiling.h"
+#include "H5FDioc.h"
+#endif
 
 namespace PHARE::hdf5::h5::detail
 {
-
+int inline determine_subfile_count()
+{
+#if H5_HAVE_SUBFILING_VFD
+    return core::get_env_as("PHARE_H5_SUB_FILES", int{20});
+#endif
+    return 0;
+}
 // https://support.hdfgroup.org/documentation/hdf5/latest/hdf5_chunking.html
 static inline auto const CHUNK_SIZE = core::get_env_as("PHARE_H5_CHUNK_SIZE", std::size_t{1024});
+static inline auto const SUB_FILES  = determine_subfile_count();
 
 } // namespace PHARE::hdf5::h5::detail
 
@@ -38,31 +53,83 @@ NO_DISCARD auto vector_for_dim()
         return std::vector<std::vector<std::vector<Data>>>();
 }
 
+
+struct SubFiler
+{
+    friend HighFive::FileAccessProps;
+
+public:
+    SubFiler([[maybe_unused]] hid_t const plist_id,
+             [[maybe_unused]] int const subfiles = detail::SUB_FILES)
+    {
+#if H5_HAVE_SUBFILING_VFD
+        if (H5Pget_fapl_subfiling(plist_id, &vfd_config) < 0)
+            throw std::runtime_error("Failed to get subfiling config!");
+        if (H5Pget_fapl_ioc(plist_id, &ioc_config) < 0)
+            throw std::runtime_error("Failed to get IOC config!");
+
+        // review https://github.com/HDFGroup/hdf5-examples/blob/master/C/H5PAR/ph5_subfiling.c#L217
+        vfd_config.shared_cfg.stripe_count = subfiles;
+
+        if (H5Pset_fapl_ioc(vfd_config.ioc_fapl_id, &ioc_config) < 0)
+            throw std::runtime_error("Subfiling ioc failed!");
+#else
+        throw std::runtime_error("Subfiling not available!");
+#endif
+    }
+
+    ~SubFiler()
+    {
+#if H5_HAVE_SUBFILING_VFD
+        H5Pclose(vfd_config.ioc_fapl_id);
+#endif
+    }
+
+private:
+    void apply(hid_t const plist_id) const
+    {
+#if H5_HAVE_SUBFILING_VFD
+        if (H5Pset_fapl_subfiling(plist_id, &vfd_config) < 0)
+            throw std::runtime_error("Subfiling failed!");
+#endif
+    }
+
+private:
+#if H5_HAVE_SUBFILING_VFD
+    H5FD_subfiling_config_t vfd_config;
+    H5FD_ioc_config_t ioc_config;
+#endif
+};
+
 class HighFiveFile
 {
 public:
-    template<typename FileAccessProps>
-    static auto createHighFiveFile(std::string const path, FileOp flags, bool para,
-                                   FileAccessProps& fapl)
+    static auto createHighFiveFile(std::string const path, FileOp flags, bool para)
     {
+        auto fapl = HighFive::FileAccessProps::Empty();
         if (para)
         {
 #if defined(H5_HAVE_PARALLEL)
             fapl.add(HighFive::MPIOFileAccess{MPI_COMM_WORLD, MPI_INFO_NULL});
-#else
+
+#if H5_HAVE_SUBFILING_VFD
+            if (detail::SUB_FILES > 0) // run time disabled
+                fapl.add(SubFiler{fapl.getId()});
+
+#endif // H5_HAVE_SUBFILING_VFD
+
+#else  // NOT H5_HAVE_PARALLEL
             std::cout << "WARNING: PARALLEL HDF5 not available" << std::endl;
             if (core::mpi::size() > 1)
-            {
                 throw std::runtime_error("HDF5 NOT PARALLEL!");
-            }
-#endif
+#endif // defined(H5_HAVE_PARALLEL)
         }
+
         return HiFile{path, flags, fapl};
     }
 
     HighFiveFile(std::string const path, FileOp flags = HiFile::ReadWrite, bool para = true)
-        : fapl_{}
-        , h5file_{createHighFiveFile(path, flags, para, fapl_)}
+        : h5file_{createHighFiveFile(path, flags, para)}
     {
     }
 
@@ -287,7 +354,6 @@ class HighFiveFile
     HighFiveFile& operator=(HighFiveFile&&)      = delete;
 
 private:
-    HighFive::FileAccessProps fapl_;
     HiFile h5file_;
 
 

tools/config/config.py

@@ -25,6 +25,7 @@ class SystemSettings:
     cxx_compiler_version: str
     hdf5_version: str
     hdf5_is_parallel: str
+    hdf5_has_subfiling: str
     mpi_version: str
     python_binary: str
     python_version: str
@@ -34,6 +35,7 @@ class SystemSettings:
 
 
 SYSTEM_CPP_ = """
+#include <string>
 #include <string_view>
 #include <unordered_map>
 
@@ -46,6 +48,7 @@ class SystemSettings:
     constexpr static std::string_view CXX_COMPILER_VERSION = R"({})";
     constexpr static std::string_view HDF5_VERSION = R"({})";
     constexpr static std::string_view HDF5_IS_PARALLEL = R"({})";
+    constexpr static std::string_view HDF5_HAS_SUBFILING = R"({})";
     constexpr static std::string_view _MPI_VERSION_ = R"({})";
     constexpr static std::string_view PYTHON_BINARY = R"({})";
     constexpr static std::string_view PYTHON_VERSION = R"({})";
@@ -55,14 +58,15 @@ class SystemSettings:
 
 }};
 
-std::unordered_map<std::string, std::string> build_config(){{
+std::unordered_map<std::string, std::string> inline build_config(){{
   return {{
       {{"CMAKE_BINARY", std::string{{SystemConfig::CMAKE_BINARY}}}},
       {{"CMAKE_VERSION", std::string{{SystemConfig::CMAKE_VERSION}}}},
       {{"CXX_COMPILER", std::string{{SystemConfig::CXX_COMPILER}}}},
       {{"CXX_COMPILER_VERSION", std::string{{SystemConfig::CXX_COMPILER_VERSION}}}},
       {{"HDF5_VERSION", std::string{{SystemConfig::HDF5_VERSION}}}},
       {{"HDF5_IS_PARALLEL", std::string{{SystemConfig::HDF5_IS_PARALLEL}}}},
+      {{"HDF5_HAS_SUBFILING", std::string{{SystemConfig::HDF5_HAS_SUBFILING}}}},
       {{"MPI_VERSION", std::string{{SystemConfig::_MPI_VERSION_}}}},
       {{"PYTHON_BINARY", std::string{{SystemConfig::PYTHON_BINARY}}}},
       {{"PYTHON_VERSION", std::string{{SystemConfig::PYTHON_VERSION}}}},
@@ -197,6 +201,9 @@ def gen_system_file():
         hdf5_is_parallel=file_string_or(
             "PHARE_HDF5_is_parallel.txt", "HDF5 is parallel check failed"
         ),
+        hdf5_has_subfiling=file_string_or(
+            "PHARE_HDF5_has_subfiling.txt", "HDF5 has subfiling check failed"
+        ),
         mpi_version=file_string_or(
             "PHARE_MPI_Get_library_version.txt", "MPI version failed"
         ),

tools/config/main.cpp

@@ -9,10 +9,19 @@
 
 #if __has_include("hdf5.h")
 #include "hdf5.h"
+
+#include "H5pubconf.h" // may define H5_HAVE_SUBFILING_VFD
+
+#if !defined(H5_HAVE_SUBFILING_VFD)
+#define H5_HAVE_SUBFILING_VFD 0
+#endif
+
 constexpr static std::string_view hdf5_version = H5_VERSION;
+constexpr static bool hdf5_subfiling           = H5_HAVE_SUBFILING_VFD;
 #else
-constexpr static std::string_view hdf5_version     = "HDF5 was not found!";
-constexpr static std::string_view H5_HAVE_PARALLEL = false;
+constexpr static std::string_view hdf5_version = "HDF5 was not found!";
+constexpr static bool H5_HAVE_PARALLEL         = false;
+constexpr static bool hdf5_subfiling           = false;
 #endif
 constexpr std::string_view hdf5_is_parallel()
 {
@@ -40,10 +49,16 @@ void write_hdf5_is_parallel()
     write_string_to_file(std::string{hdf5_is_parallel()}, "PHARE_HDF5_is_parallel.txt");
 }
 
+void write_hdf5_has_subfiling()
+{
+    write_string_to_file(std::string{hdf5_subfiling}, "PHARE_HDF5_has_subfiling.txt");
+}
+
 void write_hdf5_info()
 {
     write_hdf5_version();
     write_hdf5_is_parallel();
+    write_hdf5_has_subfiling();
 }
 
 void config_mpi()
@@ -72,7 +87,7 @@ int main(int argc, char** argv)
         config_mpi();
         write_hdf5_info();
     }
-    catch (const std::exception& e)
+    catch (std::exception const& e)
     {
         std::cerr << e.what() << std::endl;
         MPI_Abort(MPI_COMM_WORLD, 1);