ADIOS2: Require v2.9 as minimum (#1711)

franzpoeschel · web-flow · commit 8cace7e03ece · 2025-02-18T10:02:47.000-08:00
* CMake: Require ADIOS2 v2.9 * Remove openPMD_HAS_ADIOS_2_8 macro * Remove openPMD_HAS_ADIOS_2_9 macro * Fixes * Update ADIOS2 in the CI * Only ADIOS2 v2.9 on Spack? * Revert "Only ADIOS2 v2.9 on Spack?" This reverts commit 1740136. * Update spack * Only specify major ADIOS2 versions * Fixes: Mgard, /usr/bin/cmake * Try ADIOS2 v2.10 for failing runs * Take mgard out of clang7_nopy_ompi_h5_ad2_libcpp run * Take blosc2 out of failing runs * Update documentation * Reimplement "modifiable_attributes" parameter * Revert "Reimplement "modifiable_attributes" parameter" This reverts commit f3efc99.
diff --git a/.github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml
@@ -13,7 +13,7 @@ spack:
     hdf5:
       variants: ~mpi
     adios2:
-      variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard
     cmake:
       externals:
       - spec: cmake@3.22.1
diff --git a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml
@@ -12,7 +12,7 @@ spack:
 
   packages:
     adios2:
-      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~blosc2
     cmake:
       externals:
       - spec: cmake@3.23.0
diff --git a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml
@@ -6,13 +6,13 @@
 #
 spack:
   specs:
-  - adios2
+  - adios2@2.10
   - hdf5
   - openmpi
 
   packages:
     adios2:
-      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~mgard ~blosc2
     cmake:
       externals:
       - spec: cmake@3.23.0
diff --git a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml b/.github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml
@@ -6,13 +6,13 @@
 #
 spack:
   specs:
-  - adios2@2.7.1
+  - adios2@2.10
   - hdf5
   - mpich
 
   packages:
     adios2:
-      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2
+      variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 ~blosc2
     cmake:
       externals:
       - spec: cmake@3.23.0
diff --git a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml
@@ -6,7 +6,7 @@
 #
 spack:
   specs:
-  - adios2
+  - adios2@2.10
   - hdf5
   - openmpi
 
diff --git a/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml b/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml
@@ -6,7 +6,7 @@
 #
 spack:
   specs:
-  - adios2
+  - adios2@2.10
   - hdf5
   - openmpi
 
diff --git a/.github/workflows/dependencies/install_spack b/.github/workflows/dependencies/install_spack
@@ -3,7 +3,7 @@
 
 set -eu -o pipefail
 
-spack_ver="0.17.1"
+spack_ver="0.22.3"
 
 cd /opt
 if [[ -d spack && ! -f spack_${spack_ver} ]]
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -63,6 +63,7 @@ jobs:
         mpiexec --version
         perl --version
         python --version
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/)
         spack install
 
@@ -99,6 +100,7 @@ jobs:
     - name: Build
       env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/)
         spack install
 
@@ -178,6 +180,7 @@ jobs:
         mpiexec --version
         perl --version
         python --version
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/)
         spack install
 
@@ -229,6 +232,7 @@ jobs:
     - name: Build
       env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/)
         spack install
 
diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml
@@ -24,6 +24,7 @@ jobs:
     - name: Build
       env: {CC: clang, CXX: clang++}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/)
         spack install
 
@@ -54,6 +55,7 @@ jobs:
     - name: Build
       env: {CC: mpicc, CXX: mpic++, OMPI_CC: clang-10, OMPI_CXX: clang++-10, CXXFLAGS: -Werror, OPENPMD_HDF5_CHUNKS: none, OPENPMD_TEST_NFILES_MAX: 100}
       run: |
+        sudo ln -s "$(which cmake)" /usr/bin/cmake
         eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/)
         spack install
         SOURCEPATH="$(pwd)"
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -3,6 +3,17 @@
 Changelog
 =========
 
+0.16.0
+------
+
+Changes to "0.15.0"
+^^^^^^^^^^^^^^^^^^^
+
+Other
+"""""
+
+- ADIOS2: require version 2.9.0 #1711
+
 0.16.0
 ------
 **Date:** 2024-10-07
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -365,14 +365,14 @@ if(openPMD_HAVE_MPI)
     list(APPEND openPMD_REQUIRED_ADIOS2_COMPONENTS MPI)
 endif()
 if(openPMD_USE_ADIOS2 STREQUAL AUTO)
-    find_package(ADIOS2 2.7.0 CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
+    find_package(ADIOS2 2.9.0 CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
     if(ADIOS2_FOUND)
         set(openPMD_HAVE_ADIOS2 TRUE)
     else()
         set(openPMD_HAVE_ADIOS2 FALSE)
     endif()
 elseif(openPMD_USE_ADIOS2)
-    find_package(ADIOS2 2.7.0 REQUIRED CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
+    find_package(ADIOS2 2.9.0 REQUIRED CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS})
     set(openPMD_HAVE_ADIOS2 TRUE)
 else()
     set(openPMD_HAVE_ADIOS2 FALSE)
diff --git a/Dockerfile b/Dockerfile
@@ -56,7 +56,7 @@ RUN        curl -sLo c-blosc-1.15.0.tar.gz https://github.com/Blosc/c-blosc/arch
            && make \
            && make install
 
-RUN        curl -sLo adios2-2.7.1.tar.gz https://github.com/ornladios/ADIOS2/archive/v2.7.1.tar.gz \
+RUN        curl -sLo adios2-2.9.2.tar.gz https://github.com/ornladios/ADIOS2/archive/v2.9.2.tar.gz \
            && file adios2*.tar.gz \
            && tar -xzf adios2*.tar.gz \
            && rm adios2*.tar.gz \
diff --git a/NEWS.rst b/NEWS.rst
@@ -3,6 +3,11 @@
 Upgrade Guide
 =============
 
+0.17.0
+------
+
+ADIOS 2.9.0 is now the minimally supported version for ADIOS2 support.
+
 0.16.0
 ------
 
diff --git a/README.md b/README.md
@@ -108,7 +108,7 @@ Shipped internally (downloaded by CMake unless `openPMD_SUPERBUILD=OFF` is set):
 I/O backends:
 * [JSON](https://en.wikipedia.org/wiki/JSON)
 * [HDF5](https://support.hdfgroup.org/HDF5) 1.8.13+ (optional)
-* [ADIOS2](https://github.com/ornladios/ADIOS2) 2.7.0+ (optional)
+* [ADIOS2](https://github.com/ornladios/ADIOS2) 2.9.0+ (optional)
 
 while those can be built either with or without:
 * MPI 2.1+, e.g. OpenMPI 1.6.5+ or MPICH2
diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst
@@ -58,9 +58,6 @@ Steps
 ADIOS2 is optimized towards organizing the process of reading/writing data into IO steps.
 In order to activate steps, it is imperative to use the :ref:`Streaming API <usage-streaming>` (which can be used for either file-based or streaming-based workflows).
 
-ADIOS2 release 2.6.0 contained a bug (fixed in ADIOS 2.7.0, see `PR #2348 <https://github.com/ornladios/ADIOS2/pull/2348>`_) that disallows random-accessing steps in file-based engines.
-With this ADIOS2 release, files written with steps may only be read using the streaming API.
-
 Upon reading a file, the ADIOS2 backend will automatically recognize whether it has been written with or without steps, ignoring the JSON option mentioned above.
 Steps are mandatory for streaming-based engines and trying to switch them off will result in a runtime error.
 
diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst
@@ -27,7 +27,7 @@ Optional: I/O backends
 
 * `JSON <https://en.wikipedia.org/wiki/JSON>`_
 * `HDF5 <https://support.hdfgroup.org/HDF5>`_ 1.8.13+
-* `ADIOS2 <https://github.com/ornladios/ADIOS2>`_ 2.7.0+
+* `ADIOS2 <https://github.com/ornladios/ADIOS2>`_ 2.9.0+
 
 while those can be build either with or without:
 
diff --git a/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp
@@ -198,9 +198,7 @@ namespace detail
         case adios2::Mode::Write:
             return false;
         case adios2::Mode::Read:
-#if openPMD_HAS_ADIOS_2_8
         case adios2::Mode::ReadRandomAccess:
-#endif
             return true;
         case adios2::Mode::Undefined:
         case adios2::Mode::Sync:
@@ -217,9 +215,7 @@ namespace detail
         case adios2::Mode::Write:
             return true;
         case adios2::Mode::Read:
-#if openPMD_HAS_ADIOS_2_8
         case adios2::Mode::ReadRandomAccess:
-#endif
             return false;
         case adios2::Mode::Undefined:
         case adios2::Mode::Sync:
diff --git a/include/openPMD/IO/ADIOS/macros.hpp b/include/openPMD/IO/ADIOS/macros.hpp
@@ -6,19 +6,6 @@
 
 #include <adios2.h>
 
-/*
- * ADIOS2 v2.8 brings mode::ReadRandomAccess
- */
-#define openPMD_HAS_ADIOS_2_8                                                  \
-    (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 208)
-/*
- * ADIOS2 v2.9 brings modifiable attributes (technically already in v2.8, but
- * there are too many bugs, so we only support it beginning with v2.9).
- * Group table feature requires ADIOS2 v2.9.
- */
-#define openPMD_HAS_ADIOS_2_9                                                  \
-    (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 209)
-
 #define openPMD_HAS_ADIOS_2_10                                                 \
     (ADIOS2_VERSION_MAJOR * 100 + ADIOS2_VERSION_MINOR >= 210)
 
diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp
@@ -428,8 +428,6 @@ void ADIOS2File::configure_IO()
     // these properties are inferred from the opened dataset in read mode
     if (writeOnly(m_mode))
     {
-
-#if openPMD_HAS_ADIOS_2_9
         if (!m_impl->m_useGroupTable.has_value())
         {
             switch (m_impl->m_handler->m_encoding)
@@ -456,15 +454,6 @@ void ADIOS2File::configure_IO()
                 ? ADIOS2IOHandlerImpl::ModifiableAttributes::Yes
                 : ADIOS2IOHandlerImpl::ModifiableAttributes::No;
         }
-#else
-        if (!m_impl->m_useGroupTable.has_value())
-        {
-            m_impl->m_useGroupTable = UseGroupTable::No;
-        }
-
-        m_impl->m_modifiableAttributes =
-            ADIOS2IOHandlerImpl::ModifiableAttributes::No;
-#endif
     }
 
     // set engine type
@@ -764,9 +753,7 @@ adios2::Engine &ADIOS2File::getEngine()
             streamStatus = StreamStatus::DuringStep;
             break;
         }
-#if openPMD_HAS_ADIOS_2_8
         case adios2::Mode::ReadRandomAccess:
-#endif
         case adios2::Mode::Read: {
             m_engine =
                 std::make_optional(adios2::Engine(m_IO.Open(m_file, m_mode)));
@@ -1320,9 +1307,7 @@ void ADIOS2File::markActive(Writable *writable)
     {
     case UseGroupTable::No:
         break;
-    case UseGroupTable::Yes:
-#if openPMD_HAS_ADIOS_2_9
-    {
+    case UseGroupTable::Yes: {
         if (writeOnly(m_mode) && m_impl->m_writeAttributesFromThisRank)
         {
             auto currentStepBuffered = currentStep();
@@ -1346,12 +1331,6 @@ void ADIOS2File::markActive(Writable *writable)
                          m_pathsMarkedAsActive.end());
         }
     }
-#else
-        (void)writable;
-        throw error::OperationUnsupportedInBackend(
-            m_impl->m_handler->backendName(),
-            "Group table feature requires ADIOS2 >= v2.9.");
-#endif
     break;
     }
 }
diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp
diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp
diff --git a/test/Files_SerialIO/close_and_reopen_test.cpp b/test/Files_SerialIO/close_and_reopen_test.cpp
diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp
diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp
diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`	`#`
`7`	`7`	`spack:`
`8`	`8`	`specs:`
`9`		`- - adios2`
	`9`	`+ - adios2@2.10`
`10`	`10`	`- hdf5`
`11`	`11`	`- openmpi`
`12`	`12`
Original file line number	Diff line number	Diff line change
`@@ -428,8 +428,6 @@ void ADIOS2File::configure_IO()`
`428`	`428`	`// these properties are inferred from the opened dataset in read mode`
`429`	`429`	`if (writeOnly(m_mode))`
`430`	`430`	`{`
`431`		`-`
`432`		`-#if openPMD_HAS_ADIOS_2_9`
`433`	`431`	`if (!m_impl->m_useGroupTable.has_value())`
`434`	`432`	`{`
`435`	`433`	`switch (m_impl->m_handler->m_encoding)`
`@@ -456,15 +454,6 @@ void ADIOS2File::configure_IO()`
`456`	`454`	`? ADIOS2IOHandlerImpl::ModifiableAttributes::Yes`
`457`	`455`	`: ADIOS2IOHandlerImpl::ModifiableAttributes::No;`
`458`	`456`	`}`
`459`		`-#else`
`460`		`- if (!m_impl->m_useGroupTable.has_value())`
`461`		`- {`
`462`		`- m_impl->m_useGroupTable = UseGroupTable::No;`
`463`		`- }`
`464`		`-`
`465`		`- m_impl->m_modifiableAttributes =`
`466`		`- ADIOS2IOHandlerImpl::ModifiableAttributes::No;`
`467`		`-#endif`
`468`	`457`	`}`
`469`	`458`
`470`	`459`	`// set engine type`
`@@ -764,9 +753,7 @@ adios2::Engine &ADIOS2File::getEngine()`
`764`	`753`	`streamStatus = StreamStatus::DuringStep;`
`765`	`754`	`break;`
`766`	`755`	`}`
`767`		`-#if openPMD_HAS_ADIOS_2_8`
`768`	`756`	`case adios2::Mode::ReadRandomAccess:`
`769`		`-#endif`
`770`	`757`	`case adios2::Mode::Read: {`
`771`	`758`	`m_engine =`
`772`	`759`	`std::make_optional(adios2::Engine(m_IO.Open(m_file, m_mode)));`
`@@ -1320,9 +1307,7 @@ void ADIOS2File::markActive(Writable *writable)`
`1320`	`1307`	`{`
`1321`	`1308`	`case UseGroupTable::No:`
`1322`	`1309`	`break;`
`1323`		`- case UseGroupTable::Yes:`
`1324`		`-#if openPMD_HAS_ADIOS_2_9`
`1325`		`- {`
	`1310`	`+ case UseGroupTable::Yes: {`
`1326`	`1311`	`if (writeOnly(m_mode) && m_impl->m_writeAttributesFromThisRank)`
`1327`	`1312`	`{`
`1328`	`1313`	`auto currentStepBuffered = currentStep();`
`@@ -1346,12 +1331,6 @@ void ADIOS2File::markActive(Writable *writable)`
`1346`	`1331`	`m_pathsMarkedAsActive.end());`
`1347`	`1332`	`}`
`1348`	`1333`	`}`
`1349`		`-#else`
`1350`		`- (void)writable;`
`1351`		`- throw error::OperationUnsupportedInBackend(`
`1352`		`- m_impl->m_handler->backendName(),`
`1353`		`- "Group table feature requires ADIOS2 >= v2.9.");`
`1354`		`-#endif`
`1355`	`1334`	`break;`
`1356`	`1335`	`}`
`1357`	`1336`	`}`