Fixes to invocation of DBWY's linalg CMake modules, bumped scalapackp…

…p version to fix problems in BLACS/ScaLAPACK discovery (#256) + fixed associated API changes

CMakeLists.txt

@@ -289,8 +289,26 @@ detect_MADNESS_configuration()
 include(external/eigen.cmake)
 # use David Williams-Young's linear algebra discovery modules
 include("${vg_cmake_kit_SOURCE_DIR}/modules/FetchWfn91LinAlgModules.cmake")
-find_package(BLAS REQUIRED)
-find_package(LAPACK REQUIRED)
+if(ENABLE_SCALAPACK)
+  find_package(ScaLAPACK REQUIRED)
+  # Propagate ScaLAPACK -> BLAS/LAPACK if not set 
+  # (ScaLAPACK necessacarily contains a BLAS/LAPACK linker by standard)
+  # TODO: Tell David to write a macro that hides this verbosity from user space
+  if( NOT BLAS_LIBRARIES )
+    set( BLAS_LIBRARIES "${ScaLAPACK_LIBRARIES}" CACHE STRING "BLAS LIBRARIES" )
+  endif()
+  if( NOT LAPACK_LIBRARIES )
+    set( LAPACK_LIBRARIES "${ScaLAPACK_LIBRARIES}" CACHE STRING "LAPACK LIBRARIES" )
+  endif()
+else()
+  find_package(LAPACK REQUIRED)
+  # Propagate LAPACK -> BLAS if not set 
+  # (LAPACK necessacarily contains a BLAS linker by standard)
+  # TODO: Tell David to write a macro that hides this verbosity from user space
+  if( NOT BLAS_LIBRARIES )
+    set( BLAS_LIBRARIES "${LAPACK_LIBRARIES}" CACHE STRING "BLAS LIBRARIES" )
+  endif()
+endif()
 # BTAS does a better job of building and checking Boost since it uses Boost::serialization
 # it also memorized the location of its config for use from install tree
 include(FindOrFetchBTAS)

external/scalapack.cmake

@@ -16,40 +16,21 @@ if( TARGET scalapackpp::scalapackpp )
 else()
 
   message(STATUS "Could not find scalapackpp! Building..." )
+  include(FetchContent)
 
-  include( DownloadProject )
-  download_project(
-    PROJ                blacspp
-    GIT_REPOSITORY      https://github.com/wavefunction91/blacspp.git
-    GIT_TAG             ${TA_TRACKED_BLACSPP_TAG}
-    PREFIX              ${PROJECT_BINARY_DIR}/external
-    UPDATE_DISCONNECTED 1
-  )
-  download_project(
-    PROJ                scalapackpp
+  FetchContent_Declare( scalapackpp
     GIT_REPOSITORY      https://github.com/wavefunction91/scalapackpp.git
     GIT_TAG             ${TA_TRACKED_SCALAPACKPP_TAG}
-    PREFIX              ${PROJECT_BINARY_DIR}/external
-    UPDATE_DISCONNECTED 1
   )
-
-  if( DEFINED SCALAPACK_LIBRARIES )
-    set( scalapack_LIBRARIES ${SCALAPACK_LIBRARIES} )
-    set( blacs_LIBRARIES     ${SCALAPACK_LIBRARIES} )
-  endif()
-
-  set( BLACSPP_ENABLE_TESTS OFF )
-  set( SCALAPACKPP_ENABLE_TESTS OFF )
-  add_subdirectory( ${blacspp_SOURCE_DIR} ${blacspp_BINARY_DIR} )
-  add_subdirectory( ${scalapackpp_SOURCE_DIR} ${scalapackpp_BINARY_DIR} )
+  FetchContent_MakeAvailable( scalapackpp )
 
   # propagate MPI_CXX_SKIP_MPICXX=ON
-  target_compile_definitions( blacspp PRIVATE ${MPI_CXX_COMPILE_DEFINITIONS} )
+  target_compile_definitions( blacspp     PRIVATE ${MPI_CXX_COMPILE_DEFINITIONS} )
   target_compile_definitions( scalapackpp PRIVATE ${MPI_CXX_COMPILE_DEFINITIONS} )
 
   install( TARGETS blacspp scalapackpp EXPORT tiledarray COMPONENT tiledarray )
   # Add these dependencies to External
-  add_dependencies(External-tiledarray scalapackpp blacspp)
+  add_dependencies(External-tiledarray scalapackpp )
 
   # set {blacspp,scalapackpp}_CONFIG to the install location so that we know where to find it
   set(blacspp_CONFIG ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/cmake/blacspp/blacspp-config.cmake)

external/versions.cmake

@@ -31,11 +31,11 @@ set(TA_TRACKED_CUTT_PREVIOUS_TAG 592198b93c93b7ca79e7900b9a9f2e79f9dafec3)
 set(TA_TRACKED_UMPIRE_TAG f04abd1dd038c84262915a493d8f78576bb80fd0)
 set(TA_TRACKED_UMPIRE_PREVIOUS_TAG f04abd1dd038c84262915a493d8f78576bb80fd0)
 
-set(TA_TRACKED_BLACSPP_TAG 20cfd414c5b719be1c958f4a2d57abef06df83b6 )
-set(TA_TRACKED_BLACSPP_PREVIOUS_TAG     da4ada57e578cf944325a7152164306742551596 )
+#set(TA_TRACKED_BLACSPP_TAG 20cfd414c5b719be1c958f4a2d57abef06df83b6 )
+#set(TA_TRACKED_BLACSPP_PREVIOUS_TAG     da4ada57e578cf944325a7152164306742551596 )
 
-set(TA_TRACKED_SCALAPACKPP_TAG 534deef123b91494c7718b30f00d191fc4c7ede1 )
-set(TA_TRACKED_SCALAPACKPP_PREVIOUS_TAG 534deef123b91494c7718b30f00d191fc4c7ede1 )
+set(TA_TRACKED_SCALAPACKPP_TAG 04a018cf24a74bd7d4cec66967c4bec850d8175c )
+set(TA_TRACKED_SCALAPACKPP_PREVIOUS_TAG 04a018cf24a74bd7d4cec66967c4bec850d8175c )
 
 set(TA_TRACKED_RANGEV3_TAG 2e0591c57fce2aca6073ad6e4fdc50d841827864)
 set(TA_TRACKED_RANGEV3_PREVIOUS_TAG dbdaa247a25a0daa24c68f1286a5693c72ea0006)

src/TiledArray/math/linalg/scalapack/block_cyclic.h

@@ -40,7 +40,7 @@
 #include <blacspp/information.hpp>
 
 #include <scalapackpp/block_cyclic.hpp>
-#include <scalapackpp/util/sfinae.hpp>
+#include <scalapackpp/util/type_traits.hpp>
 
 namespace TiledArray::math::linalg::scalapack {
 

src/TiledArray/math/linalg/scalapack/cholesky.h

@@ -73,12 +73,12 @@ auto cholesky(const Array& A, TiledRange l_trange = TiledRange(),
   auto [Mloc, Nloc] = matrix.dist().get_local_dims(N, N);
   auto desc = matrix.dist().descinit_noerror(N, N, Mloc);
 
-  auto info = scalapackpp::ppotrf(blacspp::Triangle::Lower, N,
+  auto info = scalapackpp::ppotrf(blacspp::Uplo::Lower, N,
                                   matrix.local_mat().data(), 1, 1, desc);
   if (info) TA_EXCEPTION("Cholesky Failed");
 
   // Zero out the upper triangle
-  zero_triangle(blacspp::Triangle::Upper, matrix);
+  zero_triangle(blacspp::Uplo::Upper, matrix);
 
   if (l_trange.rank() == 0) l_trange = A.trange();
 
@@ -129,12 +129,12 @@ auto cholesky_linv(const Array& A, TiledRange l_trange = TiledRange(),
   auto [Mloc, Nloc] = matrix.dist().get_local_dims(N, N);
   auto desc = matrix.dist().descinit_noerror(N, N, Mloc);
 
-  auto info = scalapackpp::ppotrf(blacspp::Triangle::Lower, N,
+  auto info = scalapackpp::ppotrf(blacspp::Uplo::Lower, N,
                                   matrix.local_mat().data(), 1, 1, desc);
   if (info) TA_EXCEPTION("Cholesky Failed");
 
   // Zero out the upper triangle
-  zero_triangle(blacspp::Triangle::Upper, matrix);
+  zero_triangle(blacspp::Uplo::Upper, matrix);
 
   // Copy L if needed
   std::shared_ptr<scalapack::BlockCyclicMatrix<value_type>> L_sca = nullptr;
@@ -146,7 +146,7 @@ auto cholesky_linv(const Array& A, TiledRange l_trange = TiledRange(),
 
   // Compute inverse
   info =
-      scalapackpp::ptrtri(blacspp::Triangle::Lower, blacspp::Diagonal::NonUnit,
+      scalapackpp::ptrtri(blacspp::Uplo::Lower, blacspp::Diag::NonUnit,
                           N, matrix.local_mat().data(), 1, 1, desc);
   if (info) TA_EXCEPTION("TRTRI Failed");
 
@@ -200,7 +200,7 @@ auto cholesky_solve(const Array& A, const Array& B,
     desc_b = B_sca.dist().descinit_noerror(N, NRHS, Mloc);
   }
 
-  auto info = scalapackpp::pposv(blacspp::Triangle::Lower, N, NRHS,
+  auto info = scalapackpp::pposv(blacspp::Uplo::Lower, N, NRHS,
                                  A_sca.local_mat().data(), 1, 1, desc_a,
                                  B_sca.local_mat().data(), 1, 1, desc_b);
   if (info) TA_EXCEPTION("Cholesky Solve Failed");
@@ -250,18 +250,18 @@ auto cholesky_lsolve(Op trans, const Array& A, const Array& B,
     desc_b = B_sca.dist().descinit_noerror(N, NRHS, Mloc);
   }
 
-  auto info = scalapackpp::ppotrf(blacspp::Triangle::Lower, N,
+  auto info = scalapackpp::ppotrf(blacspp::Uplo::Lower, N,
                                   A_sca.local_mat().data(), 1, 1, desc_a);
   if (info) TA_EXCEPTION("Cholesky Failed");
 
   info = scalapackpp::ptrtrs(
-      blacspp::Triangle::Lower, to_scalapackpp_transposeflag(trans),
-      blacspp::Diagonal::NonUnit, N, NRHS, A_sca.local_mat().data(), 1, 1,
+      blacspp::Uplo::Lower, to_scalapackpp_transposeflag(trans),
+      blacspp::Diag::NonUnit, N, NRHS, A_sca.local_mat().data(), 1, 1,
       desc_a, B_sca.local_mat().data(), 1, 1, desc_b);
   if (info) TA_EXCEPTION("TRTRS Failed");
 
   // Zero out the upper triangle
-  zero_triangle(blacspp::Triangle::Upper, A_sca);
+  zero_triangle(blacspp::Uplo::Upper, A_sca);
 
   if (l_trange.rank() == 0) l_trange = A.trange();
   if (x_trange.rank() == 0) x_trange = B.trange();

src/TiledArray/math/linalg/scalapack/heig.h

@@ -80,7 +80,7 @@ auto heig(const Array& A, TiledRange evec_trange = TiledRange(),
   scalapack::BlockCyclicMatrix<value_type> evecs(world, grid, N, N, NB, NB);
 
   auto info = scalapackpp::hereig(
-      scalapackpp::VectorFlag::Vectors, blacspp::Triangle::Lower, N,
+      scalapackpp::Job::Vec, blacspp::Uplo::Lower, N,
       matrix.local_mat().data(), 1, 1, desc, evals.data(),
       evecs.local_mat().data(), 1, 1, desc);
   if (info) TA_EXCEPTION("EVP Failed");
@@ -150,7 +150,7 @@ auto heig(const ArrayA& A, const ArrayB& B,
   scalapack::BlockCyclicMatrix<value_type> evecs(world, grid, N, N, NB, NB);
 
   auto info = scalapackpp::hereig_gen(
-      scalapackpp::VectorFlag::Vectors, blacspp::Triangle::Lower, N,
+      scalapackpp::Job::Vec, blacspp::Uplo::Lower, N,
       A_sca.local_mat().data(), 1, 1, desc, B_sca.local_mat().data(), 1, 1,
       desc, evals.data(), evecs.local_mat().data(), 1, 1, desc);
   if (info) TA_EXCEPTION("EVP Failed");

src/TiledArray/math/linalg/scalapack/lu.h

@@ -67,7 +67,7 @@ auto lu_solve(const ArrayA& A, const ArrayB& B,
   auto [B_Mloc, B_Nloc] = B_sca.dist().get_local_dims(N, NRHS);
   auto desc_b = B_sca.dist().descinit_noerror(N, NRHS, B_Mloc);
 
-  std::vector<scalapackpp::scalapack_int> IPIV(A_Mloc + MB);
+  std::vector<int64_t> IPIV(A_Mloc + MB);
 
   auto info =
       scalapackpp::pgesv(N, NRHS, A_sca.local_mat().data(), 1, 1, desc_a,
@@ -104,7 +104,7 @@ auto lu_inv(const Array& A, TiledRange ainv_trange = TiledRange(),
   auto [A_Mloc, A_Nloc] = A_sca.dist().get_local_dims(N, N);
   auto desc_a = A_sca.dist().descinit_noerror(N, N, A_Mloc);
 
-  std::vector<scalapackpp::scalapack_int> IPIV(A_Mloc + MB);
+  std::vector<int64_t> IPIV(A_Mloc + MB);
 
   {
     auto info = scalapackpp::pgetrf(N, N, A_sca.local_mat().data(), 1, 1,

src/TiledArray/math/linalg/scalapack/svd.h

@@ -94,22 +94,22 @@ auto svd(const Array& A, TiledRange u_trange, TiledRange vt_trange,
   std::shared_ptr<scalapack::BlockCyclicMatrix<value_type>> U = nullptr,
                                                             VT = nullptr;
 
-  scalapackpp::VectorFlag JOBU = scalapackpp::VectorFlag::NoVectors;
-  scalapackpp::VectorFlag JOBVT = scalapackpp::VectorFlag::NoVectors;
+  scalapackpp::Job JOBU = scalapackpp::Job::NoVec;
+  scalapackpp::Job JOBVT = scalapackpp::Job::NoVec;
 
   value_type* U_ptr = nullptr;
   value_type* VT_ptr = nullptr;
 
   if constexpr (need_u) {
-    JOBU = scalapackpp::VectorFlag::Vectors;
+    JOBU = scalapackpp::Job::Vec;
     U = std::make_shared<scalapack::BlockCyclicMatrix<value_type>>(
         world, grid, M, SVD_SIZE, MB, NB);
 
     U_ptr = U->local_mat().data();
   }
 
   if constexpr (need_vt) {
-    JOBVT = scalapackpp::VectorFlag::Vectors;
+    JOBVT = scalapackpp::Job::Vec;
     VT = std::make_shared<scalapack::BlockCyclicMatrix<value_type>>(
         world, grid, SVD_SIZE, N, MB, NB);
 

src/TiledArray/math/linalg/scalapack/util.h

@@ -33,21 +33,21 @@
 
 namespace TiledArray::math::linalg::scalapack {
 
-inline scalapackpp::TransposeFlag to_scalapackpp_transposeflag(Op t) {
+inline scalapackpp::Op to_scalapackpp_transposeflag(Op t) {
   switch (t) {
     case Op::NoTrans:
-      return scalapackpp::TransposeFlag::NoTranspose;
+      return scalapackpp::Op::NoTrans;
     case Op::Trans:
-      return scalapackpp::TransposeFlag::Transpose;
+      return scalapackpp::Op::Trans;
     case Op::ConjTrans:
-      return scalapackpp::TransposeFlag::ConjTranspose;
+      return scalapackpp::Op::ConjTrans;
     default:
       abort();
   }
 }
 
 template <typename T>
-void zero_triangle(blacspp::Triangle tri, scalapack::BlockCyclicMatrix<T>& A,
+void zero_triangle(blacspp::Uplo tri, scalapack::BlockCyclicMatrix<T>& A,
                    bool zero_diag = false) {
   auto zero_el = [&](size_t I, size_t J) {
     if (A.dist().i_own(I, J)) {
@@ -59,7 +59,7 @@ void zero_triangle(blacspp::Triangle tri, scalapack::BlockCyclicMatrix<T>& A,
   auto [M, N] = A.dims();
 
   // Zero the lower triangle
-  if (tri == blacspp::Triangle::Lower) {
+  if (tri == blacspp::Uplo::Lower) {
     if (zero_diag)
       for (size_t j = 0; j < N; ++j)
         for (size_t i = j; i < M; ++i) zero_el(i, j);