cylondata · nirandaperera · Sep 10, 2024 · Sep 12, 2024 · Sep 12, 2024
diff --git a/.gitignore b/.gitignore
@@ -38,6 +38,7 @@
 
 # ignore these folders
 build/
+debug/
 **/bin/
 
 

diff --git a/conda/environments/cylon.yml b/conda/environments/cylon.yml
@@ -3,19 +3,22 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.8,<3.10
-  - cmake>=3.23.1,!=3.25.0
-  - arrow-cpp=9
-  - pyarrow=9.0.0
+  - python>=3.9,<3.12
+  - cmake
+  - pyarrow=16.1.0
+  - libarrow-acero==16.1.0.*
+  - libarrow-dataset==16.1.0.*
+  - libarrow==16.1.0.*
   - glog
-  - openmpi=4.1.3=ha1ae619_105
-  - ucx>=1.12.1
-  - cython>=0.29.31,<3
-  - numpy<1.24.4
-  - pandas>=1.0,<2.0.0
+  - openmpi
+  - ucx
+  - cython>=0.29.31
+  - numpy>=1.23,<2.0a0
+  - pandas>=2.0,<2.2.3dev
   - fsspec>=0.6.0
   - setuptools
   # they are not needed for using pygcylon or compiling it
   - pytest
   - pytest-mpi
   - mpi4py
+
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -24,17 +24,18 @@
 cmake_minimum_required(VERSION 3.17 FATAL_ERROR)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-project(CYLON VERSION 0.6.0)
+set(CYLON_VERSION 0.7.0)
+
+project(CYLON VERSION ${CYLON_VERSION})
 
-set(CYLON_VERSION 0.6.0)
 
 ## defaults to release build
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release)
 endif ()
 
 # cmake modules directories
-set(CYLON_ARROW_VERSION 9.0.0)
+set(CYLON_ARROW_VERSION 16.1.0)
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake/Modules/" ${CMAKE_MODULE_PATH})
 list(APPEND CMAKE_MODULE_PATH ${CYLON_SOURCE_DIR}/CMake)
 
@@ -74,7 +75,7 @@ else ()
 endif ()
 
 # C++ standard
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(GCC_ABI_COMPILE_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0")
@@ -398,7 +399,19 @@ if (${ARROW_BUILD_TYPE} STREQUAL "SYSTEM")
     message(STATUS "Arrow dataset lib: ${ARROW_DATASET_LIB}")
 
     if (PYCYLON_BUILD)
-        find_library(ARROW_PY_LIB arrow_python ${CYLON_ARROW_VERSION} REQUIRED)
+        find_package (Python3 COMPONENTS Interpreter)
+        write_file("find_site_packages.py" [=[import sysconfig; print(sysconfig.get_paths()["purelib"])]=])
+        execute_process(COMMAND ${Python3_EXECUTABLE} "find_site_packages.py"   
+            RESULT_VARIABLE PROC_RES 
+            OUTPUT_VARIABLE PROC_OUT
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_STRIP_TRAILING_WHITESPACE)
+        if (PROC_RES)
+            message(FATAL_ERROR "Unable to find python site packages dir: ${PROC_OUT}")
+        else()
+            message(STATUS "python site packages dir: ${PROC_OUT}")
+        endif()
+        find_library(ARROW_PY_LIB arrow_python PATHS "${PROC_OUT}" PATH_SUFFIXES "pyarrow" REQUIRED NO_DEFAULT_PATH)
         message(STATUS "Arrow py lib: ${ARROW_PY_LIB}")
     endif (PYCYLON_BUILD)
 

diff --git a/cpp/src/cylon/arrow/arrow_comparator.cpp b/cpp/src/cylon/arrow/arrow_comparator.cpp
@@ -150,7 +150,7 @@ struct CompareFunc<ArrowT, Asc, arrow::enable_if_floating_point<ArrowT>> {
 template<typename ArrowT, bool Asc>
 struct CompareFunc<ArrowT, Asc, arrow::enable_if_has_string_view<ArrowT>> {
 
-  static int compare(const arrow::util::string_view &v1, const arrow::util::string_view &v2) {
+  static int compare(const std::string_view &v1, const std::string_view &v2) {
     if (Asc) {
       return v1.compare(v2);
     } else {

diff --git a/cpp/src/cylon/arrow/arrow_partition_kernels.cpp b/cpp/src/cylon/arrow/arrow_partition_kernels.cpp
@@ -260,7 +260,7 @@ class FixedSizeBinaryHashPartitionKernel : public HashPartitionKernel {
       return
           visit_chunked_array<arrow::FixedSizeBinaryType>(
               idx_col,
-              [&](uint64_t global_idx, arrow::util::string_view val) {
+              [&](uint64_t global_idx, std::string_view val) {
                 uint32_t hash = 0;
                 util::MurmurHash3_x86_32(&val, len, 0, &hash);
                 hash += 31 * target_partitions[global_idx];
@@ -276,7 +276,7 @@ class FixedSizeBinaryHashPartitionKernel : public HashPartitionKernel {
     } else {
       return visit_chunked_array<arrow::FixedSizeBinaryType>(
           idx_col,
-          [&](uint64_t global_idx, arrow::util::string_view val) {
+          [&](uint64_t global_idx, std::string_view val) {
             uint32_t hash = 0;
             util::MurmurHash3_x86_32(&val, len, 0, &hash);
             hash += 31 * target_partitions[global_idx];
@@ -302,7 +302,7 @@ class FixedSizeBinaryHashPartitionKernel : public HashPartitionKernel {
 
     return visit_chunked_array<arrow::FixedSizeBinaryType>
         (idx_col,
-         [&](uint64_t global_idx, arrow::util::string_view val) {
+         [&](uint64_t global_idx, std::string_view val) {
            uint32_t hash = 0;
            util::MurmurHash3_x86_32(&val, byte_width, 0, &hash);
            hash += 31 * partial_hashes[global_idx];
@@ -348,7 +348,7 @@ class BinaryHashPartitionKernel : public HashPartitionKernel {
       return
           visit_chunked_array<ArrowT>(
               idx_col,
-              [&](uint64_t global_idx, arrow::util::string_view val) {
+              [&](uint64_t global_idx, std::string_view val) {
                 uint32_t hash = 0;
                 util::MurmurHash3_x86_32(&val, static_cast<int>(val.size()), 0, &hash);
                 hash += 31 * target_partitions[global_idx];
@@ -365,7 +365,7 @@ class BinaryHashPartitionKernel : public HashPartitionKernel {
       return
           visit_chunked_array<ArrowT>(
               idx_col,
-              [&](uint64_t global_idx, arrow::util::string_view val) {
+              [&](uint64_t global_idx, std::string_view val) {
                 uint32_t hash = 0;
                 util::MurmurHash3_x86_32(&val, static_cast<int>(val.size()), 0, &hash);
                 hash += 31 * target_partitions[global_idx];
@@ -390,7 +390,7 @@ class BinaryHashPartitionKernel : public HashPartitionKernel {
     return
         visit_chunked_array<arrow::FixedSizeBinaryType>(
             idx_col,
-            [&](uint64_t global_idx, arrow::util::string_view val) {
+            [&](uint64_t global_idx, std::string_view val) {
               uint32_t hash = 0;
               util::MurmurHash3_x86_32(&val, static_cast<int>(val.size()), 0, &hash);
               hash += 31 * partial_hashes[global_idx];

diff --git a/cpp/src/cylon/arrow/arrow_type_traits.hpp b/cpp/src/cylon/arrow/arrow_type_traits.hpp
@@ -15,6 +15,8 @@
 #ifndef CYLON_CPP_SRC_CYLON_ARROW_ARROW_TYPE_TRAITS_HPP_
 #define CYLON_CPP_SRC_CYLON_ARROW_ARROW_TYPE_TRAITS_HPP_
 
+#include <string_view>
+
 namespace cylon {
 
 template<typename ArrowT, typename Enable = void>
@@ -35,7 +37,7 @@ template<typename ArrowT>
 struct ArrowTypeTraits<ArrowT, arrow::enable_if_has_string_view<ArrowT>> {
   using ScalarT = typename arrow::TypeTraits<ArrowT>::ScalarType;
   using ArrayT = typename arrow::TypeTraits<ArrowT>::ArrayType;
-  using ValueT = arrow::util::string_view;
+  using ValueT = std::string_view;
 
   static ValueT ExtractFromScalar(const std::shared_ptr<arrow::Scalar> &scalar) {
     return ValueT(*(std::static_pointer_cast<ScalarT>(scalar))->value);