Memory64 support

Signed-off-by: Andrew Stein <[email protected]>

.cargo/config.toml

@@ -7,6 +7,10 @@ target-dir = "rust/target"
 
 [target.wasm32-unknown-unknown]
 runner = 'wasm-bindgen-test-runner'
+rustflags = [
+    "--cfg=web_sys_unstable_apis",
+    "-Ctarget-feature=+bulk-memory,+simd128,+relaxed-simd",
+]
 
 [target.i686-pc-windows-msvc]
 rustflags = ["-C", "target-feature=+crt-static", "--cfg=web_sys_unstable_apis"]

Cargo.toml

@@ -45,3 +45,6 @@ opt-level = "z"
 codegen-units = 1
 lto = true
 strip = true
+
+[patch.crates-io]
+simd-adler32 = { git = "https://github.com/mcountryman/simd-adler32.git", rev = "140cde033e8b9a12d4de840648c65ccd5320bcc5" }

cmake/Boost.txt.in

@@ -7,7 +7,7 @@ cmake_policy(SET CMP0097 NEW)
 
 include(ExternalProject)
 ExternalProject_Add(Boost
-  URL               "https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz"
+  URL               "https://sourceforge.net/projects/boost/files/boost/1.82.0/boost_1_82_0.tar.gz"
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/Boost-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/Boost-build"
   SOURCE_SUBDIR     ""

cmake/modules/FindInstallDependency.cmake

@@ -38,9 +38,7 @@ function(psp_build_dep name cmake_file)
         endif()
     endif()
     if(${name} STREQUAL arrow)
-        set(ARROW_SIMD_LEVEL "NONE")
         set(ARROW_DEFINE_OPTIONS ON)
-        set(ARROW_RUNTIME_SIMD_LEVEL "NONE")
         set(ARROW_BUILD_SHARED OFF)
         set(ARROW_BUILD_STATIC ON)
         set(ARROW_BUILD_INTEGRATION OFF)
@@ -51,6 +49,10 @@ function(psp_build_dep name cmake_file)
         set(ARROW_WITH_LZ4 ON)
         set(ARROW_NO_EXPORT ON)
         set(ARROW_DEPENDENCY_SOURCE "BUNDLED" CACHE STRING "override arrow's dependency source" FORCE)
+        # if (PSP_ENABLE_WASM)
+        #     set(ARROW_CXX_FLAGS_RELEASE " -msimd128 -mbulk-memory -mrelaxed-simd -s MEMORY64=1 " CACHE STRING "override arrow's dependency source" FORCE)
+        #     set(ARROW_C_FLAGS_RELEASE " -msimd128 -mbulk-memory -mrelaxed-simd -s MEMORY64=1 " CACHE STRING "override arrow's dependency source" FORCE    )
+        # endif()
         if(PSP_WASM_BUILD)
             set(ARROW_ENABLE_THREADING OFF)
         else()
@@ -79,6 +81,8 @@ function(psp_build_dep name cmake_file)
             EXCLUDE_FROM_ALL)
         set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src/src PARENT_SCOPE)
     elseif(${name} STREQUAL rapidjson)
+        set(RAPIDJSON_SSE42 ON CACHE BOOL "Enable RapidJSON SIMD")
+        set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests")
         add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
             ${CMAKE_BINARY_DIR}/${name}-build
             EXCLUDE_FROM_ALL)

cmake/rapidjson.txt.in

@@ -5,7 +5,7 @@ project(rapidjson-download NONE)
 include(ExternalProject)
 ExternalProject_Add(rapidjson
   GIT_REPOSITORY    https://github.com/tencent/rapidjson.git
-  GIT_TAG           v1.1.0
+  GIT_TAG           d621dc9e9c77f81e5c8a35b8dcc16dcd63351321
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/rapidjson-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/rapidjson-build"
   CONFIGURE_COMMAND ""

cpp/perspective/CMakeLists.txt

@@ -29,6 +29,8 @@ endif()
 set(CMAKE_MODULE_PATH "${PSP_CMAKE_MODULE_PATH}/modules" ${CMAKE_MODULE_PATH})
 set(MSVC_RUNTIME_LIBRARY MultiThreaded)
 
+option(PSP_WASM64 "Enable WASM Memory64 support" OFF)
+
 if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
     set(WIN32 ON)
     set(MACOS OFF)
@@ -229,11 +231,13 @@ if(PSP_WASM_BUILD)
             ")
         endif()
     else()
-        # Pyodide RELEASE block
-        set(OPT_FLAGS " \
-            -O3 \
-            -g0 \
+        set(OPT_FLAGS " -O3 -g0 ")
+        if(NOT PSP_PYODIDE)
+            set(OPT_FLAGS " \
+                ${OPT_FLAGS} \
             ")
+        endif()
+
         if (PSP_WASM_EXCEPTIONS)
             set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
         endif()
@@ -299,27 +303,43 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
     endif()
 endif()
 
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
--O3 \
-")
 
 if (PSP_WASM_EXCEPTIONS)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
+        -O3 \
+        -g0 \
+    ")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
-    -fwasm-exceptions \
-    -O3 \
-    -g0 \
+        -fwasm-exceptions \
+        -O3 \
+        -g0 \
     ")
 else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
     -O3 \
     ")
 endif()
 
+if (PSP_WASM64)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
+        -mbulk-memory \
+        -msimd128 \
+        -mrelaxed-simd \
+        -s MEMORY64=1 \
+    ")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+        -mbulk-memory \
+        -msimd128 \
+        -mrelaxed-simd \
+        -s MEMORY64=1 \
+    ")
+endif()
+
 set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests")
 
 if(PSP_PYODIDE)
-    set(RELOCATABLE_FLAGS "-sRELOCATABLE=1 -sSIDE_MODULE=2 -sWASM_BIGINT=1")
-    string(APPEND CMAKE_EXE_LINKER_FLAGS "${RELOCATABLE_FLAGS}")
+    set(RELOCATABLE_FLAGS " -sRELOCATABLE=1 -sSIDE_MODULE=2 ")
+    string(APPEND CMAKE_EXE_LINKER_FLAGS "${RELOCATABLE_FLAGS} -sWASM_BIGINT=1 ")
     string(APPEND CMAKE_C_FLAGS "${RELOCATABLE_FLAGS}")
     string(APPEND CMAKE_CXX_FLAGS "${RELOCATABLE_FLAGS}")
 endif()
@@ -490,10 +510,12 @@ endif()
 if(PSP_PYODIDE)
     set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
         --no-entry \
-        -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \
+        -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server,_psp_is_memory64 \
         -s SIDE_MODULE=2 \
     ")
 else()
+# -s MEMORY_GROWTH_GEOMETRIC_STEP=1.0 \
+# -s MEMORY_GROWTH_GEOMETRIC_CAP=536870912 \
     set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
         --no-entry \
         --closure=1 \
@@ -505,16 +527,26 @@ else()
         -s TEXTDECODER=2 \
         -s STANDALONE_WASM=1 \
         -s DYNAMIC_EXECUTION=0 \
-        -s BINARYEN_EXTRA_PASSES=--one-caller-inline-max-function-size=19306 \
+        -s POLYFILL=0 \
         -s EXPORT_NAME=\"load_perspective\" \
-        -s MAXIMUM_MEMORY=4gb \
+        -s MAXIMUM_MEMORY=16gb \
         -s ERROR_ON_UNDEFINED_SYMBOLS=0 \
         -s NODEJS_CATCH_EXIT=0 \
         -s NODEJS_CATCH_REJECTION=0 \
         -s USE_ES6_IMPORT_META=1 \
         -s EXPORT_ES6=1 \
-        -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \
+        -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server,_psp_is_memory64 \
     ")
+
+    if(PSP_WASM64)
+        set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \
+            -s MAXIMUM_MEMORY=16gb \
+        ")
+    else()
+        set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \
+            -s MAXIMUM_MEMORY=4gb \
+        ")
+    endif()
 endif()
 
 if (PSP_WASM_EXCEPTIONS)

cpp/perspective/patches/arrow_strptime.patch

@@ -1,3 +1,19 @@
+diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
+index c8457eae8..aa2b41cc9 100644
+--- a/cpp/src/arrow/util/decimal.cc
++++ b/cpp/src/arrow/util/decimal.cc
+@@ -111,7 +111,11 @@ struct DecimalRealConversion : public BaseDecimalRealConversion {
+
+     // 2. Losslessly convert `real` to `mant * 2**k`
+     int binary_exp = 0;
++#ifdef __EMSCRIPTEN__
++    const Real real_mant = std::frexpf(real, &binary_exp);
++#else
+     const Real real_mant = std::frexp(real, &binary_exp);
++#endif
+     // `real_mant` is within 0.5 and 1 and has M bits of precision.
+     // Multiply it by 2^M to get an exact integer.
+     const uint64_t mant = static_cast<uint64_t>(std::ldexp(real_mant, kMantissaBits));
 diff --git a/cpp/src/arrow/util/value_parsing.h b/cpp/src/arrow/util/value_parsing.h
 index 609906052..1e3dfae7c 100644
 --- a/cpp/src/arrow/util/value_parsing.h
@@ -11,3 +27,27 @@ index 609906052..1e3dfae7c 100644
    char* ret = arrow_strptime(clean_copy.c_str(), format, &result);
  #else
    char* ret = strptime(clean_copy.c_str(), format, &result);
+diff --git a/cpp/src/arrow/vendored/xxhash/xxhash.h b/cpp/src/arrow/vendored/xxhash/xxhash.h
+index a18e8c762..235590b19 100644
+--- a/cpp/src/arrow/vendored/xxhash/xxhash.h
++++ b/cpp/src/arrow/vendored/xxhash/xxhash.h
+@@ -169,6 +169,11 @@
+  * xxHash prototypes and implementation
+  */
+
++
++#ifdef __EMSCRIPTEN__
++#include <emscripten.h>
++#endif
++
+ #if defined (__cplusplus)
+ extern "C" {
+ #endif
+@@ -3422,6 +3427,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can
+ #  endif
+ #endif
+
++
+ #if defined(__GNUC__) || defined(__clang__)
+ #  if defined(__ARM_FEATURE_SVE)
+ #    include <arm_sve.h>

cpp/perspective/src/cpp/binding_api.cpp

@@ -119,4 +119,14 @@ psp_delete_server(void* proto_server) {
     delete server;
 }
 
+PERSPECTIVE_EXPORT
+bool
+psp_is_memory64() {
+#if UINTPTR_MAX == 0xffffffff
+    return false;
+#elif UINTPTR_MAX == 0xffffffffffffffff
+    return true;
+#endif
+}
+
 } // end extern "C"

cpp/perspective/src/cpp/computed_function.cpp

@@ -1233,9 +1233,9 @@ bucket::operator()(t_parameter_list parameters) {
     }
     std::string allowed_units = "smhDWMY";
     if (allowed_units.find(temp_unit) == std::string::npos) {
-        std::cerr << "[bucket] unknown unit in bucket - the valid units "
-                     "are 's', 'm', 'h', 'D', 'W', 'M', and 'Y'."
-                  << '\n';
+        // std::cerr << "[bucket] unknown unit in bucket - the valid units "
+        //              "are 's', 'm', 'h', 'D', 'W', 'M', and 'Y'."
+        //           << '\n';
         rval.m_type = DTYPE_TIME;
         rval.m_status = STATUS_CLEAR;
         return rval;

cpp/perspective/src/cpp/gnode.cpp

@@ -997,7 +997,7 @@ t_gnode::get_registered_contexts() const {
 
 void
 t_gnode::_register_context(
-    const std::string& name, t_ctx_type type, std::int64_t ptr
+    const std::string& name, t_ctx_type type, std::uintptr_t ptr
 ) {
     PSP_TRACE_SENTINEL();
     PSP_VERBOSE_ASSERT(m_init, "touching uninited object");

cpp/perspective/src/cpp/pool.cpp

@@ -10,6 +10,7 @@
 // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
 // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
 
+#include <cstdint>
 #include <perspective/first.h>
 #include <perspective/base.h>
 #include <perspective/pool.h>
@@ -182,39 +183,21 @@ t_pool::get_trees() {
     return rval;
 }
 
-#if defined PSP_ENABLE_WASM and !defined(PSP_ENABLE_PYTHON)
 void
 t_pool::register_context(
     t_uindex gnode_id,
     const std::string& name,
     t_ctx_type type,
-    std::int32_t ptr
-) {
-
-    if (!validate_gnode_id(gnode_id)) {
-        return;
-    }
-    m_gnodes[gnode_id]->_register_context(name, type, ptr);
-}
-
-#else
-void
-t_pool::register_context(
-    t_uindex gnode_id,
-    const std::string& name,
-    t_ctx_type type,
-    std::int64_t ptr
+    std::uintptr_t ptr
 ) {
 #ifdef PSP_PARALLEL_FOR
     PSP_WRITE_LOCK(*m_lock)
 #endif
-
     if (!validate_gnode_id(gnode_id)) {
         return;
     }
     m_gnodes[gnode_id]->_register_context(name, type, ptr);
 }
-#endif
 
 void
 t_pool::notify_userspace(t_uindex port_id) {

cpp/perspective/src/cpp/server.cpp

@@ -38,6 +38,10 @@
 #include <vector>
 #include <ctime>
 
+#if !defined(WIN32) && !defined(PSP_ENABLE_WASM)
+#include <sys/resource.h>
+#endif
+
 namespace perspective {
 std::uint32_t server::ProtoServer::m_client_id = 1;
 
@@ -2414,7 +2418,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) {
 #if defined(PSP_ENABLE_WASM) && !defined(PSP_ENABLE_PYTHON)
             auto heap_size = psp_heap_size();
             sys_info->set_heap_size(heap_size);
-#elif !defined(WIN32)
+#elif !defined(WIN32) && !defined(PSP_ENABLE_WASM)
             rusage out;
             getrusage(RUSAGE_SELF, &out);
             sys_info->set_heap_size(out.ru_maxrss);

cpp/perspective/src/include/perspective/gnode.h

@@ -143,7 +143,7 @@ class PERSPECTIVE_EXPORT t_gnode {
      * @param ptr
      */
     void _register_context(
-        const std::string& name, t_ctx_type type, std::int64_t ptr
+        const std::string& name, t_ctx_type type, std::uintptr_t ptr
     );
 
     /**

cpp/perspective/src/include/perspective/pool.h

@@ -48,21 +48,12 @@ class PERSPECTIVE_EXPORT t_pool {
     //   void set_update_delegate(t_val ud);
     // #endif
 
-#if defined PSP_ENABLE_WASM and !defined PSP_ENABLE_PYTHON
     void register_context(
         t_uindex gnode_id,
         const std::string& name,
         t_ctx_type type,
-        std::int32_t ptr
+        std::uintptr_t ptr
     );
-#else
-    void register_context(
-        t_uindex gnode_id,
-        const std::string& name,
-        t_ctx_type type,
-        std::int64_t ptr
-    );
-#endif
 
 #ifdef PSP_PARALLEL_FOR
     std::shared_mutex* get_lock() const;