From b28f4a3e31b4216f2040b3389c3952b47d42b587 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 29 Dec 2024 23:24:13 -0500 Subject: [PATCH] Memory64 support Signed-off-by: Andrew Stein --- .cargo/config.toml | 4 + Cargo.lock | 4 +- Cargo.toml | 3 + cmake/Boost.txt.in | 2 +- cmake/modules/FindInstallDependency.cmake | 8 +- cmake/rapidjson.txt.in | 2 +- cpp/perspective/CMakeLists.txt | 64 ++++++++++---- cpp/perspective/patches/arrow_strptime.patch | 40 +++++++++ cpp/perspective/src/cpp/binding_api.cpp | 10 +++ cpp/perspective/src/cpp/computed_function.cpp | 6 +- cpp/perspective/src/cpp/gnode.cpp | 2 +- cpp/perspective/src/cpp/pool.cpp | 21 +---- cpp/perspective/src/cpp/server.cpp | 6 +- .../src/include/perspective/gnode.h | 2 +- .../src/include/perspective/pool.h | 11 +-- examples/blocks/package.json | 2 +- pnpm-lock.yaml | 28 ++++++ rust/bootstrap-runtime/Cargo.toml | 13 ++- rust/bootstrap/main.rs | 4 +- rust/bundle/main.rs | 11 ++- rust/perspective-js/Cargo.toml | 3 + rust/perspective-js/src/ts/emscripten_api.ts | 29 ++++--- rust/perspective-js/src/ts/engine.ts | 85 +++++++++++++------ .../test/js/constructors.spec.js | 2 +- .../test/js/expressions/conversions.spec.js | 2 +- tools/perspective-bench/package.json | 4 +- tools/perspective-bench/src/js/benchmark.mjs | 2 +- 27 files changed, 264 insertions(+), 106 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 932b619b2a..5e898565e2 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -7,6 +7,10 @@ target-dir = "rust/target" [target.wasm32-unknown-unknown] runner = 'wasm-bindgen-test-runner' +rustflags = [ + "--cfg=web_sys_unstable_apis", + "-Ctarget-feature=+bulk-memory,+simd128,+relaxed-simd", +] [target.i686-pc-windows-msvc] rustflags = ["-C", "target-feature=+crt-static", "--cfg=web_sys_unstable_apis"] diff --git a/Cargo.lock b/Cargo.lock index 1bcb50263b..180035c1b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1985,7 +1985,6 @@ dependencies = [ name = "perspective-bootstrap-runtime" version = "0.0.0" dependencies = [ - "simd-adler32", "talc", "zune-inflate", ] @@ -2906,8 +2905,7 @@ dependencies = [ [[package]] name = "simd-adler32" version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +source = "git+https://github.com/mcountryman/simd-adler32.git?rev=140cde033e8b9a12d4de840648c65ccd5320bcc5#140cde033e8b9a12d4de840648c65ccd5320bcc5" [[package]] name = "slab" diff --git a/Cargo.toml b/Cargo.toml index 85c0f54580..5da854a7b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,3 +45,6 @@ opt-level = "z" codegen-units = 1 lto = true strip = true + +[patch.crates-io] +simd-adler32 = { git = "https://github.com/mcountryman/simd-adler32.git", rev = "140cde033e8b9a12d4de840648c65ccd5320bcc5" } diff --git a/cmake/Boost.txt.in b/cmake/Boost.txt.in index cb799bd086..139c9c28f0 100644 --- a/cmake/Boost.txt.in +++ b/cmake/Boost.txt.in @@ -7,7 +7,7 @@ cmake_policy(SET CMP0097 NEW) include(ExternalProject) ExternalProject_Add(Boost - URL "https://boostorg.jfrog.io/artifactory/main/release/1.82.0/source/boost_1_82_0.tar.gz" + URL "https://sourceforge.net/projects/boost/files/boost/1.82.0/boost_1_82_0.tar.gz" SOURCE_DIR "${CMAKE_BINARY_DIR}/Boost-src" BINARY_DIR "${CMAKE_BINARY_DIR}/Boost-build" SOURCE_SUBDIR "" diff --git a/cmake/modules/FindInstallDependency.cmake b/cmake/modules/FindInstallDependency.cmake index 431c0c0252..840d6d9b4e 100644 --- a/cmake/modules/FindInstallDependency.cmake +++ b/cmake/modules/FindInstallDependency.cmake @@ -38,9 +38,7 @@ function(psp_build_dep name cmake_file) endif() endif() if(${name} STREQUAL arrow) - set(ARROW_SIMD_LEVEL "NONE") set(ARROW_DEFINE_OPTIONS ON) - set(ARROW_RUNTIME_SIMD_LEVEL "NONE") set(ARROW_BUILD_SHARED OFF) set(ARROW_BUILD_STATIC ON) set(ARROW_BUILD_INTEGRATION OFF) @@ -51,6 +49,10 @@ function(psp_build_dep name cmake_file) set(ARROW_WITH_LZ4 ON) set(ARROW_NO_EXPORT ON) set(ARROW_DEPENDENCY_SOURCE "BUNDLED" CACHE STRING "override arrow's dependency source" FORCE) + # if (PSP_ENABLE_WASM) + # set(ARROW_CXX_FLAGS_RELEASE " -msimd128 -mbulk-memory -mrelaxed-simd -s MEMORY64=1 " CACHE STRING "override arrow's dependency source" FORCE) + # set(ARROW_C_FLAGS_RELEASE " -msimd128 -mbulk-memory -mrelaxed-simd -s MEMORY64=1 " CACHE STRING "override arrow's dependency source" FORCE ) + # endif() if(PSP_WASM_BUILD) set(ARROW_ENABLE_THREADING OFF) else() @@ -79,6 +81,8 @@ function(psp_build_dep name cmake_file) EXCLUDE_FROM_ALL) set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src/src PARENT_SCOPE) elseif(${name} STREQUAL rapidjson) + set(RAPIDJSON_SSE42 ON CACHE BOOL "Enable RapidJSON SIMD") + set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests") add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src ${CMAKE_BINARY_DIR}/${name}-build EXCLUDE_FROM_ALL) diff --git a/cmake/rapidjson.txt.in b/cmake/rapidjson.txt.in index 3d85b06ac3..18ef8c7ed1 100644 --- a/cmake/rapidjson.txt.in +++ b/cmake/rapidjson.txt.in @@ -5,7 +5,7 @@ project(rapidjson-download NONE) include(ExternalProject) ExternalProject_Add(rapidjson GIT_REPOSITORY https://github.com/tencent/rapidjson.git - GIT_TAG v1.1.0 + GIT_TAG d621dc9e9c77f81e5c8a35b8dcc16dcd63351321 SOURCE_DIR "${CMAKE_BINARY_DIR}/rapidjson-src" BINARY_DIR "${CMAKE_BINARY_DIR}/rapidjson-build" CONFIGURE_COMMAND "" diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index 1ca3aac88e..a2ae272b0c 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -29,6 +29,8 @@ endif() set(CMAKE_MODULE_PATH "${PSP_CMAKE_MODULE_PATH}/modules" ${CMAKE_MODULE_PATH}) set(MSVC_RUNTIME_LIBRARY MultiThreaded) +option(PSP_WASM64 "Enable WASM Memory64 support" OFF) + if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(WIN32 ON) set(MACOS OFF) @@ -229,11 +231,13 @@ if(PSP_WASM_BUILD) ") endif() else() - # Pyodide RELEASE block - set(OPT_FLAGS " \ - -O3 \ - -g0 \ + set(OPT_FLAGS " -O3 -g0 ") + if(NOT PSP_PYODIDE) + set(OPT_FLAGS " \ + ${OPT_FLAGS} \ ") + endif() + if (PSP_WASM_EXCEPTIONS) set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ") endif() @@ -299,15 +303,16 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) endif() endif() -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \ --O3 \ -") if (PSP_WASM_EXCEPTIONS) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \ + -O3 \ + -g0 \ + ") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ - -fwasm-exceptions \ - -O3 \ - -g0 \ + -fwasm-exceptions \ + -O3 \ + -g0 \ ") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ @@ -315,11 +320,26 @@ else() ") endif() +if (PSP_WASM64) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \ + -mbulk-memory \ + -msimd128 \ + -mrelaxed-simd \ + -s MEMORY64=1 \ + ") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \ + -mbulk-memory \ + -msimd128 \ + -mrelaxed-simd \ + -s MEMORY64=1 \ + ") +endif() + set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests") if(PSP_PYODIDE) - set(RELOCATABLE_FLAGS "-sRELOCATABLE=1 -sSIDE_MODULE=2 -sWASM_BIGINT=1") - string(APPEND CMAKE_EXE_LINKER_FLAGS "${RELOCATABLE_FLAGS}") + set(RELOCATABLE_FLAGS " -sRELOCATABLE=1 -sSIDE_MODULE=2 ") + string(APPEND CMAKE_EXE_LINKER_FLAGS "${RELOCATABLE_FLAGS} -sWASM_BIGINT=1 ") string(APPEND CMAKE_C_FLAGS "${RELOCATABLE_FLAGS}") string(APPEND CMAKE_CXX_FLAGS "${RELOCATABLE_FLAGS}") endif() @@ -490,10 +510,12 @@ endif() if(PSP_PYODIDE) set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ --no-entry \ - -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \ + -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server,_psp_is_memory64 \ -s SIDE_MODULE=2 \ ") else() +# -s MEMORY_GROWTH_GEOMETRIC_STEP=1.0 \ +# -s MEMORY_GROWTH_GEOMETRIC_CAP=536870912 \ set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ --no-entry \ --closure=1 \ @@ -505,16 +527,26 @@ else() -s TEXTDECODER=2 \ -s STANDALONE_WASM=1 \ -s DYNAMIC_EXECUTION=0 \ - -s BINARYEN_EXTRA_PASSES=--one-caller-inline-max-function-size=19306 \ + -s POLYFILL=0 \ -s EXPORT_NAME=\"load_perspective\" \ - -s MAXIMUM_MEMORY=4gb \ + -s MAXIMUM_MEMORY=16gb \ -s ERROR_ON_UNDEFINED_SYMBOLS=0 \ -s NODEJS_CATCH_EXIT=0 \ -s NODEJS_CATCH_REJECTION=0 \ -s USE_ES6_IMPORT_META=1 \ -s EXPORT_ES6=1 \ - -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \ + -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server,_psp_is_memory64 \ ") + + if(PSP_WASM64) + set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \ + -s MAXIMUM_MEMORY=16gb \ + ") + else() + set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \ + -s MAXIMUM_MEMORY=4gb \ + ") + endif() endif() if (PSP_WASM_EXCEPTIONS) diff --git a/cpp/perspective/patches/arrow_strptime.patch b/cpp/perspective/patches/arrow_strptime.patch index 8a09ad4a1a..5949ebcc8e 100644 --- a/cpp/perspective/patches/arrow_strptime.patch +++ b/cpp/perspective/patches/arrow_strptime.patch @@ -1,3 +1,19 @@ +diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc +index c8457eae8..aa2b41cc9 100644 +--- a/cpp/src/arrow/util/decimal.cc ++++ b/cpp/src/arrow/util/decimal.cc +@@ -111,7 +111,11 @@ struct DecimalRealConversion : public BaseDecimalRealConversion { + + // 2. Losslessly convert `real` to `mant * 2**k` + int binary_exp = 0; ++#ifdef __EMSCRIPTEN__ ++ const Real real_mant = std::frexpf(real, &binary_exp); ++#else + const Real real_mant = std::frexp(real, &binary_exp); ++#endif + // `real_mant` is within 0.5 and 1 and has M bits of precision. + // Multiply it by 2^M to get an exact integer. + const uint64_t mant = static_cast(std::ldexp(real_mant, kMantissaBits)); diff --git a/cpp/src/arrow/util/value_parsing.h b/cpp/src/arrow/util/value_parsing.h index 609906052..1e3dfae7c 100644 --- a/cpp/src/arrow/util/value_parsing.h @@ -11,3 +27,27 @@ index 609906052..1e3dfae7c 100644 char* ret = arrow_strptime(clean_copy.c_str(), format, &result); #else char* ret = strptime(clean_copy.c_str(), format, &result); +diff --git a/cpp/src/arrow/vendored/xxhash/xxhash.h b/cpp/src/arrow/vendored/xxhash/xxhash.h +index a18e8c762..235590b19 100644 +--- a/cpp/src/arrow/vendored/xxhash/xxhash.h ++++ b/cpp/src/arrow/vendored/xxhash/xxhash.h +@@ -169,6 +169,11 @@ + * xxHash prototypes and implementation + */ + ++ ++#ifdef __EMSCRIPTEN__ ++#include ++#endif ++ + #if defined (__cplusplus) + extern "C" { + #endif +@@ -3422,6 +3427,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can + # endif + #endif + ++ + #if defined(__GNUC__) || defined(__clang__) + # if defined(__ARM_FEATURE_SVE) + # include diff --git a/cpp/perspective/src/cpp/binding_api.cpp b/cpp/perspective/src/cpp/binding_api.cpp index 709d2d0f6c..46f791363c 100644 --- a/cpp/perspective/src/cpp/binding_api.cpp +++ b/cpp/perspective/src/cpp/binding_api.cpp @@ -119,4 +119,14 @@ psp_delete_server(void* proto_server) { delete server; } +PERSPECTIVE_EXPORT +bool +psp_is_memory64() { +#if UINTPTR_MAX == 0xffffffff + return false; +#elif UINTPTR_MAX == 0xffffffffffffffff + return true; +#endif +} + } // end extern "C" diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp index e70f42e961..f0c26b8ff0 100644 --- a/cpp/perspective/src/cpp/computed_function.cpp +++ b/cpp/perspective/src/cpp/computed_function.cpp @@ -1233,9 +1233,9 @@ bucket::operator()(t_parameter_list parameters) { } std::string allowed_units = "smhDWMY"; if (allowed_units.find(temp_unit) == std::string::npos) { - std::cerr << "[bucket] unknown unit in bucket - the valid units " - "are 's', 'm', 'h', 'D', 'W', 'M', and 'Y'." - << '\n'; + // std::cerr << "[bucket] unknown unit in bucket - the valid units " + // "are 's', 'm', 'h', 'D', 'W', 'M', and 'Y'." + // << '\n'; rval.m_type = DTYPE_TIME; rval.m_status = STATUS_CLEAR; return rval; diff --git a/cpp/perspective/src/cpp/gnode.cpp b/cpp/perspective/src/cpp/gnode.cpp index 3113cbddac..e1292e4bad 100644 --- a/cpp/perspective/src/cpp/gnode.cpp +++ b/cpp/perspective/src/cpp/gnode.cpp @@ -997,7 +997,7 @@ t_gnode::get_registered_contexts() const { void t_gnode::_register_context( - const std::string& name, t_ctx_type type, std::int64_t ptr + const std::string& name, t_ctx_type type, std::uintptr_t ptr ) { PSP_TRACE_SENTINEL(); PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); diff --git a/cpp/perspective/src/cpp/pool.cpp b/cpp/perspective/src/cpp/pool.cpp index 80c56d9d0a..914b1b3c81 100644 --- a/cpp/perspective/src/cpp/pool.cpp +++ b/cpp/perspective/src/cpp/pool.cpp @@ -10,6 +10,7 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +#include #include #include #include @@ -182,39 +183,21 @@ t_pool::get_trees() { return rval; } -#if defined PSP_ENABLE_WASM and !defined(PSP_ENABLE_PYTHON) void t_pool::register_context( t_uindex gnode_id, const std::string& name, t_ctx_type type, - std::int32_t ptr -) { - - if (!validate_gnode_id(gnode_id)) { - return; - } - m_gnodes[gnode_id]->_register_context(name, type, ptr); -} - -#else -void -t_pool::register_context( - t_uindex gnode_id, - const std::string& name, - t_ctx_type type, - std::int64_t ptr + std::uintptr_t ptr ) { #ifdef PSP_PARALLEL_FOR PSP_WRITE_LOCK(*m_lock) #endif - if (!validate_gnode_id(gnode_id)) { return; } m_gnodes[gnode_id]->_register_context(name, type, ptr); } -#endif void t_pool::notify_userspace(t_uindex port_id) { diff --git a/cpp/perspective/src/cpp/server.cpp b/cpp/perspective/src/cpp/server.cpp index 899a16add9..5d19e9fa39 100644 --- a/cpp/perspective/src/cpp/server.cpp +++ b/cpp/perspective/src/cpp/server.cpp @@ -38,6 +38,10 @@ #include #include +#if !defined(WIN32) && !defined(PSP_ENABLE_WASM) +#include +#endif + namespace perspective { std::uint32_t server::ProtoServer::m_client_id = 1; @@ -2414,7 +2418,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) { #if defined(PSP_ENABLE_WASM) && !defined(PSP_ENABLE_PYTHON) auto heap_size = psp_heap_size(); sys_info->set_heap_size(heap_size); -#elif !defined(WIN32) +#elif !defined(WIN32) && !defined(PSP_ENABLE_WASM) rusage out; getrusage(RUSAGE_SELF, &out); sys_info->set_heap_size(out.ru_maxrss); diff --git a/cpp/perspective/src/include/perspective/gnode.h b/cpp/perspective/src/include/perspective/gnode.h index 2206e85bb5..5db5910a20 100644 --- a/cpp/perspective/src/include/perspective/gnode.h +++ b/cpp/perspective/src/include/perspective/gnode.h @@ -143,7 +143,7 @@ class PERSPECTIVE_EXPORT t_gnode { * @param ptr */ void _register_context( - const std::string& name, t_ctx_type type, std::int64_t ptr + const std::string& name, t_ctx_type type, std::uintptr_t ptr ); /** diff --git a/cpp/perspective/src/include/perspective/pool.h b/cpp/perspective/src/include/perspective/pool.h index e2c5db30aa..209857d180 100644 --- a/cpp/perspective/src/include/perspective/pool.h +++ b/cpp/perspective/src/include/perspective/pool.h @@ -48,21 +48,12 @@ class PERSPECTIVE_EXPORT t_pool { // void set_update_delegate(t_val ud); // #endif -#if defined PSP_ENABLE_WASM and !defined PSP_ENABLE_PYTHON void register_context( t_uindex gnode_id, const std::string& name, t_ctx_type type, - std::int32_t ptr + std::uintptr_t ptr ); -#else - void register_context( - t_uindex gnode_id, - const std::string& name, - t_ctx_type type, - std::int64_t ptr - ); -#endif #ifdef PSP_PARALLEL_FOR std::shared_mutex* get_lock() const; diff --git a/examples/blocks/package.json b/examples/blocks/package.json index 6ac0575e93..96d70a35c4 100644 --- a/examples/blocks/package.json +++ b/examples/blocks/package.json @@ -4,7 +4,7 @@ "version": "3.2.1", "description": "A collection of simple client-side Perspective examples for `http://bl.ocks.org`.", "scripts": { - "start": "mkdirp dist && node --experimental-modules server.mjs", + "start": "mkdirp dist && node --experimental-wasm-memory64 --experimental-modules server.mjs", "repl": "node --experimental-repl-await" }, "main": "index.mjs", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44442d99ac..f68bdb5535 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -971,6 +971,12 @@ importers: perspective-3-0-0: specifier: npm:@finos/perspective@3.0.0 version: '@finos/perspective@3.0.0' + perspective-3-1-0: + specifier: npm:@finos/perspective@3.1.0 + version: '@finos/perspective@3.1.0' + perspective-3-2-0: + specifier: npm:@finos/perspective@3.2.0 + version: '@finos/perspective@3.2.0' devDependencies: '@finos/perspective': specifier: workspace:^ @@ -2288,6 +2294,12 @@ packages: '@finos/perspective@3.0.0': resolution: {integrity: sha512-myWGlfxN97vV3qvv34QAxCH389Oo2SH0EGN4eyrGdZSb+UlFjv9HR1TYM18GRhdM1ASg7jFwnIvGDcw7oM+qNA==} + '@finos/perspective@3.1.0': + resolution: {integrity: sha512-nyO4NWU1+C4C0SOYDdHM7p01uPl9eeJUYMG+YNy+LNOg5xGSpNZ4CI4iT9+1N7mSKLl9mrly8OAoRQXEHoDKTA==} + + '@finos/perspective@3.2.0': + resolution: {integrity: sha512-shV80NUE44V6AX2jhEAD4XS4uhMyv5A22SJ9kbop/ZKAC3Z9rbrmJffQqymZCgM2457fkjy/GxIcwcDur7GgFQ==} + '@fontsource/roboto-mono@4.5.10': resolution: {integrity: sha512-KrJdmkqz6DszT2wV/bbhXef4r0hV3B0vw2mAqei8A2kRnvq+gcJLmmIeQ94vu9VEXrUQzos5M9lH1TAAXpRphw==} @@ -10951,6 +10963,22 @@ snapshots: - bufferutil - utf-8-validate + '@finos/perspective@3.1.0': + dependencies: + stoppable: 1.1.0 + ws: 8.18.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@finos/perspective@3.2.0': + dependencies: + stoppable: 1.1.0 + ws: 8.18.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@fontsource/roboto-mono@4.5.10': {} '@fortawesome/fontawesome-free@5.15.4': {} diff --git a/rust/bootstrap-runtime/Cargo.toml b/rust/bootstrap-runtime/Cargo.toml index 099c84a451..6a3be4a628 100644 --- a/rust/bootstrap-runtime/Cargo.toml +++ b/rust/bootstrap-runtime/Cargo.toml @@ -23,11 +23,16 @@ test = false [dependencies] talc = "4.4.1" -simd-adler32 = { version = "0.3.7", default-features = false } -zune-inflate = { version = "0.2", default-features = false, features = [ - "zlib", -] } + +[dependencies.zune-inflate] +version = "0.2" +default-features = false +features = ["zlib"] [features] env_target = [] default_features = [] + +# This crate needs a patch for wasm32 +[patch.crates-io] +simd-adler32 = { git = "https://github.com/mcountryman/simd-adler32.git", rev = "140cde033e8b9a12d4de840648c65ccd5320bcc5" } diff --git a/rust/bootstrap/main.rs b/rust/bootstrap/main.rs index 05be36c9d0..c07ee824db 100644 --- a/rust/bootstrap/main.rs +++ b/rust/bootstrap/main.rs @@ -71,7 +71,9 @@ fn main() { .join("perspective_bootstrap_runtime.wasm"); OptimizationOptions::new_optimize_for_size() - .one_caller_inline_max_size(19306) + // .one_caller_inline_max_size(19306) + .enable_feature(wasm_opt::Feature::BulkMemory) + .enable_feature(wasm_opt::Feature::Simd) .run(inpath.clone(), args.output.unwrap_or(args.input)) .unwrap(); } diff --git a/rust/bundle/main.rs b/rust/bundle/main.rs index 770338d01f..86c81d7922 100644 --- a/rust/bundle/main.rs +++ b/rust/bundle/main.rs @@ -30,8 +30,8 @@ struct BundleArgs { features: Option, } -use wasm_bindgen_cli_support::Bindgen; -use wasm_opt::OptimizationOptions; +use wasm_bindgen_cli_support::{Bindgen, EncodeInto}; +use wasm_opt::{Feature, OptimizationOptions}; /// Run the packages `build` task with the appropriate flags. These can't be /// defined in the `/.cargo/config.toml` because they would define this build @@ -69,7 +69,9 @@ fn bindgen(outdir: &Path, artifact: &str, is_release: bool) { .unwrap() .keep_debug(!is_release) .input_path(input) + .encode_into(EncodeInto::Always) .typescript(true) + .reference_types(true) .out_name(&format!("{}.wasm", artifact.replace('_', "-"))) .generate(outdir) .unwrap(); @@ -79,7 +81,10 @@ fn bindgen(outdir: &Path, artifact: &str, is_release: bool) { fn opt(outpath: &Path, is_release: bool) { if is_release { OptimizationOptions::new_optimize_for_size_aggressively() - .one_caller_inline_max_size(19306) + .enable_feature(Feature::BulkMemory) + .enable_feature(Feature::ReferenceTypes) + .enable_feature(Feature::Simd) + .enable_feature(Feature::RelaxedSimd) .run(outpath, outpath) .unwrap(); } diff --git a/rust/perspective-js/Cargo.toml b/rust/perspective-js/Cargo.toml index 73086bcceb..b1843551bf 100644 --- a/rust/perspective-js/Cargo.toml +++ b/rust/perspective-js/Cargo.toml @@ -127,3 +127,6 @@ features = [ "VisibilityState", "Window", ] + +[patch.crates-io] +simd-adler32 = { git = "https://github.com/mcountryman/simd-adler32.git", rev = "140cde033e8b9a12d4de840648c65ccd5320bcc5" } diff --git a/rust/perspective-js/src/ts/emscripten_api.ts b/rust/perspective-js/src/ts/emscripten_api.ts index 877a695295..2b79939279 100644 --- a/rust/perspective-js/src/ts/emscripten_api.ts +++ b/rust/perspective-js/src/ts/emscripten_api.ts @@ -16,6 +16,8 @@ import { load_wasm_stage_0 } from "./decompress.ts"; export type * from "../../dist/pkg/perspective-js.d.ts"; +export type PspPtr = bigint | number; + export interface EmscriptenServer {} export interface EmscriptenApi { HEAP8: Int8Array; @@ -24,17 +26,19 @@ export interface EmscriptenApi { HEAPU16: Uint16Array; HEAP32: Int32Array; HEAPU32: Uint32Array; - _psp_alloc(size: number): number; - _psp_free(ptr: number): void; + HEAPU64: BigUint64Array; + _psp_alloc(size: PspPtr): PspPtr; + _psp_free(ptr: PspPtr): void; _psp_new_server(): EmscriptenServer; _psp_delete_server(server: EmscriptenServer): void; + _psp_is_memory64(): boolean; _psp_handle_request( server: EmscriptenServer, client_id: number, - buffer_ptr: number, - buffer_len: number - ): number; - _psp_poll(server: EmscriptenServer): number; + buffer_ptr: PspPtr, + buffer_len: PspPtr + ): PspPtr; + _psp_poll(server: EmscriptenServer): PspPtr; _psp_new_session(server: EmscriptenServer): number; _psp_close_session(server: EmscriptenServer, client_id: number): void; } @@ -42,7 +46,7 @@ export interface EmscriptenApi { export async function compile_perspective( compressed_binary: ArrayBuffer ): Promise { - const module = await perspective_server.default({ + const module: EmscriptenApi = await perspective_server.default({ locateFile(x: any) { return x; }, @@ -57,9 +61,14 @@ export async function compile_perspective( const str = Error().stack || ""; const textEncoder = new TextEncoder(); const bytes = textEncoder.encode(str); - const ptr = module._psp_alloc(bytes.byteLength + 1); - module.HEAPU8.set(bytes, ptr); - module.HEAPU8[ptr + bytes.byteLength] = 0; + const ptr = module._psp_alloc( + module._psp_is_memory64() + ? BigInt(bytes.byteLength + 1) + : bytes.byteLength + 1 + ); + + module.HEAPU8.set(bytes, Number(ptr)); + module.HEAPU8[Number(ptr) + bytes.byteLength] = 0; return ptr; }, psp_heap_size() { diff --git a/rust/perspective-js/src/ts/engine.ts b/rust/perspective-js/src/ts/engine.ts index 01aa61c2ac..5df38c9090 100644 --- a/rust/perspective-js/src/ts/engine.ts +++ b/rust/perspective-js/src/ts/engine.ts @@ -10,7 +10,7 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -import { EmscriptenApi, EmscriptenServer } from "./emscripten_api.ts"; +import { EmscriptenApi, EmscriptenServer, PspPtr } from "./emscripten_api.ts"; export type ApiResponse = { client_id: number; @@ -67,7 +67,9 @@ export class PerspectiveSession { this.server, this.client_id, viewPtr, - view.byteLength + this.mod._psp_is_memory64() + ? BigInt(view.byteLength) + : view.byteLength ); } ); @@ -92,21 +94,18 @@ export class PerspectiveSession { async function convert_typed_array_to_pointer( core: EmscriptenApi, array: Uint8Array, - callback: (_: number) => Promise -): Promise { - const ptr = core._psp_alloc(array.byteLength); - core.HEAPU8.set(array, ptr); + callback: (_: PspPtr) => Promise +): Promise { + const ptr = core._psp_alloc( + core._psp_is_memory64() ? BigInt(array.byteLength) : array.byteLength + ); + + core.HEAPU8.set(array, Number(ptr)); const msg = await callback(ptr); core._psp_free(ptr); return msg; } -function convert_pointer_to_u32_array(core: EmscriptenApi, ptr: number) { - const len = core.HEAPU32[ptr >>> 2]; - const data_ptr = core.HEAPU32[(ptr >>> 2) + 1]; - return new Uint32Array(core.HEAPU8.buffer, data_ptr, len * 3); -} - /** * Convert a pointer to WASM memory into an `ApiResponse[]`, via a custom * encoding. @@ -114,7 +113,7 @@ function convert_pointer_to_u32_array(core: EmscriptenApi, ptr: number) { * @param core The emscripten API * @param ptr A pointer to a fixed-sized struct representing a set of * `proto::Resp` payloads, encoded as a length-prefixed array of - * (char* data, size_t len, size_t client_id) tuples: + * (char* data, u32_t len, u32_t client_id) tuples: * * ```text * N data length client_id data length client_id @@ -133,26 +132,64 @@ function convert_pointer_to_u32_array(core: EmscriptenApi, ptr: number) { */ async function decode_api_responses( core: EmscriptenApi, - ptr: number, + ptr: PspPtr, callback: (_: ApiResponse) => Promise ) { - const responses = convert_pointer_to_u32_array(core, ptr); + const is_64 = core._psp_is_memory64(); + const response = new DataView( + core.HEAPU8.buffer, + Number(ptr), + is_64 ? 12 : 8 + ); + + const num_msgs = response.getUint32(0, true); + const msgs_ptr = is_64 + ? response.getBigInt64(4, true) + : response.getUint32(4, true); + + const messages = new DataView( + core.HEAPU8.buffer, + Number(msgs_ptr), + num_msgs * (is_64 ? 16 : 12) + ); + try { - for (let i = 0; i < responses.length / 3; i++) { - const data_ptr = responses[i * 3]; - const length = responses[i * 3 + 1]; - const client_id = responses[i * 3 + 2]; - const data = new Uint8Array(core.HEAPU8.buffer, data_ptr, length); + for (let i = 0; i < num_msgs; i++) { + const [data_ptr, data_len, client_id] = is_64 + ? [ + messages.getBigInt64(i * 16, true), + messages.getInt32(i * 16 + 8, true), + messages.getInt32(i * 16 + 12, true), + ] + : [ + messages.getInt32(i * 12, true), + messages.getInt32(i * 12 + 4, true), + messages.getInt32(i * 12 + 8, true), + ]; + + const data = new Uint8Array( + core.HEAPU8.buffer, + Number(data_ptr), + data_len + ); + const resp = { client_id, data }; await callback(resp); } } finally { - for (let i = 0; i < responses.length / 3; i++) { - const data_ptr = responses[i * 3]; + for (let i = 0; i < num_msgs; i++) { + const data_ptr = is_64 + ? messages.getBigInt64(i * 16, true) + : messages.getInt32(i * 12, true); + core._psp_free(data_ptr); } - core._psp_free(responses.byteOffset); - core._psp_free(ptr); + core._psp_free( + is_64 ? BigInt(messages.byteOffset) : messages.byteOffset + ); + core._psp_free( + is_64 ? BigInt(response.byteOffset) : response.byteOffset + ); } } diff --git a/rust/perspective-js/test/js/constructors.spec.js b/rust/perspective-js/test/js/constructors.spec.js index 93a0036a4e..ab1c590640 100644 --- a/rust/perspective-js/test/js/constructors.spec.js +++ b/rust/perspective-js/test/js/constructors.spec.js @@ -149,7 +149,7 @@ let arrow_lists_data = { ], float_arr: [ "[]", - "[3.14,3.141592653589793,6.283185307179586,1.4142135623730952]", + "[3.14,3.141592653589793,6.283185307179586,1.4142135623730951]", "[12.00001]", "[100000000.0,100000000.0,100000000.0,100000000.0]", ], diff --git a/rust/perspective-js/test/js/expressions/conversions.spec.js b/rust/perspective-js/test/js/expressions/conversions.spec.js index 06a72a43cd..3ea3af8c29 100644 --- a/rust/perspective-js/test/js/expressions/conversions.spec.js +++ b/rust/perspective-js/test/js/expressions/conversions.spec.js @@ -561,7 +561,7 @@ import perspective from "../perspective_client"; expect(await view.to_columns()).toEqual({ x: ["abcd", "", null, "abc"], - 'boolean("x")': [true, false, false, true], + 'boolean("x")': [true, true, false, true], }); await view.delete(); diff --git a/tools/perspective-bench/package.json b/tools/perspective-bench/package.json index 1badf7be89..cf0242cda1 100644 --- a/tools/perspective-bench/package.json +++ b/tools/perspective-bench/package.json @@ -14,8 +14,8 @@ "url": "https://github.com/finos/perspective/packages/perspective-bench" }, "scripts": { - "bench_js": "node basic_suite.mjs", - "bench_python": "node python_suite.mjs" + "bench_js": "node --experimental-wasm-memory64 basic_suite.mjs", + "bench_python": "node --experimental-wasm-memory64 python_suite.mjs" }, "author": "", "license": "Apache-2.0", diff --git a/tools/perspective-bench/src/js/benchmark.mjs b/tools/perspective-bench/src/js/benchmark.mjs index 09a7f0ad00..7d83c3f5cb 100644 --- a/tools/perspective-bench/src/js/benchmark.mjs +++ b/tools/perspective-bench/src/js/benchmark.mjs @@ -160,7 +160,7 @@ async function benchmark_node_version(version, benchmarks_table) { const suite_path = path.join(process.argv[1]); let stats = []; const worker = cp.fork(suite_path, { - execArgv: ["--expose-gc"], + execArgv: ["--expose-gc", "--experimental-wasm-memory64"], env: { BENCH_FLAG: "1" }, });