Skip to content

Commit fc554a9

Browse files
texodustexodus
authored
Merge pull request #2739 from tomjakubowski/tom-pyodide-exceptions
Emscripten wheel fixes
2 parents 4e03507 + 45c8af0 commit fc554a9

File tree

34 files changed

+508
-108
lines changed

34 files changed

+508
-108
lines changed

‎.github/actions/install-deps/action.yaml

+16-1
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,7 @@ runs:
200200
- name: manylinux deps
201201
shell: bash
202202
run: |
203-
if [ -x "$(command -v dnf)" ]; then
203+
if [ -x "$(command -v dnf)" ]; then
204204
dnf install wget -y
205205
fi
206206
if: ${{ runner.os == 'Linux' && inputs.cpp == 'true' && inputs.javascript == 'false' }}
@@ -216,6 +216,21 @@ runs:
216216
run: sudo node tools/perspective-scripts/install_tools.mjs
217217
if: ${{ runner.os != 'Windows' && inputs.cpp == 'true' && inputs.manylinux == 'false' }}
218218

219+
- name: Install Python Pyodide dependencies
220+
shell: bash
221+
run: python -m pip install -r rust/perspective-python/requirements-pyodide.txt
222+
if: ${{ inputs.pyodide == 'true' }}
223+
224+
- name: Install Pyodide distribution
225+
shell: bash
226+
run: pnpm run install_pyodide
227+
if: ${{ inputs.pyodide == 'true' }}
228+
229+
- name: Install Python Playwright browsers
230+
shell: bash
231+
run: python -m playwright install
232+
if: ${{ inputs.pyodide == 'true' }}
233+
219234
# - name: Install CCache
220235
# shell: bash
221236
# run: sudo apt install -y ccache

‎.github/workflows/build.yaml

+9-3
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ jobs:
290290
# `-'
291291
build_emscripten_wheel:
292292
runs-on: ${{ matrix.os }}
293-
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
293+
# if: ${{ startsWith(github.ref, 'refs/tags/v') }}
294294
strategy:
295295
fail-fast: false
296296
matrix:
@@ -313,21 +313,27 @@ jobs:
313313
id: init-step
314314
uses: ./.github/actions/install-deps
315315
with:
316+
pyodide: "true"
316317
javascript: "false"
317318
arch: ${{ matrix.arch }}
318319
manylinux: "false"
319320
skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }}
320321

321322
- name: Python Build Pyodide
322323
run: pnpm install && pnpm run build
323-
if: ${{ runner.os == 'Linux' && matrix.arch == 'x86_64' }}
324+
env:
325+
PSP_PYODIDE: 1
326+
PACKAGE: "perspective-python"
327+
CI: 1
328+
329+
- name: "Test Pyodide"
330+
run: pnpm run test
324331
env:
325332
PSP_PYODIDE: 1
326333
PACKAGE: "perspective-python"
327334
CI: 1
328335

329336
- uses: actions/upload-artifact@v4
330-
# if: ${{ runner.os != 'Windows' }}
331337
with:
332338
name: perspective-python-dist-wasm32-emscripten-${{ matrix.python-version }}
333339
path: rust/target/wheels/*.whl

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,7 @@ target.vscode
126126
!.vscode/extensions.json
127127
!.vscode/settings.default.json
128128
!.vscode/tasks.json
129+
!.vscode/perspective.code-snippets
129130

130131
.llvm
131132

‎.vscode/perspective.code-snippets

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"perspective authors copyright notice: #-comment": {
3+
"scope": "python,toml,yaml",
4+
"body": [
5+
"# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓",
6+
"# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃",
7+
"# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃",
8+
"# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃",
9+
"# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃",
10+
"# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫",
11+
"# ┃ Copyright (c) 2017, the Perspective Authors. ┃",
12+
"# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃",
13+
"# ┃ This file is part of the Perspective library, distributed under the terms ┃",
14+
"# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃",
15+
"# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛"
16+
],
17+
"prefix": "# copyright"
18+
},
19+
"perspective authors copyright notice: //-comment": {
20+
"scope": "javascript,cpp,c,typescript,less,rust,proto3",
21+
"body": [
22+
"// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓",
23+
"// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃",
24+
"// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃",
25+
"// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃",
26+
"// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃",
27+
"// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫",
28+
"// ┃ Copyright (c) 2017, the Perspective Authors. ┃",
29+
"// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃",
30+
"// ┃ This file is part of the Perspective library, distributed under the terms ┃",
31+
"// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃",
32+
"// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛"
33+
],
34+
"prefix": "// copyright"
35+
},
36+
// Place your perspective workspace snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and
37+
// description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope
38+
// is left empty or omitted, the snippet gets applied to all languages. The prefix is what is
39+
// used to trigger the snippet and the body will be expanded and inserted. Possible variables are:
40+
// $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders.
41+
// Placeholders with the same ids are connected.
42+
// Example:
43+
// "Print to console": {
44+
// "scope": "javascript,typescript",
45+
// "prefix": "log",
46+
// "body": [
47+
// "console.log('$1');",
48+
// "$2"
49+
// ],
50+
// "description": "Log output to console"
51+
// }
52+
}

‎cpp/perspective/CMakeLists.txt

+11-68
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ include(CheckCCompilerFlag)
55
set(CMAKE_BUILD_TYPE "Release")
66
set(CMAKE_CXX_STANDARD 17)
77
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
8-
9-
108
# CMAKE POLICIES
119
# option() should use new cmake behavior wrt variable clobbering
1210
cmake_policy(SET CMP0077 NEW)
@@ -28,9 +26,6 @@ if(NOT DEFINED PSP_CMAKE_MODULE_PATH)
2826
set(PSP_CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake")
2927
endif()
3028

31-
32-
33-
3429
set(CMAKE_MODULE_PATH "${PSP_CMAKE_MODULE_PATH}/modules" ${CMAKE_MODULE_PATH})
3530
set(MSVC_RUNTIME_LIBRARY MultiThreaded)
3631

@@ -195,38 +190,15 @@ if(PSP_PYTHON_BUILD AND MACOS)
195190
# don't link against build python
196191
# https://blog.tim-smith.us/2015/09/python-extension-modules-os-x/
197192
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup")
198-
199-
# # TODO This now needs to be set externally because we cross-compile.
200-
# # check_c_compiler_flag("-arch x86_64" x86_64Supported)
201-
# check_c_compiler_flag("-arch arm64" arm64Supported)
202-
203-
# if(x86_64Supported AND arm64Supported)
204-
# set(CMAKE_OSX_ARCHITECTURES "x86_64;arm64" CACHE STRING "Build universal architecture for OSX" FORCE)
205-
# elseif(x86_64Supported)
206-
# set(CMAKE_REQUIRED_LINK_OPTIONS "-arch;x86_64")
207-
# set(CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build x86_64 architecture for OSX" FORCE)
208-
# elseif(arm64Supported)
209-
# set(CMAKE_REQUIRED_LINK_OPTIONS "-arch;arm64")
210-
# set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Build arm64 architecture for OSX" FORCE)
211-
# endif()
212193
endif()
213194

214195
# ######################
215196
include_directories("${CMAKE_CURRENT_SOURCE_DIR}/src/include")
216197

217-
# Needs to be set early so that all translation units use it.
218-
if (NOT PSP_PYODIDE)
219-
set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} -fexceptions")
220-
endif()
221-
222198
if(NOT DEFINED PSP_WASM_EXCEPTIONS AND NOT PSP_PYTHON_BUILD)
223199
set(PSP_WASM_EXCEPTIONS ON)
224200
endif()
225201

226-
# if(NOT WIN32)
227-
# set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
228-
# set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
229-
# endif()
230202
if(PSP_WASM_BUILD)
231203
# ###################
232204
# EMSCRIPTEN BUILD #
@@ -246,6 +218,7 @@ if(PSP_WASM_BUILD)
246218
")
247219

248220
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
221+
# Pyodide DEBUG block
249222
set(OPT_FLAGS " \
250223
-O0 \
251224
-g3 \
@@ -262,6 +235,7 @@ if(PSP_WASM_BUILD)
262235
")
263236
endif()
264237
else()
238+
# Pyodide RELEASE block
265239
set(OPT_FLAGS " \
266240
-O3 \
267241
-g0 \
@@ -369,34 +343,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
369343
# so we use a custom FindPythonHeaders that is the same as the
370344
# default, but ignores when the python libraries can't be found.
371345
psp_build_message("${Red}Manylinux build has no python shared libraries${ColorReset}")
372-
# find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter)
373-
# find_package(PythonHeaders ${PSP_PYTHON_VERSION} EXACT REQUIRED)
374-
else()
375-
# psp_build_message("${Cyan}Use python shared libraries${ColorReset}")
376-
# if(PSP_PYODIDE)
377-
# find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter)
378-
# else()
379-
# find_package(Python ${PSP_PYTHON_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development.Module)
380-
# endif()
381-
382-
# link_directories(${Python_LIBRARY_DIRS})
383346
endif()
384-
385-
# psp_build_message("${Cyan}Using Python ${Python_VERSION}, \nPython_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}, \nPython_LIBRARIES: ${Python_LIBRARIES}, \nPython_EXECUTABLE: ${Python_EXECUTABLE} ${ColorReset}")
386-
# include_directories(${Python_INCLUDE_DIRS})
387-
388-
# psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in")
389-
390-
# find_package(NumPy REQUIRED)
391-
392-
# if(NOT PYTHON_NUMPY_FOUND)
393-
# message(FATAL_ERROR "${Red}Numpy could not be located${ColorReset}")
394-
# else()
395-
# psp_build_message("${Cyan}Numpy found: ${PYTHON_NUMPY_INCLUDE_DIR}${ColorReset}")
396-
# include_directories(${PYTHON_NUMPY_INCLUDE_DIR})
397-
# endif()
398-
399-
# ####################
400347
endif()
401348
endif()
402349

@@ -674,34 +621,30 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
674621
# Python extra targets #
675622
# #######################
676623
if(PSP_WASM_BUILD)
624+
# Pyodide
677625
set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
678626
set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ")
679627
add_library(psp STATIC ${PYTHON_SOURCE_FILES})
680628
target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_ENABLE_WASM=1)
629+
# support for emscripten exceptions https://emscripten.org/docs/porting/exceptions.html#emscripten-javascript-based-exception-support
630+
target_compile_options(psp PUBLIC -fexceptions -fvisibility=hidden)
631+
target_compile_options(arrow PUBLIC -fexceptions -fvisibility=hidden)
632+
target_compile_options(re2 PUBLIC -fexceptions -fvisibility=hidden)
633+
target_compile_options(protos PUBLIC -fexceptions -fvisibility=hidden)
681634
else()
635+
# Cpython
682636
add_library(psp STATIC ${PYTHON_SOURCE_FILES})
683637
target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1)
684638
endif()
685-
# add_library(psppy SHARED ${PYTHON_BINDING_SOURCE_FILES})
686639

687640
include_directories(${PSP_PYTHON_SRC}/include)
688641

689-
# target_compile_definitions(psppy PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1)
690-
691-
if(WIN32)
692-
#target_compile_definitions(psppy PRIVATE WIN32=1)
693-
# target_compile_definitions(psppy PRIVATE _WIN32=1)
694-
695-
# .dll not importable
696-
# set_property(TARGET psppy PROPERTY SUFFIX .pyd)
697-
elseif(MACOS OR NOT MANYLINUX)
698-
# target_compile_options(psppy PRIVATE -Wdeprecated-declarations)
642+
if(MACOS OR NOT MANYLINUX)
699643
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
700-
# set_property(TARGET psppy PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
701644

702645
target_compile_options(psp PRIVATE -fvisibility=hidden)
703-
# target_compile_options(psppy PRIVATE -fvisibility=hidden)
704646
elseif(MANYLINUX)
647+
# intentionally blank
705648
else()
706649
target_compile_options(psp PRIVATE -fvisibility=hidden)
707650
# target_compile_options(psppy PRIVATE -Wdeprecated-declarations)

‎cpp/perspective/build.js

+26-8
Original file line numberDiff line numberDiff line change
@@ -28,18 +28,36 @@ function bootstrap(file) {
2828
});
2929
}
3030

31+
let cmake_flags = "";
32+
let make_flags = "";
33+
34+
if (!!process.env.PSP_BUILD_VERBOSE) {
35+
cmake_flags += "-Wdev --debug-output ";
36+
make_flags += "VERBOSE=1 ";
37+
} else {
38+
cmake_flags = "-Wno-dev "; // suppress developer warnings
39+
}
40+
3141
try {
3242
execSync(`mkdirp ${cwd}`, { stdio });
3343
process.env.CLICOLOR_FORCE = 1;
34-
execSync(`emcmake cmake ${__dirname} -Wno-dev -DCMAKE_BUILD_TYPE=${env}`, {
35-
cwd,
36-
stdio,
37-
});
44+
execSync(
45+
`emcmake cmake ${__dirname} ${cmake_flags} -DCMAKE_BUILD_TYPE=${env}`,
46+
{
47+
cwd,
48+
stdio,
49+
}
50+
);
3851

39-
execSync(`emmake make -j${process.env.PSP_NUM_CPUS || os.cpus().length}`, {
40-
cwd,
41-
stdio,
42-
});
52+
execSync(
53+
`emmake make -j${
54+
process.env.PSP_NUM_CPUS || os.cpus().length
55+
} ${make_flags}`,
56+
{
57+
cwd,
58+
stdio,
59+
}
60+
);
4361

4462
execSync(`cpy web/**/* ../web`, { cwd, stdio });
4563
execSync(`cpy node/**/* ../node`, { cwd, stdio });

‎cpp/perspective/src/cpp/arrow_csv.cpp

+1-2
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,6 @@ class CustomISO8601Parser : public arrow::TimestampParser {
387387
if (length == 29) {
388388
// YYYY-MM-DD[ T]hh:mm:ss.sssssssss -- nanos
389389
// arrow handles YYYY-MM-DD[ T]hh:mm:ss.sss[+-]HH:MM
390-
std::cout << "DDD WOOHOOOOO!\n";
391390
arrow_vendored::date::year_month_day ymd;
392391
if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
393392
return false;
@@ -645,4 +644,4 @@ csvToTable(
645644
return *maybe_table;
646645
}
647646

648-
} // namespace perspective::apachearrow
647+
} // namespace perspective::apachearrow

‎cpp/perspective/src/cpp/server.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -2262,7 +2262,7 @@ ProtoServer::_handle_request(std::uint32_t client_id, const Request& req) {
22622262
case proto::Request::kServerSystemInfoReq: {
22632263
proto::Response resp;
22642264
auto* sys_info = resp.mutable_server_system_info_resp();
2265-
#ifdef PSP_ENABLE_WASM
2265+
#if defined(PSP_ENABLE_WASM) && !defined(PSP_ENABLE_PYTHON)
22662266
auto heap_size = psp_heap_size();
22672267
sys_info->set_heap_size(heap_size);
22682268
#else

‎cpp/perspective/src/include/perspective/base.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ std::is_pod<X>::value && std::is_standard_layout<X>::value , \
159159
#define LOG_DEBUG(X)
160160
#endif
161161

162-
#if defined(PSP_ENABLE_WASM)
162+
#if defined(PSP_ENABLE_WASM) && !defined(PSP_PYODIDE)
163163
#define ESM_EXPORT(X) __attribute__((import_module("env"), import_name(X)))
164164

165165
PERSPECTIVE_EXPORT ESM_EXPORT("psp_stack_trace") extern "C" const
@@ -170,7 +170,7 @@ PERSPECTIVE_EXPORT ESM_EXPORT("psp_heap_size") extern "C" size_t
170170

171171
#endif
172172

173-
#if defined(PSP_DEBUG) && defined(PSP_ENABLE_WASM)
173+
#if defined(PSP_DEBUG) && defined(PSP_ENABLE_WASM) && !defined(PSP_PYODIDE)
174174
#define PSP_COMPLAIN_AND_ABORT(X) \
175175
{ \
176176
std::stringstream __SS__; \

‎package.json

+2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
"type": "module",
1616
"emscripten": "3.1.58",
1717
"llvm": "17.0.6",
18+
"pyodide": "0.26.2",
1819
"engines": {
1920
"node": ">=16"
2021
},
@@ -100,6 +101,7 @@
100101
"postinstall:playwright": "npx playwright install --with-deps",
101102
"postinstall:vscode": "cp -n ./.vscode/settings.default.json ./.vscode/settings.json || true",
102103
"install_llvm": "node tools/perspective-scripts/install_llvm.mjs",
104+
"install_pyodide": "node tools/perspective-scripts/install_pyodide.mjs",
103105
"build,test": "npm run --silent build && npm run --silent test",
104106
"build_js": "node tools/perspective-scripts/build_js.mjs",
105107
"build": "node tools/perspective-scripts/build.mjs",

0 commit comments

Comments
 (0)