diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..497178ad --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,42 @@ +version: 2.1 + +parameters: + elixir-image: + type: string + default: hexpm/elixir:1.17.3-erlang-27.0.1-debian-bookworm-20241202 + ubuntu-image: + type: string + default: ubuntu:22.04 + +executors: + test-container: + docker: + - image: << pipeline.parameters.ubuntu-image >> + +commands: + code-setup: + description: "Ensures code is checked out and basic tooling is ready" + steps: + - checkout + - run: apt-get update + - run: apt-get install -y git build-essential wget curl lldb python3 python3-lldb + +jobs: + vl-convert-rs-example2: + executor: test-container + steps: + - code-setup + - run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + . "$HOME/.cargo/env" + cd vl-convert-rs + cargo build --example debug_conversion_sequence + RUST_BACKTRACE=1 lldb ./target/debug/examples/debug_conversion_sequence -o "run" -o "bt all" -o "quit" > stacktrace.txt 2>&1 + - store_artifacts: + path: vl-convert-rs/stacktrace.txt + destination: stacktrace.txt + +workflows: + test-suite: + jobs: + - vl-convert-rs-example2 \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0e6cfebe..2cbf4ae4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -7,72 +7,10 @@ on: pull_request: jobs: - python-fmt: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.28.2 - - name: Check cargo fmt compliance - run: pixi run fmt-py-check - - rust-fmt-clippy: - runs-on: ubuntu-latest - env: - RUSTFLAGS: "-D warnings" - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.28.2 - - name: Cache rust dependencies - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: True - - name: Check cargo fmt compliance - run: pixi run fmt-rs-check - - name: Check no rustc warnings - run: pixi run check-rs - - name: Check for clippy warnings - run: pixi run clippy - - cargo-bundle-license: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.28.2 - - name: Check that license is up to date - run: pixi run bundle-licenses - - name: Check that git detects no file changes - run: git diff --exit-code - - codegen-clean: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.28.2 - - name: Cache rust dependencies - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: True - - name: Run codegen - run: pixi run vendor - - name: Check that git detects no file changes - run: git diff --exit-code - # Run linux tests without Pixi due to undiagnosed linker issues # - undefined reference to fcntl64 # - undefined reference to memfd_create - vl-convert-rs-tests-linux: + vl-convert-rs-example2: runs-on: ubuntu-latest steps: - name: Check out repository code @@ -98,82 +36,175 @@ jobs: - name: Run rs tests # Run tests on single thread for Deno, which expects this run: | - cargo test -p vl-convert-rs -- --test-threads=1 - - name: Run CLI tests - run: | - cargo test -p vl-convert -- --test-threads=1 - - name: Upload test failures - uses: actions/upload-artifact@v3 - if: always() - with: - name: failed-images - path: | - vl-convert-rs/tests/vl-specs/failed + cd vl-convert-rs/examples + cargo run --example conversion_sequence - vl-convert-rs-tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: - - windows-2022 - - macos-12 - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.28.2 - - name: Cache rust dependencies - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v1-rust" - cache-on-failure: True - - name: Install fonts on Linux - if: runner.os == 'Linux' - run: | - echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections - sudo apt-get install ttf-mscorefonts-installer - - name: Run tests - run: | - pixi run test-rs - pixi run test-cli - - name: Upload test failures - uses: actions/upload-artifact@v3 - if: always() - with: - name: failed-images - path: | - vl-convert-rs/tests/vl-specs/failed + # python-fmt: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository code + # uses: actions/checkout@v2 + # - uses: prefix-dev/setup-pixi@v0.8.1 + # with: + # pixi-version: v0.28.2 + # - name: Check cargo fmt compliance + # run: pixi run fmt-py-check - vl-convert-python-tests: - runs-on: ${{ matrix.options[0] }} - defaults: - run: - shell: ${{ matrix.options[2] }} - strategy: - matrix: - options: - - [ubuntu-latest, '3.10', 'bash -l {0}'] - - [windows-2022, '3.10', "pwsh"] - - [macos-12, '3.10', 'bash -l {0}'] - steps: - - name: Check out repository code - uses: actions/checkout@v2 - - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.28.2 - - name: Cache rust dependencies - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v1-rust" - cache-on-failure: True - - name: Install fonts on Linux - if: runner.os == 'Linux' - run: | - echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections - sudo apt-get install ttf-mscorefonts-installer - - name: Build package - run: pixi run dev-py - - name: Run tests - run: pixi run test-py + # rust-fmt-clippy: + # runs-on: ubuntu-latest + # env: + # RUSTFLAGS: "-D warnings" + # steps: + # - name: Check out repository code + # uses: actions/checkout@v2 + # - uses: prefix-dev/setup-pixi@v0.8.1 + # with: + # pixi-version: v0.28.2 + # - name: Cache rust dependencies + # uses: Swatinem/rust-cache@v2 + # with: + # cache-on-failure: True + # - name: Check cargo fmt compliance + # run: pixi run fmt-rs-check + # - name: Check no rustc warnings + # run: pixi run check-rs + # - name: Check for clippy warnings + # run: pixi run clippy + + # cargo-bundle-license: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository code + # uses: actions/checkout@v2 + # - uses: prefix-dev/setup-pixi@v0.8.1 + # with: + # pixi-version: v0.28.2 + # - name: Check that license is up to date + # run: pixi run bundle-licenses + # - name: Check that git detects no file changes + # run: git diff --exit-code + + # codegen-clean: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository code + # uses: actions/checkout@v2 + # - uses: prefix-dev/setup-pixi@v0.8.1 + # with: + # pixi-version: v0.28.2 + # - name: Cache rust dependencies + # uses: Swatinem/rust-cache@v2 + # with: + # cache-on-failure: True + # - name: Run codegen + # run: pixi run vendor + # - name: Check that git detects no file changes + # run: git diff --exit-code + + # # Run linux tests without Pixi due to undiagnosed linker issues + # # - undefined reference to fcntl64 + # # - undefined reference to memfd_create + # vl-convert-rs-tests-linux: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository code + # uses: actions/checkout@v2 + # - name: Install latest stable Rust toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # - name: Cache rust dependencies + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v1-rust" + # cache-on-failure: True + # - name: Install Protoc + # uses: arduino/setup-protoc@v2 + # with: + # repo-token: ${{ secrets.GITHUB_TOKEN }} + # - name: Install fonts on Linux + # if: runner.os == 'Linux' + # run: | + # echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections + # sudo apt-get install ttf-mscorefonts-installer + # - name: Run rs tests + # # Run tests on single thread for Deno, which expects this + # run: | + # cargo test -p vl-convert-rs -- --test-threads=1 + # - name: Run CLI tests + # run: | + # cargo test -p vl-convert -- --test-threads=1 + # - name: Upload test failures + # uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: failed-images + # path: | + # vl-convert-rs/tests/vl-specs/failed + + # vl-convert-rs-tests: + # runs-on: ${{ matrix.os }} + # strategy: + # matrix: + # os: + # - windows-2022 + # - macos-12 + # steps: + # - name: Check out repository code + # uses: actions/checkout@v2 + # - uses: prefix-dev/setup-pixi@v0.8.1 + # with: + # pixi-version: v0.28.2 + # - name: Cache rust dependencies + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v1-rust" + # cache-on-failure: True + # - name: Install fonts on Linux + # if: runner.os == 'Linux' + # run: | + # echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections + # sudo apt-get install ttf-mscorefonts-installer + # - name: Run tests + # run: | + # pixi run test-rs + # pixi run test-cli + # - name: Upload test failures + # uses: actions/upload-artifact@v3 + # if: always() + # with: + # name: failed-images + # path: | + # vl-convert-rs/tests/vl-specs/failed + # vl-convert-python-tests: + # runs-on: ${{ matrix.options[0] }} + # defaults: + # run: + # shell: ${{ matrix.options[2] }} + # strategy: + # matrix: + # options: + # - [ubuntu-latest, "3.10", "bash -l {0}"] + # - [windows-2022, "3.10", "pwsh"] + # - [macos-12, "3.10", "bash -l {0}"] + # steps: + # - name: Check out repository code + # uses: actions/checkout@v2 + # - uses: prefix-dev/setup-pixi@v0.8.1 + # with: + # pixi-version: v0.28.2 + # - name: Cache rust dependencies + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v1-rust" + # cache-on-failure: True + # - name: Install fonts on Linux + # if: runner.os == 'Linux' + # run: | + # echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections + # sudo apt-get install ttf-mscorefonts-installer + # - name: Build package + # run: pixi run dev-py + # - name: Run tests + # run: pixi run test-py diff --git a/vl-convert-rs/empty_main.js b/vl-convert-rs/empty_main.js new file mode 100644 index 00000000..e69de29b diff --git a/vl-convert-rs/examples/conversion_sequence.rs b/vl-convert-rs/examples/conversion_sequence.rs new file mode 100644 index 00000000..226bb4c4 --- /dev/null +++ b/vl-convert-rs/examples/conversion_sequence.rs @@ -0,0 +1,44 @@ +use vl_convert_rs::converter::VlOpts; +use vl_convert_rs::{VlConverter, VlVersion}; + +#[tokio::main] +async fn main() { + let vl_spec: serde_json::Value = serde_json::from_str( + r#" +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "data": {"url": "data/movies.json"}, + "mark": "circle", + "encoding": { + "x": { + "bin": {"maxbins": 10}, + "field": "IMDB Rating" + }, + "y": { + "bin": {"maxbins": 10}, + "field": "Rotten Tomatoes Rating" + }, + "size": {"aggregate": "count"} + } +} "#, + ) + .unwrap(); + + println!("{}", convert(vl_spec.clone()).await); + println!("{}", convert(vl_spec.clone()).await) +} + +async fn convert(vl_spec: serde_json::Value) -> String { + let mut converter = VlConverter::new(); + + converter + .vegalite_to_svg( + vl_spec, + VlOpts { + vl_version: VlVersion::v5_8, + ..Default::default() + }, + ) + .await + .expect("Failed to perform Vega-Lite to Vega conversion") +} diff --git a/vl-convert-rs/examples/debug_conversion_sequence.rs b/vl-convert-rs/examples/debug_conversion_sequence.rs new file mode 100644 index 00000000..e37d562f --- /dev/null +++ b/vl-convert-rs/examples/debug_conversion_sequence.rs @@ -0,0 +1,162 @@ +use deno_core::error::AnyError; +use deno_runtime::deno_permissions::{Permissions, PermissionsContainer}; +use deno_runtime::worker::{MainWorker, WorkerOptions}; +use futures::channel::{mpsc, mpsc::Sender, oneshot}; +use futures_util::{SinkExt, StreamExt}; +use std::path::Path; +use std::sync::Arc; +use std::thread; +use std::thread::JoinHandle; +use tokio::io::AsyncWriteExt; +use vl_convert_rs::converter::TOKIO_RUNTIME; + +#[tokio::main(flavor = "multi_thread", worker_threads = 1)] +async fn main() { + convert3().await; + convert3().await; +} + +struct ConvertCommand { + responder: oneshot::Sender>, + shutdown: bool, +} + +pub struct Converter { + sender: Option>, + handle: Arc>, +} + +impl Converter { + pub fn new() -> Self { + let tokio_runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + + let (sender, mut receiver) = mpsc::channel::(32); + + // println!("TOKIO_RUNTIME.block_on"); + let handle = Arc::new(thread::spawn(move || { + tokio_runtime.block_on(async { + println!("in block_on"); + let mut inner = InnerConverter::new().await; + while let Some(cmd) = receiver.next().await { + println!("receiver.next()"); + if cmd.shutdown { + cmd.responder.send(Ok(())).unwrap(); + break; + } + inner.convert(); + cmd.responder.send(Ok(())).unwrap() + } + println!("Worker thread shutting down"); + }) + })); + + Self { + sender: Some(sender), + handle, + } + } + + pub async fn convert(&mut self) { + println!("Send convert"); + + let (resp_tx, resp_rx) = oneshot::channel::>(); + let cmd = ConvertCommand { + responder: resp_tx, + shutdown: false, + }; + + // Send request + match self.sender.as_mut().unwrap().send(cmd).await { + Ok(_) => { + // All good + } + Err(err) => { + panic!("Failed to send get_themes request: {}", err.to_string()) + } + } + + // Wait for result + resp_rx.await.unwrap().unwrap() + } +} + +impl Drop for Converter { + fn drop(&mut self) { + if let Some(mut sender) = self.sender.take() { + // Send shutdown command + let (resp_tx, resp_rx) = oneshot::channel::>(); + let cmd = ConvertCommand { + responder: resp_tx, + shutdown: true, + }; + + // Use std::sync channels for shutdown + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + futures::executor::block_on(async { + sender.send(cmd).await.unwrap(); + resp_rx.await.unwrap().unwrap(); + }); + tx.send(()).unwrap(); + }); + rx.recv().unwrap(); + } + + // Wait for the thread to finish + if Arc::strong_count(&self.handle) == 1 { + let handle = std::mem::replace(&mut self.handle, Arc::new(thread::spawn(|| {}))); + if let Ok(thread_handle) = Arc::try_unwrap(handle) { + thread_handle.join().unwrap(); + } + } + } +} + +pub struct InnerConverter { + worker: MainWorker, +} + +impl InnerConverter { + pub async fn new() -> Self { + println!("build inner converter"); + let options = WorkerOptions { + ..Default::default() + }; + + let main_module = + deno_core::resolve_path("empty_main.js", Path::new(env!("CARGO_MANIFEST_DIR"))) + .unwrap(); + + let permissions = PermissionsContainer::new(Permissions::allow_all()); + + let mut worker = + MainWorker::bootstrap_from_options(main_module.clone(), permissions, options); + + worker.execute_main_module(&main_module).await.unwrap(); + worker.run_event_loop(false).await.unwrap(); + Self { worker } + } + + fn convert(&mut self) { + println!("inner convert"); + let code = r"1 + 1".to_string(); + self.worker + .execute_script("ext:", code.into()) + .unwrap(); + } +} + +impl Drop for InnerConverter { + fn drop(&mut self) { + println!("drop inner converter"); + } +} + +async fn convert3() { + let mut converter = Converter::new(); + converter.convert().await; +} diff --git a/vl-convert-rs/src/converter.rs b/vl-convert-rs/src/converter.rs index c4b59640..70663334 100644 --- a/vl-convert-rs/src/converter.rs +++ b/vl-convert-rs/src/converter.rs @@ -246,6 +246,15 @@ struct InnerVlConverter { vega_initialized: bool, } +impl Drop for InnerVlConverter { + fn drop(&mut self) { + if let Err(e) = self.worker.dispatch_unload_event() { + // Log error but continue cleanup + eprintln!("Error dispatching unload event during cleanup: {}", e); + } + } +} + impl InnerVlConverter { async fn init_vega(&mut self) -> Result<(), AnyError> { if !self.vega_initialized { @@ -938,6 +947,34 @@ delete themes.default let value = self.execute_script_to_json("themes").await?; Ok(value) } + + pub async fn does_it_crash(&mut self) -> Result<(), AnyError> { + let import_code = format!( + r#" +1 + 1; +"#, + ); + // var vega; + // import('{vega_url}').then((imported) => {{ + // vega = imported; + // }}) + // var vegaThemes; + // import('{vega_themes_url}').then((imported) => {{ + // vegaThemes = imported; + // }}) + // + // var op_text_width; + // var op_get_json_arg; + // import("ext:core/ops").then((imported) => {{ + // op_text_width = imported.op_text_width; + // op_get_json_arg = imported.op_get_json_arg; + // }}) + + self.worker + .execute_script("ext:", import_code.into())?; + + Ok(()) + } } pub enum VlConvertCommand { @@ -972,6 +1009,9 @@ pub enum VlConvertCommand { GetThemes { responder: oneshot::Sender>, }, + DoesItCrash { + responder: oneshot::Sender>, + }, } /// Struct for performing Vega-Lite to Vega conversions using the Deno v8 Runtime @@ -1075,6 +1115,10 @@ impl VlConverter { let themes = inner.get_themes().await; responder.send(themes).ok(); } + VlConvertCommand::DoesItCrash { responder } => { + let res = inner.does_it_crash().await; + let _ = responder.send(res); + } } } Ok::<(), AnyError>(()) @@ -1447,6 +1491,27 @@ impl VlConverter { Err(err) => bail!("Failed to retrieve get_themes result: {}", err.to_string()), } } + + pub async fn does_it_crash(&mut self) -> Result<(), AnyError> { + let (resp_tx, resp_rx) = oneshot::channel::>(); + let cmd = VlConvertCommand::DoesItCrash { responder: resp_tx }; + + // Send request + match self.sender.send(cmd).await { + Ok(_) => { + // All good + } + Err(err) => { + bail!("Failed to send get_themes request: {}", err.to_string()) + } + } + + // Wait for result + match resp_rx.await { + Ok(result) => result, + Err(err) => bail!("Failed to retrieve get_themes result: {}", err.to_string()), + } + } } impl Default for VlConverter {