diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61959d7fb3c..ba1c2df014a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,6 +87,9 @@ jobs: TARGET: ${{ matrix.target }} steps: - uses: actions/checkout@v3 + - uses: webfactory/ssh-agent@v0.7.0 + with: + ssh-private-key: ${{ secrets.PIRITA_SSH_PRIVATE_KEY }} - name: Set up libstdc++ on Linux if: matrix.build == 'linux-x64' run: | @@ -275,14 +278,14 @@ jobs: target: aarch64-unknown-linux-gnu - name: Build cross image run: | - docker build -t wasmer/aarch64 /home/runner/work/wasmer/wasmer/.github/cross-linux-aarch64/ + docker build -t wasmer/aarch64 ${GITHUB_WORKSPACE}/.github/cross-linux-aarch64/ env: CROSS_DOCKER_IN_DOCKER: true - name: Build Wasmer binary run: | make build-wasmer env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig @@ -293,7 +296,7 @@ jobs: run: | make package-capi-headless env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig @@ -305,7 +308,7 @@ jobs: run: | make build-capi env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig @@ -315,7 +318,7 @@ jobs: run: | make distribution env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig diff --git a/.github/workflows/test-sys.yaml b/.github/workflows/test-sys.yaml index 6e7402fbc54..7c546f613e0 100644 --- a/.github/workflows/test-sys.yaml +++ b/.github/workflows/test-sys.yaml @@ -199,6 +199,28 @@ jobs: '${{ runner.tool_cache }}/cargo-sccache/bin/sccache' -s echo 'RUSTC_WRAPPER=${{ runner.tool_cache }}/cargo-sccache/bin/sccache' >> $GITHUB_ENV shell: bash + - name: Test integration CLI + if: matrix.run_test && matrix.os != 'windows-2019' + shell: bash + run: | + make && make build-wasmer && make build-capi && make package-capi && make package + export WASMER_DIR=`pwd`/package + make test-integration-cli + env: + TARGET: ${{ matrix.target }} + TARGET_DIR: target/${{ matrix.target }}/release + CARGO_TARGET: --target ${{ matrix.target }} + - name: Test integration CLI + if: matrix.run_test && matrix.os == 'windows-2019' + shell: bash + run: | + make && make build-wasmer && make build-capi && make package-capi && make package + export WASMER_DIR=`pwd`/package + make test-integration-cli + env: + TARGET: ${{ matrix.target }} + TARGET_DIR: target/${{ matrix.target }}/release + CARGO_TARGET: --target x86_64-pc-windows-msvc - name: Test if: matrix.run_test && matrix.os != 'windows-2019' run: | @@ -223,16 +245,6 @@ jobs: TARGET: ${{ matrix.target }} TARGET_DIR: target/${{ matrix.target }}/release CARGO_TARGET: --target ${{ matrix.target }} - - name: Test integration CLI - if: matrix.run_test && matrix.os != 'windows-2019' - run: | - make && make build-capi && make package-capi && make package - export WASMER_DIR=`pwd`/package - make test-integration-cli - env: - TARGET: ${{ matrix.target }} - TARGET_DIR: target/${{ matrix.target }}/release - CARGO_TARGET: --target ${{ matrix.target }} - name: Test if: matrix.run_test && matrix.os == 'windows-2019' shell: bash diff --git a/Cargo.lock b/Cargo.lock index 128e98b66d8..8f9d65fdff6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30275ad0ad84ec1c06dde3b3f7d23c6006b7d76d61a85e7060b426b747eff70d" +[[package]] +name = "any_ascii" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e" + [[package]] name = "anyhow" version = "1.0.66" @@ -213,6 +219,12 @@ dependencies = [ "glob", ] +[[package]] +name = "build_const" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" + [[package]] name = "bumpalo" version = "3.11.1" @@ -496,6 +508,15 @@ dependencies = [ "windows-sys 0.33.0", ] +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + [[package]] name = "cranelift-bforest" version = "0.86.1" @@ -564,6 +585,15 @@ version = "0.86.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed16b14363d929b8c37e3c557d0a7396791b383ecc302141643c054343170aad" +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -1085,6 +1115,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.25" @@ -1677,6 +1713,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +[[package]] +name = "lexical-sort" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c09e4591611e231daf4d4c685a66cb0410cc1e502027a20ae55f2bb9e997207a" +dependencies = [ + "any_ascii", +] + [[package]] name = "libc" version = "0.2.136" @@ -1751,6 +1796,16 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "lzma-rs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aba8ecb0450dfabce4ad72085eed0a75dffe8f21f7ada05638564ea9db2d7fb1" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "mach" version = "0.3.2" @@ -1929,6 +1984,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "nuke-dir" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6842d8099b88d19a64158a6cfdc3e9ad82c738c041dab98280ef7ba98d64fa" + [[package]] name = "num-integer" version = "0.1.45" @@ -2053,6 +2114,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +[[package]] +name = "path-clean" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd" + [[package]] name = "percent-encoding" version = "2.2.0" @@ -2280,6 +2347,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -2288,7 +2368,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -2298,9 +2378,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -2353,6 +2448,15 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2861,6 +2965,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -3027,6 +3142,16 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -3467,6 +3592,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3743,7 +3869,9 @@ dependencies = [ "wasmer-inline-c", "wasmer-middlewares", "wasmer-types", + "wasmer-vfs", "wasmer-wasi", + "webc", ] [[package]] @@ -3763,7 +3891,7 @@ dependencies = [ "blake3", "criterion", "hex", - "rand", + "rand 0.8.5", "tempfile", "thiserror", "wasmer", @@ -3796,11 +3924,15 @@ dependencies = [ "fern", "http_req", "log", + "nuke-dir", "prettytable-rs", "regex", + "reqwest", + "serde", "serde_json", "spinner", "target-lexicon 0.12.4", + "tempdir", "tempfile", "toml", "unix_mode", @@ -3822,6 +3954,7 @@ dependencies = [ "wasmer-wasi", "wasmer-wasi-experimental-io-devices", "wasmer-wast", + "webc", ] [[package]] @@ -3990,7 +4123,10 @@ name = "wasmer-integration-tests-cli" version = "3.0.0-rc.1" dependencies = [ "anyhow", - "rand", + "flate2", + "rand 0.8.5", + "tar", + "target-lexicon 0.12.4", "tempfile", ] @@ -4024,6 +4160,7 @@ dependencies = [ "dirs 4.0.0", "flate2", "graphql_client", + "lzma-rs", "reqwest", "semver 1.0.14", "serde", @@ -4064,12 +4201,14 @@ dependencies = [ name = "wasmer-vfs" version = "3.0.0-rc.1" dependencies = [ + "anyhow", "libc", "serde", "slab", "thiserror", "tracing", "typetag", + "webc", ] [[package]] @@ -4108,6 +4247,7 @@ dependencies = [ name = "wasmer-wasi" version = "3.0.0-rc.1" dependencies = [ + "anyhow", "bincode", "bytes", "cfg-if 1.0.0", @@ -4117,6 +4257,7 @@ dependencies = [ "getrandom", "libc", "serde", + "serde_cbor", "thiserror", "tracing", "tracing-wasm", @@ -4124,11 +4265,13 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-test", "wasmer", + "wasmer-emscripten", "wasmer-vbus", "wasmer-vfs", "wasmer-vnet", "wasmer-wasi-local-networking", "wasmer-wasi-types", + "webc", "winapi", ] @@ -4432,6 +4575,29 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webc" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef87e7b955d5d1feaa8697ae129f1a9ce8859e151574ad3baceae9413b48d2f0" +dependencies = [ + "anyhow", + "base64", + "indexmap", + "leb128", + "lexical-sort", + "memchr", + "memmap2", + "path-clean", + "rand 0.8.5", + "serde", + "serde_cbor", + "serde_json", + "sha2", + "url", + "walkdir", +] + [[package]] name = "webpki" version = "0.21.4" diff --git a/Makefile b/Makefile index 557ea873a19..2241b1b33f6 100644 --- a/Makefile +++ b/Makefile @@ -362,10 +362,10 @@ check-capi: capi-setup --no-default-features --features wat,compiler,wasi,middlewares $(capi_compiler_features) build-wasmer: - $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --bin wasmer + $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --features="webc_runner" --bin wasmer build-wasmer-debug: - $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "debug" --bin wasmer + $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "webc_runner,debug" --bin wasmer bench: $(CARGO_BINARY) bench $(CARGO_TARGET) $(compiler_features) @@ -426,31 +426,31 @@ build-docs-capi: capi-setup build-capi: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,wasi,middlewares $(capi_compiler_features) + --no-default-features --features wat,compiler,wasi,middlewares,webc_runner $(capi_compiler_features) build-capi-singlepass: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,singlepass,wasi,middlewares + --no-default-features --features wat,compiler,singlepass,wasi,middlewares,webc_runner build-capi-singlepass-universal: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,singlepass,wasi,middlewares + --no-default-features --features wat,compiler,singlepass,wasi,middlewares,webc_runner build-capi-cranelift: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,cranelift,wasi,middlewares + --no-default-features --features wat,compiler,cranelift,wasi,middlewares,webc_runner build-capi-cranelift-universal: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,cranelift,wasi,middlewares + --no-default-features --features wat,compiler,cranelift,wasi,middlewares,webc_runner build-capi-llvm: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,llvm,wasi,middlewares + --no-default-features --features wat,compiler,llvm,wasi,middlewares,webc_runner build-capi-llvm-universal: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,llvm,wasi,middlewares + --no-default-features --features wat,compiler,llvm,wasi,middlewares,webc_runner # Headless (we include the minimal to be able to run) @@ -524,7 +524,7 @@ test-capi: build-capi package-capi $(foreach compiler_engine,$(capi_compilers_en test-capi-crate-%: WASMER_CAPI_CONFIG=$(shell echo $@ | sed -e s/test-capi-crate-//) $(CARGO_BINARY) test $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ - --no-default-features --features wat,compiler,wasi,middlewares $(capi_compiler_features) -- --nocapture + --no-default-features --features wat,compiler,wasi,middlewares,webc_runner $(capi_compiler_features) -- --nocapture test-capi-integration-%: # Test the Wasmer C API tests for C @@ -543,10 +543,10 @@ test-examples: $(CARGO_BINARY) test $(CARGO_TARGET) --release $(compiler_features) --features wasi --examples test-integration-cli: - $(CARGO_BINARY) test $(CARGO_TARGET) --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture + $(CARGO_BINARY) test $(CARGO_TARGET) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture test-integration-ios: - $(CARGO_BINARY) test $(CARGO_TARGET) -p wasmer-integration-tests-ios + $(CARGO_BINARY) test $(CARGO_TARGET) --features webc_runner -p wasmer-integration-tests-ios generate-wasi-tests: # Uncomment the following for installing the toolchain diff --git a/deny.toml b/deny.toml index 5f87fed14c2..a9e3f9a4007 100644 --- a/deny.toml +++ b/deny.toml @@ -105,7 +105,7 @@ confidence-threshold = 0.8 exceptions = [ # Each entry is the crate and version constraint, and its specific allow # list - #{ allow = ["Zlib"], name = "adler32", version = "*" }, + { allow = ["LicenseRef-LICENSE.txt"], name = "webc", version = "*" }, ] # Some crates don't have (easily) machine readable licensing information, diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index 1c896654b8e..1767e6ef42e 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -31,6 +31,8 @@ wasmer-compiler = { version = "=3.0.0-rc.1", path = "../compiler" } wasmer-middlewares = { version = "=3.0.0-rc.1", path = "../middlewares", optional = true } wasmer-wasi = { version = "=3.0.0-rc.1", path = "../wasi", default-features = false, features = ["host-fs", "sys"], optional = true } wasmer-types = { version = "=3.0.0-rc.1", path = "../types" } +wasmer-vfs = { version = "=3.0.0-rc.1", path = "../vfs", optional = true, default-features = false, features = ["static-fs"] } +webc = { version = "3.0.1", optional = true } enumset = "1.0.2" cfg-if = "1.0" lazy_static = "1.4" @@ -90,7 +92,7 @@ wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] static-artifact-load = ["wasmer-compiler/static-artifact-load"] static-artifact-create = ["wasmer-compiler/static-artifact-create"] - +webc_runner = ["wasmer-wasi/webc_runner", "wasmer-vfs", "webc"] # Deprecated features. jit = ["compiler"] diff --git a/lib/c-api/examples/assets/python-0.1.0.wasmer b/lib/c-api/examples/assets/python-0.1.0.wasmer new file mode 100644 index 00000000000..cf6fb446a93 Binary files /dev/null and b/lib/c-api/examples/assets/python-0.1.0.wasmer differ diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index b83dc3fd243..0d5f9aa7d76 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -8,6 +8,7 @@ use super::{ instance::wasm_instance_t, module::wasm_module_t, store::{wasm_store_t, StoreRef}, + types::wasm_byte_vec_t, }; use crate::error::update_last_error; use std::convert::TryInto; @@ -22,6 +23,8 @@ use std::{ io::{self, SeekFrom}, sync::MutexGuard, }; +#[cfg(feature = "webc_runner")] +use wasmer_api::{AsStoreMut, Imports, Module}; use wasmer_wasi::{ get_wasi_version, FsError, VirtualFile, WasiBidirectionalPipePair, WasiFile, WasiFunctionEnv, WasiPipe, WasiState, WasiStateBuilder, WasiVersion, @@ -518,7 +521,7 @@ fn test_wasi_pipe_with_destructor() { let wasi_pipe_t_ptr = unsafe { &mut *wasi_pipe_t_ptr }; let second_wasi_pipe_t_ptr = unsafe { &mut *second_wasi_pipe_t_ptr }; - let data = b"hello".into_iter().map(|v| *v as i8).collect::>(); + let data = b"hello".iter().map(|v| *v as i8).collect::>(); let result = unsafe { wasi_pipe_write_bytes(wasi_pipe_t_ptr, data.as_ptr(), data.len()) }; assert_eq!(result, 5); @@ -813,6 +816,134 @@ pub unsafe extern "C" fn wasi_config_overwrite_stderr( .stderr(Box::from_raw(stderr_overwrite)); } +#[repr(C)] +pub struct wasi_filesystem_t { + ptr: *const c_char, + size: usize, +} + +#[no_mangle] +pub unsafe extern "C" fn wasi_filesystem_init_static_memory( + volume_bytes: Option<&wasm_byte_vec_t>, +) -> Option> { + let volume_bytes = volume_bytes.as_ref()?; + Some(Box::new(wasi_filesystem_t { + ptr: { + let ptr = (volume_bytes.data.as_ref()?) as *const _ as *const c_char; + if ptr.is_null() { + return None; + } + ptr + }, + size: volume_bytes.size, + })) +} + +#[no_mangle] +pub unsafe extern "C" fn wasi_filesystem_delete(ptr: *mut wasi_filesystem_t) { + let _ = Box::from_raw(ptr); +} + +/// Initializes the `imports` with an import object that links to +/// the custom file system +#[cfg(feature = "webc_runner")] +#[no_mangle] +pub unsafe extern "C" fn wasi_env_with_filesystem( + config: Box, + store: Option<&mut wasm_store_t>, + module: Option<&wasm_module_t>, + fs: Option<&wasi_filesystem_t>, + imports: Option<&mut wasm_extern_vec_t>, + package: *const c_char, +) -> Option> { + wasi_env_with_filesystem_inner(config, store, module, fs, imports, package) +} + +#[cfg(feature = "webc_runner")] +unsafe fn wasi_env_with_filesystem_inner( + config: Box, + store: Option<&mut wasm_store_t>, + module: Option<&wasm_module_t>, + fs: Option<&wasi_filesystem_t>, + imports: Option<&mut wasm_extern_vec_t>, + package: *const c_char, +) -> Option> { + let store = &mut store?.inner; + let fs = fs.as_ref()?; + let package_str = CStr::from_ptr(package); + let package = package_str.to_str().unwrap_or(""); + let module = &module.as_ref()?.inner; + let imports = imports?; + + let (wasi_env, import_object) = prepare_webc_env( + config, + &mut store.store_mut(), + module, + std::mem::transmute(fs.ptr), // cast wasi_filesystem_t.ptr as &'static [u8] + fs.size, + package, + )?; + + imports_set_buffer(&store, module, import_object, imports)?; + + Some(Box::new(wasi_env_t { + inner: wasi_env, + store: store.clone(), + })) +} + +#[cfg(feature = "webc_runner")] +fn prepare_webc_env( + config: Box, + store: &mut impl AsStoreMut, + module: &Module, + bytes: &'static u8, + len: usize, + package_name: &str, +) -> Option<(WasiFunctionEnv, Imports)> { + use wasmer_vfs::static_fs::StaticFileSystem; + use webc::FsEntryType; + + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + let volumes = webc::WebC::parse_volumes_from_fileblock(slice).ok()?; + let top_level_dirs = volumes + .into_iter() + .flat_map(|(_, volume)| { + volume + .header + .top_level + .iter() + .cloned() + .filter(|e| e.fs_type == FsEntryType::Dir) + .map(|e| e.text.to_string()) + .collect::>() + .into_iter() + }) + .collect::>(); + + let filesystem = Box::new(StaticFileSystem::init(slice, &package_name)?); + let mut wasi_env = config.state_builder; + + if let Some(s) = config.stdout { + wasi_env.stdout(s); + } + + if let Some(s) = config.stderr { + wasi_env.stderr(s); + } + + wasi_env.set_fs(filesystem); + + for f_name in top_level_dirs.iter() { + wasi_env + .preopen(|p| p.directory(f_name).read(true).write(true).create(true)) + .ok()?; + } + let env = wasi_env.finalize(store).ok()?; + let import_object = env.import_object(store, &module).ok()?; + Some((env, import_object)) +} + #[allow(non_camel_case_types)] pub struct wasi_env_t { /// cbindgen:ignore @@ -1013,8 +1144,18 @@ unsafe fn wasi_get_imports_inner( let import_object = c_try!(wasi_env.inner.import_object(&mut store_mut, &module.inner)); + imports_set_buffer(store, &module.inner, import_object, imports)?; + + Some(()) +} + +pub(crate) fn imports_set_buffer( + store: &StoreRef, + module: &wasmer_api::Module, + import_object: wasmer_api::Imports, + imports: &mut wasm_extern_vec_t, +) -> Option<()> { imports.set_buffer(c_try!(module - .inner .imports() .map(|import_type| { let ext = import_object diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 7eb728cbfc6..337196b1073 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -54,7 +54,10 @@ cfg-if = "1.0" fern = { version = "0.6", features = ["colored"], optional = true } log = { version = "0.4", optional = true } tempfile = "3" +tempdir = "0.3.7" http_req = { version="^0.8", default-features = false, features = ["rust-tls"], optional = true } +reqwest = { version = "^0.11", default-features = false, feature = ["rustls-tls", "json"], optional = true } +serde = { version = "1.0.147", features = ["derive"], optional = true } dirs = { version = "4.0", optional = true } serde_json = { version = "1.0", optional = true } target-lexicon = { version = "0.12", features = ["std"] } @@ -64,6 +67,8 @@ walkdir = "2.3.2" regex = "1.6.0" toml = "0.5.9" url = "2.3.1" +nuke-dir = { version = "0.1.0", optional = true } +webc = { version = "3.0.1", optional = true } [build-dependencies] chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] } @@ -91,6 +96,7 @@ wast = ["wasmer-wast"] wasi = ["wasmer-wasi"] emscripten = ["wasmer-emscripten"] wat = ["wasmer/wat"] +webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"] compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", @@ -150,8 +156,10 @@ enable-serde = [ http = [ "http_req", + "reqwest", "dirs", "serde_json", + "serde", ] [package.metadata.binstall] diff --git a/lib/cli/src/c_gen/staticlib_header.rs b/lib/cli/src/c_gen/staticlib_header.rs index 6449ef8b490..96fe194fe6a 100644 --- a/lib/cli/src/c_gen/staticlib_header.rs +++ b/lib/cli/src/c_gen/staticlib_header.rs @@ -58,8 +58,8 @@ wasm_byte_vec_t generate_serialized_data() { return module_byte_vec; } -wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) { - // module_name intentionally unused for now: will be used in the future. +wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* wasm_name) { + // wasm_name intentionally unused for now: will be used in the future. wasm_byte_vec_t module_byte_vec = generate_serialized_data(); wasm_module_t* module = wasm_module_deserialize(store, &module_byte_vec); free(module_byte_vec.data); diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index a3b51f0836a..e2f8d6204d1 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -4,6 +4,10 @@ use super::ObjectFormat; use crate::store::CompilerOptions; use anyhow::{Context, Result}; use clap::Parser; +#[cfg(feature = "http")] +use serde::{Deserialize, Serialize}; +#[cfg(feature = "http")] +use std::collections::BTreeMap; use std::env; use std::fs; use std::fs::File; @@ -13,6 +17,8 @@ use std::path::{Path, PathBuf}; use std::process::Command; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; +#[cfg(feature = "webc_runner")] +use webc::{ParseOptions, WebCMmap}; /// The `prefixer` returns the a String to prefix each of the /// functions in the static object generated by the @@ -20,26 +26,27 @@ use wasmer_object::{emit_serialized, get_object_for_target}; #[cfg(feature = "static-artifact-create")] pub type PrefixerFn = Box String + Send>; -const WASMER_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_create_exe_main.c"); -const WASMER_DESERIALIZE_HEADER: &str = include_str!("wasmer_deserialize_module.h"); +const WASMER_MAIN_C_SOURCE: &str = include_str!("wasmer_create_exe_main.c"); +#[cfg(feature = "static-artifact-create")] +const WASMER_STATIC_MAIN_C_SOURCE: &str = include_str!("wasmer_static_create_exe_main.c"); #[derive(Debug, Clone)] -struct CrossCompile { +pub(crate) struct CrossCompile { /// Cross-compilation library path. - library_path: Option, + pub(crate) library_path: Option, /// Cross-compilation tarball library path. - tarball: Option, + pub(crate) tarball: Option, /// Specify `zig` binary path - zig_binary_path: Option, + pub(crate) zig_binary_path: Option, } -struct CrossCompileSetup { - target: Triple, - zig_binary_path: PathBuf, - library: PathBuf, - working_dir: PathBuf, +#[derive(Debug)] +pub(crate) struct CrossCompileSetup { + pub(crate) target: Triple, + pub(crate) zig_binary_path: PathBuf, + pub(crate) library: PathBuf, } #[derive(Debug, Parser)] @@ -80,6 +87,12 @@ pub struct CreateExe { #[clap(long = "zig-binary-path")] zig_binary_path: Option, + /// When compiling .webc files in separate steps + /// (create-obj, then create-exe on the resulting dir) + /// the create-exe step needs the path to the .webc file again + #[clap(long = "webc-volume-path")] + webc_volume_path: Option, + /// Object format options /// /// This flag accepts two options: `symbols` or `serialized`. @@ -111,11 +124,6 @@ pub struct CreateExe { impl CreateExe { /// Runs logic for the `compile` subcommand pub fn execute(&self) -> Result<()> { - let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); - let working_dir = tempfile::tempdir()?; - let starting_cd = env::current_dir()?; - let output_path = starting_cd.join(&self.output); - /* Making library_path, tarball zig_binary_path flags require that target_triple flag * is set cannot be encoded with structopt, so we have to perform cli flag validation * manually here */ @@ -155,152 +163,98 @@ impl CreateExe { }) .unwrap_or_default(); - env::set_current_dir(&working_dir)?; + let starting_cd = env::current_dir()?; + let wasm_module_path = starting_cd.join(&self.path); + let output_path = starting_cd.join(&self.output); + let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); - let cross_compilation: Option = if let Some(mut cross_subc) = - cross_compile.or_else(|| { - if self.target_triple.is_some() { - Some(CrossCompile { - library_path: None, - tarball: None, - zig_binary_path: None, - }) - } else { - None - } - }) { - if let ObjectFormat::Serialized = object_format { - return Err(anyhow!( - "Cross-compilation with serialized object format is not implemented." - )); - } + let cross_compilation = Self::get_cross_compile_setup( + cross_compile, + self.target_triple.clone(), + object_format, + &starting_cd, + )?; - let target = if let Some(target_triple) = self.target_triple.clone() { - target_triple - } else { - return Err(anyhow!( - "To cross-compile an executable, you must specify a target triple with --target" - )); - }; - if let Some(tarball_path) = cross_subc.tarball.as_mut() { - if tarball_path.is_relative() { - *tarball_path = starting_cd.join(&tarball_path); - if !tarball_path.exists() { - return Err(anyhow!( - "Tarball path `{}` does not exist.", - tarball_path.display() - )); - } else if tarball_path.is_dir() { - return Err(anyhow!( - "Tarball path `{}` is a directory.", - tarball_path.display() - )); - } - } + #[cfg(feature = "webc_runner")] + { + let working_dir = tempdir::TempDir::new("testpirita")?; + let working_dir = working_dir.path().to_path_buf(); + + if let Ok(pirita) = WebCMmap::parse(wasm_module_path.clone(), &ParseOptions::default()) + { + return self.create_exe_pirita( + &pirita, + target, + cross_compilation, + &working_dir, + output_path, + object_format, + ); } - let zig_binary_path = - find_zig_binary(cross_subc.zig_binary_path.as_ref().and_then(|p| { - if p.is_absolute() { - p.canonicalize().ok() - } else { - starting_cd.join(p).canonicalize().ok() - } - }))?; - let library = if let Some(v) = cross_subc.library_path.clone() { - v - } else { - { - let libwasmer_path = if self - .target_triple - .clone() - .unwrap_or(Triple::host()) - .operating_system - == wasmer_types::OperatingSystem::Windows - { - "lib/wasmer.lib" - } else { - "lib/libwasmer.a" - }; - let libwasmer_headless_path = if self - .target_triple - .clone() - .unwrap_or(Triple::host()) - .operating_system - == wasmer_types::OperatingSystem::Windows - { - "lib/wasmer.lib" - } else { - "lib/libwasmer-headless.a" - }; - let filename = if let Some(local_tarball) = cross_subc.tarball { - let files = untar(local_tarball)?; - files.clone().into_iter().find(|f| f.contains(libwasmer_headless_path)).or_else(|| - files.into_iter().find(|f| f.contains(libwasmer_path))).ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the provided tarball path.", target)})? - } else { - #[cfg(feature = "http")] - { - let release = http_fetch::get_latest_release()?; - let tarball = http_fetch::download_release(release, target.clone())?; - let files = untar(tarball)?; - files.clone().into_iter().find(|f| f.contains(libwasmer_headless_path)).or_else(|| - files.into_iter().find(|f| f.contains(libwasmer_path))).ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})? - } - #[cfg(not(feature = "http"))] - return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); - }; - filename.into() - } - }; - Some(CrossCompileSetup { - target, - zig_binary_path, - library, - working_dir: working_dir.path().to_path_buf(), - }) - } else { - None - }; + } let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + // Object is likely a file created from create-obj + #[cfg(feature = "webc_runner")] + if self.path.is_dir() { + let pirita_volume_path = self.webc_volume_path.clone() + .ok_or_else(|| anyhow::anyhow!("If compiling using a directory (created by create-obj), you need to also specify the webc path again using --webc-volume-path because the volumes.o is not part of the directory"))?; + + let file = WebCMmap::parse(pirita_volume_path.clone(), &ParseOptions::default()) + .map_err(|e| { + anyhow::anyhow!( + "could not parse {} as webc: {e}", + pirita_volume_path.display() + ) + })?; + + let volume_bytes = file.get_volumes_as_fileblock(); + + return self.link_exe_from_dir( + volume_bytes.as_slice(), + &store, + &target, + cross_compilation, + &self.path, + output_path, + object_format, + ); + } + println!("Compiler: {}", compiler_type.to_string()); println!("Target: {}", target.triple()); println!("Format: {:?}", object_format); #[cfg(not(windows))] - let wasm_object_path = working_dir.path().join("wasm.o"); + let wasm_object_path = PathBuf::from("wasm.o"); #[cfg(windows)] - let wasm_object_path = working_dir.path().join("wasm.obj"); - - let wasm_module_path = starting_cd.join(&self.path); - - let static_defs_header_path: PathBuf = working_dir.path().join("static_defs.h"); + let wasm_object_path = PathBuf::from("wasm.obj"); if let Some(header_path) = self.header.as_ref() { /* In this case, since a header file is given, the input file is expected to be an * object created with `create-obj` subcommand */ let header_path = starting_cd.join(&header_path); - std::fs::copy(&header_path, &static_defs_header_path) + std::fs::copy(&header_path, Path::new("static_defs.h")) .context("Could not access given header file")?; - let object_file_path = wasm_module_path; if let Some(setup) = cross_compilation.as_ref() { self.compile_zig( output_path, - object_file_path, - static_defs_header_path, + wasm_module_path, + std::path::Path::new("static_defs.h").into(), setup, + &[], + None, + None, )?; } else { self.link( - static_defs_header_path, - LinkCode { - object_paths: vec![object_file_path, "main_obj.obj".into()], - output_path, - working_dir: working_dir.path().to_path_buf(), - ..Default::default() - }, + output_path, + wasm_module_path, + std::path::Path::new("static_defs.h").into(), + &[], + None, + None, )?; } } else { @@ -310,50 +264,15 @@ impl CreateExe { .context("failed to compile Wasm")?; let bytes = module.serialize()?; let mut obj = get_object_for_target(target.triple())?; - emit_serialized(&mut obj, &bytes, target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple(), "WASMER_MODULE")?; let mut writer = BufWriter::new(File::create(&wasm_object_path)?); obj.write_stream(&mut writer) .map_err(|err| anyhow::anyhow!(err.to_string()))?; writer.flush()?; drop(writer); - // Write down header file that includes deserialize function - { - let mut writer = BufWriter::new(File::create(&static_defs_header_path)?); - writer.write_all(WASMER_DESERIALIZE_HEADER.as_bytes())?; - writer.flush()?; - } - - // write C src to disk - let c_src_path: PathBuf = working_dir.path().join("wasmer_main.c"); - #[cfg(not(windows))] - let c_src_obj: PathBuf = working_dir.path().join("wasmer_main.o"); - #[cfg(windows)] - let c_src_obj: PathBuf = working_dir.path().join("wasmer_main.obj"); - { - let mut c_src_file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&c_src_path) - .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; - } - run_c_compile( - &c_src_path, - &c_src_obj, - static_defs_header_path, - self.target_triple.clone(), - ) - .context("Failed to compile C source code")?; - LinkCode { - object_paths: vec![c_src_obj, wasm_object_path], - output_path, - additional_libraries: self.libraries.clone(), - target: self.target_triple.clone(), - ..Default::default() - } - .run() - .context("Failed to link objects together")?; + let cli_given_triple = self.target_triple.clone(); + self.compile_c(wasm_object_path, cli_given_triple, output_path)?; } #[cfg(not(feature = "static-artifact-create"))] ObjectFormat::Symbols => { @@ -380,31 +299,33 @@ impl CreateExe { ); // Write object file with functions let object_file_path: std::path::PathBuf = - working_dir.path().join("functions.o"); + std::path::Path::new("functions.o").into(); let mut writer = BufWriter::new(File::create(&object_file_path)?); obj.write_stream(&mut writer) .map_err(|err| anyhow::anyhow!(err.to_string()))?; writer.flush()?; // Write down header file that includes pointer arrays and the deserialize function - let mut writer = BufWriter::new(File::create(&static_defs_header_path)?); + let mut writer = BufWriter::new(File::create("static_defs.h")?); writer.write_all(header_file_src.as_bytes())?; writer.flush()?; if let Some(setup) = cross_compilation.as_ref() { self.compile_zig( output_path, object_file_path, - static_defs_header_path, + std::path::Path::new("static_defs.h").into(), setup, + &[], + None, + None, )?; } else { self.link( - static_defs_header_path, - LinkCode { - object_paths: vec![object_file_path, "main_obj.obj".into()], - output_path, - working_dir: working_dir.path().to_path_buf(), - ..Default::default() - }, + output_path, + object_file_path, + std::path::Path::new("static_defs.h").into(), + &[], + None, + None, )?; } } @@ -427,32 +348,190 @@ impl CreateExe { Ok(()) } + pub(crate) fn get_cross_compile_setup( + cross_compile: Option, + target_triple: Option, + object_format: ObjectFormat, + starting_cd: &Path, + ) -> Result, anyhow::Error> { + if let Some(mut cross_subc) = cross_compile.or_else(|| { + if target_triple.is_some() { + Some(CrossCompile { + library_path: None, + tarball: None, + zig_binary_path: None, + }) + } else { + None + } + }) { + if let ObjectFormat::Serialized = object_format { + return Err(anyhow!( + "Cross-compilation with serialized object format is not implemented." + )); + } + + let target = if let Some(target_triple) = target_triple.clone() { + target_triple + } else { + return Err(anyhow!( + "To cross-compile an executable, you must specify a target triple with --target" + )); + }; + if let Some(tarball_path) = cross_subc.tarball.as_mut() { + if tarball_path.is_relative() { + *tarball_path = starting_cd.join(&tarball_path); + if !tarball_path.exists() { + return Err(anyhow!( + "Tarball path `{}` does not exist.", + tarball_path.display() + )); + } else if tarball_path.is_dir() { + return Err(anyhow!( + "Tarball path `{}` is a directory.", + tarball_path.display() + )); + } + } + } + let zig_binary_path = + find_zig_binary(cross_subc.zig_binary_path.as_ref().and_then(|p| { + if p.is_absolute() { + p.canonicalize().ok() + } else { + starting_cd.join(p).canonicalize().ok() + } + }))?; + let library = if let Some(v) = cross_subc.library_path.clone() { + v.canonicalize().unwrap_or(v) + } else { + { + let libwasmer_path = if target_triple.unwrap_or(Triple::host()).operating_system + == wasmer_types::OperatingSystem::Windows + { + "lib/wasmer.lib" + } else { + "lib/libwasmer.a" + }; + let tarball_dir; + let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() { + let target_file_path = local_tarball + .parent() + .and_then(|parent| Some(parent.join(local_tarball.file_stem()?))) + .unwrap_or_else(|| local_tarball.clone()); + + let target_file_path = target_file_path + .parent() + .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) + .unwrap_or_else(|| target_file_path.clone()); + + let _ = std::fs::create_dir_all(&target_file_path); + let files = untar(local_tarball.clone(), target_file_path.clone())?; + tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path); + files.iter().find(|f| f.contains(libwasmer_path)).cloned().ok_or_else(|| { + anyhow!("Could not find libwasmer for {} target in the provided tarball path (files = {files:#?}, libwasmer_path = {libwasmer_path:?})", target)})? + } else { + #[cfg(feature = "http")] + { + let release = http_fetch::get_latest_release()?; + let tarball = http_fetch::download_release(release, target.clone())?; + let target_file_path = tarball + .parent() + .and_then(|parent| Some(parent.join(tarball.file_stem()?))) + .unwrap_or_else(|| tarball.clone()); + + let target_file_path = target_file_path + .parent() + .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) + .unwrap_or_else(|| target_file_path.clone()); + + tarball_dir = target_file_path + .canonicalize() + .unwrap_or_else(|_| target_file_path.clone()); + let files = untar(tarball, target_file_path)?; + files.into_iter().find(|f| f.contains(libwasmer_path)).ok_or_else(|| { + anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})? + } + #[cfg(not(feature = "http"))] + return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); + }; + tarball_dir.join(&filename) + } + }; + let ccs = CrossCompileSetup { + target, + zig_binary_path, + library, + }; + Ok(Some(ccs)) + } else { + Ok(None) + } + } + + fn compile_c( + &self, + wasm_object_path: PathBuf, + target_triple: Option, + output_path: PathBuf, + ) -> anyhow::Result<()> { + let tempdir = tempdir::TempDir::new("compile-c")?; + let tempdir_path = tempdir.path(); + + // write C src to disk + let c_src_path = tempdir_path.join("wasmer_main.c"); + #[cfg(not(windows))] + let c_src_obj = tempdir_path.join("wasmer_main.o"); + #[cfg(windows)] + let c_src_obj = tempdir_path.join("wasmer_main.obj"); + + std::fs::write( + &c_src_path, + WASMER_MAIN_C_SOURCE + .replace("// WASI_DEFINES", "#define WASI") + .as_bytes(), + )?; + + run_c_compile(&c_src_path, &c_src_obj, target_triple.clone()) + .context("Failed to compile C source code")?; + LinkCode { + object_paths: vec![c_src_obj, wasm_object_path], + output_path, + additional_libraries: self.libraries.clone(), + target: target_triple, + ..Default::default() + } + .run() + .context("Failed to link objects together")?; + + Ok(()) + } + + #[allow(clippy::too_many_arguments)] fn compile_zig( &self, output_path: PathBuf, object_path: PathBuf, - mut header_path: PathBuf, + mut header_code_path: PathBuf, setup: &CrossCompileSetup, + pirita_atoms: &[String], + pirita_main_atom: Option<&str>, + pirita_volume_path: Option, ) -> anyhow::Result<()> { - debug_assert!( - header_path.is_absolute(), - "compile_zig() called with relative header file path {}", - header_path.display() - ); + let tempdir = tempdir::TempDir::new("wasmer-static-compile-zig")?; + let tempdir_path = tempdir.path(); + let c_src_path = tempdir_path.join("wasmer_main.c"); let CrossCompileSetup { ref target, ref zig_binary_path, ref library, - ref working_dir, } = setup; - let c_src_path = working_dir.join("wasmer_main.c"); let mut libwasmer_path = library.to_path_buf(); - println!("Library Path: {}", libwasmer_path.display()); /* Cross compilation is only possible with zig */ println!("Using zig binary: {}", zig_binary_path.display()); let zig_triple = triple_to_zig_triple(target); - eprintln!("Using zig target triple: {}", &zig_triple); + println!("Using zig target triple: {}", &zig_triple); let lib_filename = libwasmer_path .file_name() @@ -461,17 +540,20 @@ impl CreateExe { .unwrap() .to_string(); libwasmer_path.pop(); - { - let mut c_src_file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&c_src_path) - .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; + + if let Some(entrypoint) = pirita_main_atom.as_ref() { + let c_code = Self::generate_pirita_wasmer_main_c_static(pirita_atoms, entrypoint); + std::fs::write(&c_src_path, c_code)?; + } else { + std::fs::write(&c_src_path, WASMER_STATIC_MAIN_C_SOURCE)?; + } + + if !header_code_path.is_dir() { + header_code_path.pop(); } - if !header_path.is_dir() { - header_path.pop(); + if header_code_path.display().to_string().is_empty() { + header_code_path = std::env::current_dir()?; } /* Compile main function */ @@ -480,30 +562,25 @@ impl CreateExe { include_dir.pop(); include_dir.push("include"); - let compiler_cmd = match std::process::Command::new("cc").output() { - Ok(_) => "cc", - Err(_) => "gcc", - }; - let mut cmd = Command::new(zig_binary_path); let mut cmd_mut: &mut Command = cmd - .arg(compiler_cmd) - .arg("-w") - .arg("-fgnu-inline-asm") - .arg("-fsanitize=undefined") - .arg("-fsanitize-trap=undefined") + .arg("cc") .arg("-target") .arg(&zig_triple) .arg(&format!("-L{}", libwasmer_path.display())) .arg(&format!("-l:{}", lib_filename)) .arg(&format!("-I{}", include_dir.display())) - .arg(&format!("-I{}", header_path.display())); + .arg(&format!("-I{}", header_code_path.display())); if !zig_triple.contains("windows") { cmd_mut = cmd_mut.arg("-lunwind"); } + cmd_mut = cmd_mut.arg(&object_path).arg(&c_src_path); + + if let Some(volume_obj) = pirita_volume_path.as_ref() { + cmd_mut = cmd_mut.arg(volume_obj.clone()); + } + cmd_mut - .arg(&object_path) - .arg(&c_src_path) .arg("-o") .arg(&output_path) .output() @@ -518,14 +595,443 @@ impl CreateExe { Ok(()) } + // Write the volumes.o file + #[cfg(feature = "webc_runner")] + fn write_volume_obj( + volume_bytes: &[u8], + target: &Target, + output_path: &Path, + ) -> anyhow::Result { + #[cfg(not(windows))] + let volume_object_path = output_path.join("volumes.o"); + #[cfg(windows)] + let volume_object_path = output_path.join("volumes.obj"); + + let mut volumes_object = get_object_for_target(target.triple())?; + emit_serialized( + &mut volumes_object, + volume_bytes, + target.triple(), + "VOLUMES", + )?; + + let mut writer = BufWriter::new(File::create(&volume_object_path)?); + volumes_object + .write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + drop(writer); + + Ok(volume_object_path) + } + + #[cfg(feature = "webc_runner")] + pub(crate) fn create_objs_pirita( + store: &Store, + file: &WebCMmap, + target: &Target, + output_path: &Path, + object_format: ObjectFormat, + ) -> anyhow::Result<()> { + if !output_path.is_dir() { + return Err(anyhow::anyhow!( + "Expected {} to be an output directory, not a file", + output_path.display() + )); + } + + std::fs::create_dir_all(output_path)?; + std::fs::create_dir_all(output_path.join("atoms"))?; + + let atom_to_run = file + .manifest + .entrypoint + .as_ref() + .and_then(|s| file.get_atom_name_for_command("wasi", s).ok()); + if let Some(atom_to_run) = atom_to_run.as_ref() { + std::fs::write(output_path.join("entrypoint"), atom_to_run)?; + } + + for (atom_name, atom_bytes) in file.get_all_atoms() { + std::fs::create_dir_all(output_path.join("atoms"))?; + + #[cfg(not(windows))] + let object_path = output_path.join("atoms").join(&format!("{atom_name}.o")); + #[cfg(windows)] + let object_path = output_path.join("atoms").join(&format!("{atom_name}.obj")); + + std::fs::create_dir_all(output_path.join("atoms").join(&atom_name))?; + + let header_path = output_path + .join("atoms") + .join(&atom_name) + .join("static_defs.h"); + + match object_format { + ObjectFormat::Serialized => { + let module = Module::new(&store, &atom_bytes) + .context(format!("Failed to compile atom {atom_name:?} to wasm"))?; + let bytes = module.serialize()?; + let mut obj = get_object_for_target(target.triple())?; + let atom_name_uppercase = atom_name.to_uppercase(); + emit_serialized(&mut obj, &bytes, target.triple(), &atom_name_uppercase)?; + + let mut writer = BufWriter::new(File::create(&object_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + drop(writer); + } + #[cfg(feature = "static-artifact-create")] + ObjectFormat::Symbols => { + let engine = store.engine(); + let engine_inner = engine.inner(); + let compiler = engine_inner.compiler()?; + let features = engine_inner.features(); + let tunables = store.tunables(); + let prefixer: Option = None; + let (module_info, obj, metadata_length, symbol_registry) = + Artifact::generate_object( + compiler, atom_bytes, prefixer, target, tunables, features, + )?; + + let header_file_src = crate::c_gen::staticlib_header::generate_header_file( + &module_info, + &*symbol_registry, + metadata_length, + ); + + let mut writer = BufWriter::new(File::create(&object_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + + let mut writer = BufWriter::new(File::create(&header_path)?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + } + #[cfg(not(feature = "static-artifact-create"))] + ObjectFormat::Symbols => { + return Err(anyhow!("Objects cannot be compiled in format \"symbols\" without static-artifact-create feature")); + } + } + } + + Ok(()) + } + + #[cfg(feature = "webc_runner")] + #[allow(clippy::too_many_arguments)] + fn link_exe_from_dir( + &self, + volume_bytes: &[u8], + _store: &Store, + target: &Target, + cross_compilation: Option, + working_dir: &Path, + output_path: PathBuf, + object_format: ObjectFormat, + ) -> anyhow::Result<()> { + let tempdir = tempdir::TempDir::new("link-exe-from-dir")?; + let tempdir_path = tempdir.path(); + + let entrypoint = std::fs::read_to_string(working_dir.join("entrypoint")) + .map_err(|_| anyhow::anyhow!("file has no entrypoint to run"))?; + + if !working_dir.join("atoms").exists() { + return Err(anyhow::anyhow!("file has no atoms to compile")); + } + + let mut atom_names = Vec::new(); + for obj in std::fs::read_dir(working_dir.join("atoms"))? { + let path = obj?.path(); + if !path.is_dir() { + if let Some(s) = path.file_stem() { + atom_names.push( + s.to_str() + .ok_or_else(|| anyhow::anyhow!("wrong atom name"))? + .to_string(), + ); + } + } + } + + match object_format { + ObjectFormat::Serialized => { + let mut link_objects: Vec = Vec::new(); + + let volume_object_path = + Self::write_volume_obj(volume_bytes, target, tempdir_path)?; + + link_objects.push(volume_object_path); + + #[cfg(not(windows))] + let c_src_obj = working_dir.join("wasmer_main.o"); + #[cfg(windows)] + let c_src_obj = working_dir.join("wasmer_main.obj"); + + for obj in std::fs::read_dir(working_dir.join("atoms"))? { + let path = obj?.path(); + if !path.is_dir() { + link_objects.push(path.to_path_buf()); + } + } + + let c_code = Self::generate_pirita_wasmer_main_c(&atom_names, &entrypoint); + + let c_src_path = working_dir.join("wasmer_main.c"); + + std::fs::write(&c_src_path, c_code.as_bytes()) + .context("Failed to open C source code file")?; + + if let Some(setup) = cross_compilation { + let CrossCompileSetup { + ref target, + ref zig_binary_path, + ref library, + } = setup; + + let mut libwasmer_path = library.to_path_buf(); + let zig_triple = triple_to_zig_triple(target); + + // Cross compilation is only possible with zig + println!("Library Path: {}", libwasmer_path.display()); + println!("Using zig binary: {}", zig_binary_path.display()); + println!("Using zig target triple: {}", &zig_triple); + + let lib_filename = libwasmer_path + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + + /* Compile main function */ + let compilation = { + libwasmer_path.pop(); + let mut include_dir = libwasmer_path.clone(); + include_dir.pop(); + include_dir.push("include"); + println!("include dir: {}", include_dir.display()); + let mut cmd = Command::new(zig_binary_path); + let mut cmd_mut: &mut Command = cmd + .arg("cc") + .arg("--verbose") + .arg("-target") + .arg(&zig_triple) + .arg(&format!("-L{}", libwasmer_path.display())) + .arg(&format!("-l:{}", lib_filename)) + .arg(&format!("-I{}", include_dir.display())); + if !zig_triple.contains("windows") { + cmd_mut = cmd_mut.arg("-lunwind"); + } + cmd_mut + .args(link_objects.into_iter()) + .arg(&c_src_path) + .arg("-o") + .arg(&output_path) + .output() + .context("Could not execute `zig`")? + }; + if !compilation.status.success() { + return Err(anyhow::anyhow!(String::from_utf8_lossy( + &compilation.stderr + ) + .to_string())); + } + } else { + run_c_compile(c_src_path.as_path(), &c_src_obj, self.target_triple.clone()) + .context("Failed to compile C source code")?; + + link_objects.push(c_src_obj); + LinkCode { + object_paths: link_objects, + output_path, + additional_libraries: self.libraries.clone(), + target: self.target_triple.clone(), + ..Default::default() + } + .run() + .context("Failed to link objects together")?; + } + } + ObjectFormat::Symbols => { + let object_file_path = working_dir.join("atoms").join(&format!("{entrypoint}.o")); + let static_defs_file_path = working_dir + .join("atoms") + .join(&entrypoint) + .join("static_defs.h"); + let volumes_obj_path = Self::write_volume_obj(volume_bytes, target, tempdir_path)?; + + if let Some(setup) = cross_compilation.as_ref() { + self.compile_zig( + output_path, + object_file_path, + static_defs_file_path, + setup, + &atom_names, + Some(&entrypoint), + Some(volumes_obj_path), + )?; + } else { + self.link( + output_path, + object_file_path, + static_defs_file_path, + &atom_names, + Some(&entrypoint), + Some(volumes_obj_path), + )?; + } + } + } + + Ok(()) + } + + fn normalize_atom_name(s: &str) -> String { + s.chars() + .filter_map(|c| { + if char::is_alphabetic(c) { + Some(c) + } else if c == '-' { + Some('_') + } else { + None + } + }) + .collect() + } + + fn generate_pirita_wasmer_main_c_static(atom_names: &[String], atom_to_run: &str) -> String { + let mut c_code_to_instantiate = String::new(); + let mut deallocate_module = String::new(); + + let atom_to_run = Self::normalize_atom_name(atom_to_run); + + for atom_name in atom_names.iter() { + let atom_name = Self::normalize_atom_name(atom_name); + + c_code_to_instantiate.push_str(&format!( + " + + wasm_module_t *atom_{atom_name} = wasmer_object_module_new(store, \"{atom_name}\"); + + if (!atom_{atom_name}) {{ + fprintf(stderr, \"Failed to create module from atom \\\"{atom_name}\\\"\\n\"); + print_wasmer_error(); + return -1; + }} + " + )); + deallocate_module.push_str(&format!("wasm_module_delete(atom_{atom_name});")); + } + + c_code_to_instantiate.push_str(&format!("wasm_module_t *module = atom_{atom_to_run};")); + + WASMER_STATIC_MAIN_C_SOURCE + .replace("#define WASI", "#define WASI\r\n#define WASI_PIRITA") + .replace("// INSTANTIATE_MODULES", &c_code_to_instantiate) + .replace("##atom-name##", &atom_to_run) + .replace("wasm_module_delete(module);", &deallocate_module) + } + + #[cfg(feature = "webc_runner")] + fn generate_pirita_wasmer_main_c(atom_names: &[String], atom_to_run: &str) -> String { + let mut c_code_to_add = String::new(); + let mut c_code_to_instantiate = String::new(); + let mut deallocate_module = String::new(); + + for atom_name in atom_names.iter() { + let atom_name = Self::normalize_atom_name(atom_name); + let atom_name_uppercase = atom_name.to_uppercase(); + + c_code_to_add.push_str(&format!( + " + extern size_t {atom_name_uppercase}_LENGTH asm(\"{atom_name_uppercase}_LENGTH\"); + extern char {atom_name_uppercase}_DATA asm(\"{atom_name_uppercase}_DATA\"); + " + )); + + c_code_to_instantiate.push_str(&format!(" + wasm_byte_vec_t atom_{atom_name}_byte_vec = {{ + .size = {atom_name_uppercase}_LENGTH, + .data = &{atom_name_uppercase}_DATA, + }}; + wasm_module_t *atom_{atom_name} = wasm_module_deserialize(store, &atom_{atom_name}_byte_vec); + + if (!atom_{atom_name}) {{ + fprintf(stderr, \"Failed to create module from atom \\\"{atom_name}\\\"\\n\"); + print_wasmer_error(); + return -1; + }} + ")); + deallocate_module.push_str(&format!("wasm_module_delete(atom_{atom_name});")); + } + + c_code_to_instantiate.push_str(&format!("wasm_module_t *module = atom_{atom_to_run};")); + + WASMER_MAIN_C_SOURCE + .replace("#define WASI", "#define WASI\r\n#define WASI_PIRITA") + .replace("// DECLARE_MODULES", &c_code_to_add) + .replace("// INSTANTIATE_MODULES", &c_code_to_instantiate) + .replace("##atom-name##", atom_to_run) + .replace("wasm_module_delete(module);", &deallocate_module) + } + + #[cfg(feature = "webc_runner")] + fn create_exe_pirita( + &self, + file: &WebCMmap, + target: Target, + cross_compilation: Option, + working_dir: &Path, + output_path: PathBuf, + object_format: ObjectFormat, + ) -> anyhow::Result<()> { + let _ = std::fs::create_dir_all(&working_dir); + let (store, _) = self.compiler.get_store_for_target(target.clone())?; + + Self::create_objs_pirita(&store, file, &target, working_dir, object_format)?; + + let volumes_obj = file.get_volumes_as_fileblock(); + self.link_exe_from_dir( + volumes_obj.as_slice(), + &store, + &target, + cross_compilation, + working_dir, + output_path, + object_format, + )?; + + Ok(()) + } + #[cfg(feature = "static-artifact-create")] - fn link(&self, mut header_path: PathBuf, linkcode: LinkCode) -> anyhow::Result<()> { - debug_assert!( - header_path.is_absolute(), - "link() called with relative header file path {}", - header_path.display() - ); - let c_src_path: PathBuf = linkcode.working_dir.join("wasmer_main.c"); + fn link( + &self, + output_path: PathBuf, + object_path: PathBuf, + mut header_code_path: PathBuf, + pirita_atoms: &[String], + pirita_main_atom: Option<&str>, + pirita_volume_path: Option, + ) -> anyhow::Result<()> { + let tempdir = tempdir::TempDir::new("wasmer-static-compile")?; + let tempdir_path = tempdir.path(); + + let mut object_paths = vec![object_path, "main_obj.obj".into()]; + if let Some(volume_obj) = pirita_volume_path.as_ref() { + object_paths.push(volume_obj.to_path_buf()); + } + + let linkcode = LinkCode { + object_paths, + output_path, + ..Default::default() + }; + let c_src_path = tempdir_path.join("wasmer_main.c"); let mut libwasmer_path = get_libwasmer_path()? .canonicalize() .context("Failed to find libwasmer")?; @@ -539,27 +1045,43 @@ impl CreateExe { .unwrap() .to_string(); libwasmer_path.pop(); - { - let mut c_src_file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&c_src_path) - .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; + + if let Some(entrypoint) = pirita_main_atom.as_ref() { + let c_code = Self::generate_pirita_wasmer_main_c_static(pirita_atoms, entrypoint); + std::fs::write(&c_src_path, c_code)?; + } else { + std::fs::write(&c_src_path, WASMER_STATIC_MAIN_C_SOURCE)?; + } + + if !header_code_path.is_dir() { + header_code_path.pop(); } - if !header_path.is_dir() { - header_path.pop(); + if header_code_path.display().to_string().is_empty() { + header_code_path = std::env::current_dir()?; } + let wasmer_include_dir = get_wasmer_include_directory()?; + let wasmer_h_path = wasmer_include_dir.join("wasmer.h"); + if !wasmer_h_path.exists() { + return Err(anyhow::anyhow!( + "Could not find wasmer.h in {}", + wasmer_include_dir.display() + )); + } + let wasm_h_path = wasmer_include_dir.join("wasm.h"); + if !wasm_h_path.exists() { + return Err(anyhow::anyhow!( + "Could not find wasm.h in {}", + wasmer_include_dir.display() + )); + } + std::fs::copy(wasmer_h_path, header_code_path.join("wasmer.h"))?; + std::fs::copy(wasm_h_path, header_code_path.join("wasm.h"))?; + /* Compile main function */ let compilation = { - let compiler_cmd = match Command::new("cc").output() { - Ok(_) => "cc", - Err(_) => "gcc", - }; - - Command::new(compiler_cmd) + Command::new("cc") .arg("-c") .arg(&c_src_path) .arg(if linkcode.optimization_flag.is_empty() { @@ -568,7 +1090,6 @@ impl CreateExe { linkcode.optimization_flag.as_str() }) .arg(&format!("-L{}", libwasmer_path.display())) - .arg(&format!("-I{}", get_wasmer_include_directory()?.display())) .arg(&format!("-l:{}", lib_filename)) //.arg("-lwasmer") // Add libraries required per platform. @@ -583,7 +1104,7 @@ impl CreateExe { .arg("-ldl") .arg("-lm") .arg("-pthread") - .arg(&format!("-I{}", header_path.display())) + .arg(&format!("-I{}", header_code_path.display())) .arg("-v") .arg("-o") .arg("main_obj.obj") @@ -600,6 +1121,14 @@ impl CreateExe { } } +#[test] +fn test_normalize_atom_name() { + assert_eq!( + CreateExe::normalize_atom_name("atom-name-with-dash"), + "atom_name_with_dash".to_string() + ); +} + fn triple_to_zig_triple(target_triple: &Triple) -> String { let arch = match target_triple.architecture { wasmer_types::Architecture::X86_64 => "x86_64".into(), @@ -637,22 +1166,34 @@ fn get_wasmer_dir() -> anyhow::Result { fn get_wasmer_include_directory() -> anyhow::Result { let mut path = get_wasmer_dir()?; + if path.clone().join("wasmer.h").exists() { + return Ok(path); + } path.push("include"); + if !path.clone().join("wasmer.h").exists() { + println!( + "wasmer.h does not exist in {}, will probably default to the system path", + path.canonicalize().unwrap().display() + ); + } Ok(path) } /// path to the static libwasmer fn get_libwasmer_path() -> anyhow::Result { - let mut path = get_wasmer_dir()?; - path.push("lib"); + let path = get_wasmer_dir()?; // TODO: prefer headless Wasmer if/when it's a separate library. #[cfg(not(windows))] - path.push("libwasmer.a"); + let libwasmer_static_name = "libwasmer.a"; #[cfg(windows)] - path.push("wasmer.lib"); + let libwasmer_static_name = "libwasmer.lib"; - Ok(path) + if path.exists() && path.join(libwasmer_static_name).exists() { + Ok(path.join(libwasmer_static_name)) + } else { + Ok(path.join("lib").join(libwasmer_static_name)) + } } /// path to library tarball cache dir @@ -667,15 +1208,8 @@ fn get_libwasmer_cache_path() -> anyhow::Result { fn run_c_compile( path_to_c_src: &Path, output_name: &Path, - mut header_path: PathBuf, target: Option, ) -> anyhow::Result<()> { - debug_assert!( - header_path.is_absolute(), - "run_c_compile() called with relative header file path {}", - header_path.display() - ); - #[cfg(not(windows))] let c_compiler = "cc"; // We must use a C++ compiler on Windows because wasm.h uses `static_assert` @@ -683,17 +1217,14 @@ fn run_c_compile( #[cfg(windows)] let c_compiler = "clang++"; - if !header_path.is_dir() { - header_path.pop(); - } - let mut command = Command::new(c_compiler); let command = command + .arg("-Wall") .arg("-O2") .arg("-c") .arg(path_to_c_src) - .arg(&format!("-I{}", header_path.display())) - .arg(&format!("-I{}", get_wasmer_include_directory()?.display())); + .arg("-I") + .arg(get_wasmer_include_directory()?); let command = if let Some(target) = target { command.arg("-target").arg(format!("{}", target)) @@ -702,6 +1233,13 @@ fn run_c_compile( }; let output = command.arg("-o").arg(output_name).output()?; + eprintln!( + "run_c_compile: stdout: {}\n\nstderr: {}", + std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"), + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); if !output.status.success() { bail!( @@ -732,8 +1270,6 @@ struct LinkCode { libwasmer_path: PathBuf, /// The target to link the executable for. target: Option, - /// Working directory - working_dir: PathBuf, } impl Default for LinkCode { @@ -750,7 +1286,6 @@ impl Default for LinkCode { output_path: PathBuf::from("a.out"), libwasmer_path: get_libwasmer_path().unwrap(), target: None, - working_dir: env::current_dir().expect("could not get current dir from environment"), } } } @@ -767,6 +1302,7 @@ impl LinkCode { ); let mut command = Command::new(&self.linker_path); let command = command + .arg("-Wall") .arg(&self.optimization_flag) .args( self.object_paths @@ -795,7 +1331,8 @@ impl LinkCode { .iter() .map(|lib| format!("-l{}", lib)); let command = command.args(link_against_extra_libs); - let output = command.arg("-o").arg(&self.output_path).output()?; + let command = command.arg("-o").arg(&self.output_path); + let output = command.output()?; if !output.status.success() { bail!( @@ -989,12 +1526,36 @@ mod http_fetch { } Ok(response) }); + match super::get_libwasmer_cache_path() { + Ok(mut cache_path) => { + cache_path.push(&filename); + if !cache_path.exists() { + if let Err(err) = std::fs::copy(&filename, &cache_path) { + eprintln!( + "Could not store tarball to cache path `{}`: {}", + cache_path.display(), + err + ); + } else { + eprintln!( + "Cached tarball to cache path `{}`.", + cache_path.display() + ); + } + } + } + Err(err) => { + eprintln!( + "Could not determine cache path for downloaded binaries.: {}", + err + ); + } + } let _response = download_thread .join() .expect("Could not join downloading thread"); match super::get_libwasmer_cache_path() { Ok(mut cache_path) => { - let _ = std::fs::create_dir_all(&cache_path); cache_path.push(&filename); if !cache_path.exists() { if let Err(err) = std::fs::copy(&filename, &cache_path) { @@ -1025,7 +1586,7 @@ mod http_fetch { } } -fn untar(tarball: std::path::PathBuf) -> Result> { +fn untar(tarball: std::path::PathBuf, target: std::path::PathBuf) -> Result> { let files = std::process::Command::new("tar") .arg("-tf") .arg(&tarball) @@ -1041,9 +1602,12 @@ fn untar(tarball: std::path::PathBuf) -> Result> { .map(|s| s.to_string()) .collect::>(); + let _ = std::fs::create_dir_all(&target); let _output = std::process::Command::new("tar") .arg("-xf") .arg(&tarball) + .arg("-C") + .arg(&target) .output() .expect("failed to execute process"); Ok(files) @@ -1085,7 +1649,18 @@ fn find_zig_binary(path: Option) -> Result { break; } } - retval.ok_or_else(|| anyhow!("Could not find `zig` binary in PATH."))? + retval + .or_else(|| { + #[cfg(feature = "http")] + { + try_autoinstall_zig().map(|p| p.join("zig")) + } + #[cfg(not(feature = "http"))] + { + None + } + }) + .ok_or_else(|| anyhow!("Could not find `zig` binary in PATH."))? }; let version = std::process::Command::new(&retval) @@ -1113,3 +1688,126 @@ fn find_zig_binary(path: Option) -> Result { Ok(retval) } } + +/// Tries to auto-install zig into ~/.wasmer/utils/zig/{version} +#[cfg(feature = "http")] +fn try_autoinstall_zig() -> Option { + let zig_dir = wasmer_registry::get_wasmer_root_dir()? + .join("utils") + .join("zig"); + let mut existing_version = None; + + if !zig_dir.exists() { + return install_zig(&zig_dir); + } + + if let Ok(mut rd) = std::fs::read_dir(&zig_dir) { + existing_version = rd.next().and_then(|entry| { + let string = entry.ok()?.file_name().to_str()?.to_string(); + if zig_dir.join(&string).join("zig").exists() { + Some(string) + } else { + None + } + }) + } + + if let Some(exist) = existing_version { + return Some(zig_dir.join(exist)); + } + + install_zig(&zig_dir) +} + +#[cfg(feature = "http")] +fn install_zig(target_targz_path: &Path) -> Option { + let resp = reqwest::blocking::get("https://ziglang.org/download/index.json"); + let resp = resp.ok()?; + let resp = resp.json::(); + let resp = resp.ok()?; + + let default_key = "master".to_string(); + let (latest_version, latest_version_json) = resp + .versions + .get(&default_key) + .map(|v| (&default_key, v)) + .or_else(|| resp.versions.iter().next())?; + + let latest_version = match latest_version_json.version.as_ref() { + Some(s) => s, + None => latest_version, + }; + + let install_dir = target_targz_path.join(latest_version); + if install_dir.join("zig").exists() { + return Some(install_dir); + } + + let native_host_url = latest_version_json.get_native_host_url()?; + let _ = std::fs::create_dir_all(&install_dir); + wasmer_registry::download_and_unpack_targz(&native_host_url, &install_dir, true).ok() +} + +#[cfg(feature = "http")] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct ZiglangOrgJson { + #[serde(flatten)] + versions: BTreeMap, +} + +#[cfg(feature = "http")] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct ZiglangOrgJsonTarget { + version: Option, + date: String, + src: ZiglangOrgJsonBuildTarget, + #[serde(rename = "x86_64-freebsd")] + x86_64_freebsd: Option, + #[serde(rename = "x86_64-macos")] + x86_64_macos: Option, + #[serde(rename = "aarch64-macos")] + aarch64_macos: Option, + #[serde(rename = "x86_64-windows")] + x86_64_windows: Option, + #[serde(rename = "x86_64-linux")] + x86_64_linux: Option, + #[serde(rename = "aarch64-linux")] + aarch64_linux: Option, +} + +impl ZiglangOrgJsonTarget { + pub fn get_native_host_url(&self) -> Option { + let native_host = format!("{}", target_lexicon::HOST); + if native_host.starts_with("x86_64") { + if native_host.contains("freebsd") { + Some(self.x86_64_freebsd.as_ref()?.tarball.clone()) + } else if native_host.contains("darwin") || native_host.contains("macos") { + Some(self.x86_64_macos.as_ref()?.tarball.clone()) + } else if native_host.contains("windows") { + Some(self.x86_64_windows.as_ref()?.tarball.clone()) + } else if native_host.contains("linux") { + Some(self.x86_64_linux.as_ref()?.tarball.clone()) + } else { + None + } + } else if native_host.starts_with("aarch64") { + if native_host.contains("darwin") || native_host.contains("macos") { + Some(self.aarch64_macos.as_ref()?.tarball.clone()) + } else if native_host.contains("linux") { + Some(self.aarch64_linux.as_ref()?.tarball.clone()) + } else { + None + } + } else { + None + } + } +} + +#[cfg(feature = "http")] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +struct ZiglangOrgJsonBuildTarget { + tarball: String, + shasum: String, + size: String, +} diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs index 79995fd6f15..a662724f7e2 100644 --- a/lib/cli/src/commands/create_obj.rs +++ b/lib/cli/src/commands/create_obj.rs @@ -1,4 +1,5 @@ -//! Create a compiled standalone object file for a given Wasm file. +#![allow(dead_code)] +//! Create a standalone native executable for a given Wasm file. use super::ObjectFormat; use crate::{commands::PrefixerFn, store::CompilerOptions}; @@ -10,10 +11,13 @@ use std::fs::File; use std::io::prelude::*; use std::io::BufWriter; use std::path::PathBuf; +use std::process::Command; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; +#[cfg(feature = "webc_runner")] +use webc::{ParseOptions, WebCMmap}; -const WASMER_SERIALIZED_HEADER: &[u8] = include_bytes!("wasmer_deserialize_module.h"); +const WASMER_SERIALIZED_HEADER: &[u8] = include_bytes!("wasmer_create_exe.h"); #[derive(Debug, Parser)] /// The options for the `wasmer create-exe` subcommand @@ -83,16 +87,26 @@ impl CreateObj { Target::new(target_triple.clone(), features) }) .unwrap_or_default(); - let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + + let starting_cd = env::current_dir()?; + let wasm_module_path = starting_cd.join(&self.path); + let output_path = starting_cd.join(&self.output); let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); + #[cfg(feature = "webc_runner")] + { + if let Ok(pirita) = WebCMmap::parse(wasm_module_path.clone(), &ParseOptions::default()) + { + return self.execute_pirita(&pirita, target, output_path, object_format); + } + } + + let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + println!("Compiler: {}", compiler_type.to_string()); println!("Target: {}", target.triple()); println!("Format: {:?}", object_format); - let starting_cd = env::current_dir()?; - - let output_path = starting_cd.join(&self.output); let header_output = self.header_output.clone().unwrap_or_else(|| { let mut retval = self.output.clone(); retval.set_extension("h"); @@ -100,7 +114,6 @@ impl CreateObj { }); let header_output_path = starting_cd.join(&header_output); - let wasm_module_path = starting_cd.join(&self.path); match object_format { ObjectFormat::Serialized => { @@ -108,7 +121,7 @@ impl CreateObj { .context("failed to compile Wasm")?; let bytes = module.serialize()?; let mut obj = get_object_for_target(target.triple())?; - emit_serialized(&mut obj, &bytes, target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple(), "WASMER_MODULE")?; let mut writer = BufWriter::new(File::create(&output_path)?); obj.write_stream(&mut writer) .map_err(|err| anyhow::anyhow!(err.to_string()))?; @@ -150,17 +163,98 @@ impl CreateObj { self.output.display(), header_output.display(), ); - eprintln!("\n---\n"); - eprintln!( - r#"To use, link the object file to your executable and call the `wasmer_object_module_new` function defined in the header file. For example, in the C language: - #include "{}" - - wasm_module_t *module = wasmer_object_module_new(store, "my_module_name"); - "#, - header_output.display(), - ); + Ok(()) + } + #[cfg(feature = "webc_runner")] + fn execute_pirita( + &self, + file: &WebCMmap, + target: Target, + output_path: PathBuf, + object_format: ObjectFormat, + ) -> Result<()> { + if output_path.exists() { + if output_path.is_dir() { + nuke_dir::nuke_dir(&output_path) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + } + } else { + let _ = std::fs::create_dir_all(&output_path)?; + } + println!( + "outputting create-obj to directory {}", + output_path.display() + ); + let (store, _) = self.compiler.get_store_for_target(target.clone())?; + crate::commands::create_exe::CreateExe::create_objs_pirita( + &store, + file, + &target, + &output_path, + object_format, + )?; Ok(()) } } + +fn link( + output_path: PathBuf, + object_path: PathBuf, + header_code_path: PathBuf, +) -> anyhow::Result<()> { + let libwasmer_path = get_libwasmer_path()? + .canonicalize() + .context("Failed to find libwasmer")?; + println!( + "link output {:?}", + Command::new("cc") + .arg(&header_code_path) + .arg(&format!("-L{}", libwasmer_path.display())) + //.arg(&format!("-I{}", header_code_path.display())) + .arg("-pie") + .arg("-o") + .arg("header_obj.o") + .output()? + ); + //ld -relocatable a.o b.o -o c.o + + println!( + "link output {:?}", + Command::new("ld") + .arg("-relocatable") + .arg(&object_path) + .arg("header_obj.o") + .arg("-o") + .arg(&output_path) + .output()? + ); + + Ok(()) +} + +/// path to the static libwasmer +fn get_libwasmer_path() -> anyhow::Result { + let mut path = get_wasmer_dir()?; + path.push("lib"); + + // TODO: prefer headless Wasmer if/when it's a separate library. + #[cfg(not(windows))] + path.push("libwasmer.a"); + #[cfg(windows)] + path.push("wasmer.lib"); + + Ok(path) +} +fn get_wasmer_dir() -> anyhow::Result { + Ok(PathBuf::from( + env::var("WASMER_DIR") + .or_else(|e| { + option_env!("WASMER_INSTALL_PREFIX") + .map(str::to_string) + .ok_or(e) + }) + .context("Trying to read env var `WASMER_DIR`")?, + )) +} diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index fd66b7acd3d..08bc58290e9 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -1,3 +1,4 @@ +use crate::cli::SplitVersion; use crate::common::get_cache_dir; #[cfg(feature = "debug")] use crate::logging; @@ -5,6 +6,7 @@ use crate::store::{CompilerType, StoreOptions}; use crate::suggestions::suggest_function_exports; use crate::warning; use anyhow::{anyhow, Context, Result}; +use clap::Parser; use std::collections::HashMap; use std::ops::Deref; use std::path::PathBuf; @@ -13,11 +15,10 @@ use wasmer::FunctionEnv; use wasmer::*; #[cfg(feature = "cache")] use wasmer_cache::{Cache, FileSystemCache, Hash}; -use wasmer_types::Type as ValueType; - -use crate::cli::SplitVersion; -use clap::Parser; use wasmer_registry::PackageDownloadInfo; +use wasmer_types::Type as ValueType; +#[cfg(feature = "webc_runner")] +use wasmer_wasi::runners::{Runner, WapmContainer}; #[cfg(feature = "wasi")] mod wasi; @@ -216,6 +217,17 @@ impl Run { } fn inner_execute(&self) -> Result<()> { + #[cfg(feature = "webc_runner")] + { + if let Ok(pf) = WapmContainer::new(self.path.clone()) { + return Self::run_container( + pf, + &self.command_name.clone().unwrap_or_default(), + &self.args, + ) + .map_err(|e| anyhow!("Could not run PiritaFile: {e}")); + } + } let (mut store, module) = self.get_store_module()?; #[cfg(feature = "emscripten")] { @@ -352,6 +364,43 @@ impl Run { ret } + #[cfg(feature = "webc_runner")] + fn run_container(container: WapmContainer, id: &str, args: &[String]) -> Result<(), String> { + let mut result = None; + + #[cfg(feature = "wasi")] + { + if let Some(r) = result { + return r; + } + + let mut runner = wasmer_wasi::runners::wasi::WasiRunner::default(); + runner.set_args(args.to_vec()); + result = Some(if id.is_empty() { + runner.run(&container).map_err(|e| format!("{e}")) + } else { + runner.run_cmd(&container, id).map_err(|e| format!("{e}")) + }); + } + + #[cfg(feature = "emscripten")] + { + if let Some(r) = result { + return r; + } + + let mut runner = wasmer_wasi::runners::emscripten::EmscriptenRunner::default(); + runner.set_args(args.to_vec()); + result = Some(if id.is_empty() { + runner.run(&container).map_err(|e| format!("{e}")) + } else { + runner.run_cmd(&container, id).map_err(|e| format!("{e}")) + }); + } + + result.unwrap_or_else(|| Err("neither emscripten or wasi file".to_string())) + } + fn get_store_module(&self) -> Result<(Store, Module)> { let contents = std::fs::read(self.path.clone())?; if wasmer_compiler::Artifact::is_deserializable(&contents) { diff --git a/lib/cli/src/commands/wasmer_create_exe.h b/lib/cli/src/commands/wasmer_create_exe.h new file mode 100644 index 00000000000..a94afd1149f --- /dev/null +++ b/lib/cli/src/commands/wasmer_create_exe.h @@ -0,0 +1,25 @@ +#include "wasmer.h" +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); +extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); + +wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* wasm_name) { + wasm_byte_vec_t module_byte_vec = { + .size = WASMER_MODULE_LENGTH, + .data = (const char*)&WASMER_MODULE_DATA, + }; + wasm_module_t* module = wasm_module_deserialize(store, &module_byte_vec); + + return module; +} + +#ifdef __cplusplus +} +#endif diff --git a/lib/cli/src/commands/wasmer_create_exe_main.c b/lib/cli/src/commands/wasmer_create_exe_main.c index 3fe98f62c45..4fe6a7f9402 100644 --- a/lib/cli/src/commands/wasmer_create_exe_main.c +++ b/lib/cli/src/commands/wasmer_create_exe_main.c @@ -1,16 +1,24 @@ + #include "wasmer.h" -#include "static_defs.h" +//#include "my_wasm.h" + #include #include #include #define own -// TODO: make this define templated so that the Rust code can toggle it on/off -#define WASI -extern wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) asm("wasmer_object_module_new"); +#define WASI +#ifdef WASI_PIRITA +extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH"); +extern char VOLUMES_DATA asm("VOLUMES_DATA"); +// DECLARE_MODULES +#else +extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); +extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); +#endif static void print_wasmer_error() { int error_len = wasmer_last_error_length(); @@ -93,18 +101,54 @@ int main(int argc, char *argv[]) { wasm_engine_t *engine = wasm_engine_new_with_config(config); wasm_store_t *store = wasm_store_new(engine); - wasm_module_t *module = wasmer_object_module_new(store, "module"); +#ifdef WASI_PIRITA + // INSTANTIATE_MODULES +#else + wasm_byte_vec_t module_byte_vec = { + .size = WASMER_MODULE_LENGTH, + .data = &WASMER_MODULE_DATA, + }; + wasm_module_t *module = wasm_module_deserialize(store, &module_byte_vec); if (!module) { fprintf(stderr, "Failed to create module\n"); print_wasmer_error(); return -1; } +#endif // We have now finished the memory buffer book keeping and we have a valid // Module. -#ifdef WASI +#ifdef WASI_PIRITA + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasm_byte_vec_t volume_bytes = { + .size = VOLUMES_LENGTH, + .data = &VOLUMES_DATA, + }; + + wasi_filesystem_t* filesystem = wasi_filesystem_init_static_memory(&volume_bytes); + if (!filesystem) { + printf("Error parsing filesystem from bytes\n"); + return 1; + } + + wasm_extern_vec_t imports; + wasi_env_t* wasi_env = wasi_env_with_filesystem( + wasi_config, + store, + module, + filesystem, + &imports, + "##atom-name##" + ); + if (!wasi_env) { + printf("Error setting filesystem\n"); + return 1; + } +#else wasi_config_t *wasi_config = wasi_config_new(argv[0]); handle_arguments(wasi_config, argc, argv); @@ -114,7 +158,6 @@ int main(int argc, char *argv[]) { print_wasmer_error(); return 1; } -#endif wasm_importtype_vec_t import_types; wasm_module_imports(module, &import_types); @@ -132,6 +175,7 @@ int main(int argc, char *argv[]) { return 1; } +#endif #endif wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); @@ -179,6 +223,9 @@ int main(int argc, char *argv[]) { // TODO: handle non-WASI start (maybe with invoke?) +#ifdef WASI_PIRITA + wasi_filesystem_delete(filesystem); +#endif #ifdef WASI wasi_env_delete(wasi_env); wasm_extern_vec_delete(&exports); diff --git a/lib/cli/src/commands/wasmer_deserialize_module.h b/lib/cli/src/commands/wasmer_deserialize_module.h index f0ef229aa43..a94afd1149f 100644 --- a/lib/cli/src/commands/wasmer_deserialize_module.h +++ b/lib/cli/src/commands/wasmer_deserialize_module.h @@ -10,7 +10,7 @@ extern "C" { extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); -wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) { +wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* wasm_name) { wasm_byte_vec_t module_byte_vec = { .size = WASMER_MODULE_LENGTH, .data = (const char*)&WASMER_MODULE_DATA, diff --git a/lib/cli/src/commands/wasmer_static_create_exe_main.c b/lib/cli/src/commands/wasmer_static_create_exe_main.c new file mode 100644 index 00000000000..e2d21d8a529 --- /dev/null +++ b/lib/cli/src/commands/wasmer_static_create_exe_main.c @@ -0,0 +1,230 @@ +#include "wasmer.h" +#include "static_defs.h" +#include +#include +#include + +#define own + +// TODO: make this define templated so that the Rust code can toggle it on/off +#define WASI + +#ifdef WASI_PIRITA +extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH"); +extern char VOLUMES_DATA asm("VOLUMES_DATA"); +#endif + +extern wasm_module_t* wasmer_object_module_new(wasm_store_t* store,const char* wasm_name) asm("wasmer_object_module_new"); + +static void print_wasmer_error() { + int error_len = wasmer_last_error_length(); + printf("Error len: `%d`\n", error_len); + char *error_str = (char *)malloc(error_len); + wasmer_last_error_message(error_str, error_len); + printf("%s\n", error_str); + free(error_str); +} + +#ifdef WASI +static void pass_mapdir_arg(wasi_config_t *wasi_config, char *mapdir) { + int colon_location = strchr(mapdir, ':') - mapdir; + if (colon_location == 0) { + // error malformed argument + fprintf(stderr, "Expected mapdir argument of the form alias:directory\n"); + exit(-1); + } + + char *alias = (char *)malloc(colon_location + 1); + memcpy(alias, mapdir, colon_location); + alias[colon_location] = '\0'; + + int dir_len = strlen(mapdir) - colon_location; + char *dir = (char *)malloc(dir_len + 1); + memcpy(dir, &mapdir[colon_location + 1], dir_len); + dir[dir_len] = '\0'; + + wasi_config_mapdir(wasi_config, alias, dir); + free(alias); + free(dir); +} + +// We try to parse out `--dir` and `--mapdir` ahead of time and process those +// specially. All other arguments are passed to the guest program. +static void handle_arguments(wasi_config_t *wasi_config, int argc, + char *argv[]) { + for (int i = 1; i < argc; ++i) { + // We probably want special args like `--dir` and `--mapdir` to not be + // passed directly + if (strcmp(argv[i], "--dir") == 0) { + // next arg is a preopen directory + if ((i + 1) < argc) { + i++; + wasi_config_preopen_dir(wasi_config, argv[i]); + } else { + fprintf(stderr, "--dir expects a following argument specifying which " + "directory to preopen\n"); + exit(-1); + } + } else if (strcmp(argv[i], "--mapdir") == 0) { + // next arg is a mapdir + if ((i + 1) < argc) { + i++; + pass_mapdir_arg(wasi_config, argv[i]); + } else { + fprintf(stderr, + "--mapdir expects a following argument specifying which " + "directory to preopen in the form alias:directory\n"); + exit(-1); + } + } else if (strncmp(argv[i], "--dir=", strlen("--dir=")) == 0) { + // this arg is a preopen dir + char *dir = argv[i] + strlen("--dir="); + wasi_config_preopen_dir(wasi_config, dir); + } else if (strncmp(argv[i], "--mapdir=", strlen("--mapdir=")) == 0) { + // this arg is a mapdir + char *mapdir = argv[i] + strlen("--mapdir="); + pass_mapdir_arg(wasi_config, mapdir); + } else { + // guest argument + wasi_config_arg(wasi_config, argv[i]); + } + } +} +#endif + +int main(int argc, char *argv[]) { + wasm_config_t *config = wasm_config_new(); + wasm_engine_t *engine = wasm_engine_new_with_config(config); + wasm_store_t *store = wasm_store_new(engine); + + #ifdef WASI_PIRITA + // INSTANTIATE_MODULES + #else + wasm_module_t *module = wasmer_object_module_new(store, "module"); + #endif + + if (!module) { + fprintf(stderr, "Failed to create module\n"); + print_wasmer_error(); + return -1; + } + + // We have now finished the memory buffer book keeping and we have a valid + // Module. + +#ifdef WASI_PIRITA + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasm_byte_vec_t volume_bytes = { + .size = VOLUMES_LENGTH, + .data = &VOLUMES_DATA, + }; + + wasi_filesystem_t* filesystem = wasi_filesystem_init_static_memory(&volume_bytes); + if (!filesystem) { + printf("Error parsing filesystem from bytes\n"); + return 1; + } + + wasm_extern_vec_t imports; + wasi_env_t* wasi_env = wasi_env_with_filesystem( + wasi_config, + store, + module, + filesystem, + &imports, + "##atom-name##" + ); + if (!wasi_env) { + printf("Error setting filesystem\n"); + return 1; + } +#else + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasi_env_t *wasi_env = wasi_env_new(store, wasi_config); + if (!wasi_env) { + fprintf(stderr, "Error building WASI env!\n"); + print_wasmer_error(); + return 1; + } + + wasm_importtype_vec_t import_types; + wasm_module_imports(module, &import_types); + + wasm_extern_vec_t imports; + wasm_extern_vec_new_uninitialized(&imports, import_types.size); + wasm_importtype_vec_delete(&import_types); + +#ifdef WASI + bool get_imports_result = wasi_get_imports(store, wasi_env, module, &imports); + + if (!get_imports_result) { + fprintf(stderr, "Error getting WASI imports!\n"); + print_wasmer_error(); + + return 1; + } +#endif +#endif + + wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); + + if (!instance) { + fprintf(stderr, "Failed to create instance\n"); + print_wasmer_error(); + return -1; + } + +#ifdef WASI + // Read the exports. + wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + wasm_memory_t* mem = NULL; + for (size_t i = 0; i < exports.size; i++) { + mem = wasm_extern_as_memory(exports.data[i]); + if (mem) { + break; + } + } + + if (!mem) { + fprintf(stderr, "Failed to create instance: Could not find memory in exports\n"); + print_wasmer_error(); + return -1; + } + wasi_env_set_memory(wasi_env, mem); + + own wasm_func_t *start_function = wasi_get_start_function(instance); + if (!start_function) { + fprintf(stderr, "`_start` function not found\n"); + print_wasmer_error(); + return -1; + } + + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_EMPTY_VEC; + own wasm_trap_t *trap = wasm_func_call(start_function, &args, &results); + if (trap) { + fprintf(stderr, "Trap is not NULL: TODO:\n"); + return -1; + } +#endif + + // TODO: handle non-WASI start (maybe with invoke?) + +#ifdef WASI_PIRITA + wasi_filesystem_delete(filesystem); +#endif +#ifdef WASI + wasi_env_delete(wasi_env); + wasm_extern_vec_delete(&exports); +#endif + wasm_instance_delete(instance); + wasm_module_delete(module); + wasm_store_delete(store); + wasm_engine_delete(engine); + return 0; +} diff --git a/lib/object/src/module.rs b/lib/object/src/module.rs index 40cb005af2c..ad20961acb4 100644 --- a/lib/object/src/module.rs +++ b/lib/object/src/module.rs @@ -392,7 +392,7 @@ pub fn emit_compilation( /// ```rust /// # use wasmer_types::SymbolRegistry; /// # use wasmer_types::{Compilation, Triple}; -/// # use wasmer_object::ObjectError; +/// # use wasmer_object::{ObjectError, emit_serialized}; /// use wasmer_object::{get_object_for_target, emit_compilation}; /// /// # fn emit_compilation_into_object( @@ -400,8 +400,9 @@ pub fn emit_compilation( /// # compilation: Compilation, /// # symbol_registry: impl SymbolRegistry, /// # ) -> Result<(), ObjectError> { +/// let bytes = &[ /* compilation bytes */]; /// let mut object = get_object_for_target(&triple)?; -/// emit_compilation(&mut object, compilation, &symbol_registry, &triple)?; +/// emit_serialized(&mut object, bytes, &triple, "WASMER_MODULE")?; /// # Ok(()) /// # } /// ``` @@ -409,11 +410,12 @@ pub fn emit_serialized( obj: &mut Object, sercomp: &[u8], triple: &Triple, + object_name: &str, ) -> Result<(), ObjectError> { obj.set_mangling(object::write::Mangling::None); //let module_name = module.compile_info.module.name.clone(); - let len_name = "WASMER_MODULE_LENGTH"; - let data_name = "WASMER_MODULE_DATA"; + let len_name = format!("{}_LENGTH", object_name); + let data_name = format!("{}_DATA", object_name); //let metadata_name = "WASMER_MODULE_METADATA"; let align = match triple.architecture { diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index 1b35527c500..ffca0feb87e 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -19,4 +19,5 @@ toml = "0.5.9" wapm-toml = "0.2.0" tar = "0.4.38" flate2 = "1.0.24" -semver = "1.0.14" \ No newline at end of file +semver = "1.0.14" +lzma-rs = "0.2.0" \ No newline at end of file diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index 61d4cd86fcd..267755a4766 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -878,8 +878,11 @@ pub fn query_package_from_registry( } } +pub fn get_wasmer_root_dir() -> Option { + PartialWapmConfig::get_folder().ok() +} pub fn get_checkouts_dir() -> Option { - Some(PartialWapmConfig::get_folder().ok()?.join("checkouts")) + Some(get_wasmer_root_dir()?.join("checkouts")) } /// Returs the path to the directory where all packages on this computer are being stored @@ -887,7 +890,12 @@ pub fn get_global_install_dir(registry_host: &str) -> Option { Some(get_checkouts_dir()?.join(registry_host)) } -pub fn download_and_unpack_targz(url: &str, target_path: &Path) -> Result { +/// Whether the top-level directory should be stripped +pub fn download_and_unpack_targz( + url: &str, + target_path: &Path, + strip_toplevel: bool, +) -> Result { let target_targz_path = target_path.to_path_buf().join("package.tar.gz"); let mut resp = @@ -907,28 +915,75 @@ pub fn download_and_unpack_targz(url: &str, target_path: &Path) -> Result = Vec::new(); + let mut bufread = std::io::BufReader::new(&file); + lzma_rs::xz_decompress(&mut bufread, &mut decomp) + .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display()))?; + + let cursor = std::io::Cursor::new(decomp); + let mut ar = tar::Archive::new(cursor); + if strip_toplevel { + unpack_sans_parent(ar, target_path) + .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display())) + } else { + ar.unpack(target_path) + .map_err(|e| format!("failed to unpack {}: {e}", target_targz_path.display())) + } + }; + + try_decode_gz().or_else(|_| try_decode_xz())?; let _ = std::fs::remove_file(target_targz_path); Ok(target_path.to_path_buf()) } +pub fn unpack_sans_parent(mut archive: tar::Archive, dst: &Path) -> std::io::Result<()> +where + R: std::io::Read, +{ + use std::path::Component::Normal; + + for entry in archive.entries()? { + let mut entry = entry?; + let path: PathBuf = entry + .path()? + .components() + .skip(1) // strip top-level directory + .filter(|c| matches!(c, Normal(_))) // prevent traversal attacks + .collect(); + entry.unpack(dst.join(path))?; + } + Ok(()) +} + /// Given a triple of [registry, name, version], downloads and installs the /// .tar.gz if it doesn't yet exist, returns the (package dir, entrypoint .wasm file path) pub fn install_package( @@ -1035,7 +1090,7 @@ pub fn install_package( let name = package_info.package; if !dir.join("wapm.toml").exists() || force_install { - download_and_unpack_targz(&package_info.url, &dir)?; + download_and_unpack_targz(&package_info.url, &dir, false)?; } Ok(( diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 9b9ddfb4f8c..47953e0a9a9 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -13,11 +13,15 @@ tracing = { version = "0.1" } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } slab = { version = "0.4", optional = true } +webc = { version = "3.0.1", optional = true } +anyhow = { version = "1.0.66", optional = true } [features] -default = ["host-fs", "mem-fs"] +default = ["host-fs", "mem-fs", "webc-fs", "static-fs"] host-fs = ["libc"] mem-fs = ["slab"] +webc-fs = ["webc", "anyhow"] +static-fs = ["webc", "anyhow", "mem-fs"] enable-serde = [ "serde", "typetag" diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index d7be3ac4ae6..469e8319fcf 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -15,6 +15,10 @@ compile_error!("At least the `host-fs` or the `mem-fs` feature must be enabled. pub mod host_fs; #[cfg(feature = "mem-fs")] pub mod mem_fs; +#[cfg(feature = "static-fs")] +pub mod static_fs; +#[cfg(feature = "webc-fs")] +pub mod webc_fs; pub type Result = std::result::Result; diff --git a/lib/vfs/src/mem_fs/filesystem.rs b/lib/vfs/src/mem_fs/filesystem.rs index 6943c7f20a5..233577db6ed 100644 --- a/lib/vfs/src/mem_fs/filesystem.rs +++ b/lib/vfs/src/mem_fs/filesystem.rs @@ -1190,7 +1190,7 @@ mod test_filesystem { name, children, .. - }) if name == "/" && children == &[] + }) if name == "/" && children.is_empty() ), "`/` is empty", ); diff --git a/lib/vfs/src/static_fs.rs b/lib/vfs/src/static_fs.rs new file mode 100644 index 00000000000..6dca716deef --- /dev/null +++ b/lib/vfs/src/static_fs.rs @@ -0,0 +1,384 @@ +use anyhow::anyhow; + +use std::convert::TryInto; +use std::io::{Error as IoError, ErrorKind as IoErrorKind, Read, Seek, SeekFrom, Write}; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::mem_fs::FileSystem as MemFileSystem; +use crate::{ + FileDescriptor, FileOpener, FileSystem, FsError, Metadata, OpenOptions, OpenOptionsConfig, + ReadDir, VirtualFile, +}; +use webc::{FsEntry, FsEntryType, OwnedFsEntryFile}; + +/// Custom file system wrapper to map requested file paths +#[derive(Debug)] +pub struct StaticFileSystem { + pub package: String, + pub volumes: Arc>>, + pub memory: Arc, +} + +impl StaticFileSystem { + pub fn init(bytes: &'static [u8], package: &str) -> Option { + let volumes = Arc::new(webc::WebC::parse_volumes_from_fileblock(bytes).ok()?); + let fs = Self { + package: package.to_string(), + volumes: volumes.clone(), + memory: Arc::new(MemFileSystem::default()), + }; + let volume_names = fs.volumes.keys().cloned().collect::>(); + for volume_name in volume_names { + let directories = volumes.get(&volume_name).unwrap().list_directories(); + for directory in directories { + let _ = fs.create_dir(Path::new(&directory)); + } + } + Some(fs) + } +} + +/// Custom file opener, returns a WebCFile +#[derive(Debug)] +struct WebCFileOpener { + pub package: String, + pub volumes: Arc>>, + pub memory: Arc, +} + +impl FileOpener for WebCFileOpener { + fn open( + &mut self, + path: &Path, + _conf: &OpenOptionsConfig, + ) -> Result, FsError> { + match get_volume_name_opt(path) { + Some(volume) => { + let file = (*self.volumes) + .get(&volume) + .ok_or(FsError::EntityNotFound)? + .get_file_entry(&format!("{}", path.display())) + .map_err(|_e| FsError::EntityNotFound)?; + + Ok(Box::new(WebCFile { + package: self.package.clone(), + volume, + volumes: self.volumes.clone(), + path: path.to_path_buf(), + entry: file, + cursor: 0, + })) + } + None => { + for (volume, v) in self.volumes.iter() { + let entry = match v.get_file_entry(&format!("{}", path.display())) { + Ok(s) => s, + Err(_) => continue, // error + }; + + return Ok(Box::new(WebCFile { + package: self.package.clone(), + volume: volume.clone(), + volumes: self.volumes.clone(), + path: path.to_path_buf(), + entry, + cursor: 0, + })); + } + self.memory.new_open_options().open(path) + } + } + } +} + +#[derive(Debug)] +pub struct WebCFile { + pub volumes: Arc>>, + pub package: String, + pub volume: String, + pub path: PathBuf, + pub entry: OwnedFsEntryFile, + pub cursor: u64, +} + +impl VirtualFile for WebCFile { + fn last_accessed(&self) -> u64 { + 0 + } + fn last_modified(&self) -> u64 { + 0 + } + fn created_time(&self) -> u64 { + 0 + } + fn size(&self) -> u64 { + self.entry.get_len() + } + fn set_len(&mut self, _new_size: u64) -> Result<(), FsError> { + Ok(()) + } + fn unlink(&mut self) -> Result<(), FsError> { + Ok(()) + } + fn bytes_available(&self) -> Result { + Ok(self.size().try_into().unwrap_or(u32::MAX as usize)) + } + fn sync_to_disk(&self) -> Result<(), FsError> { + Ok(()) + } + fn get_fd(&self) -> Option { + None + } +} + +impl Read for WebCFile { + fn read(&mut self, buf: &mut [u8]) -> Result { + let bytes = self + .volumes + .get(&self.volume) + .ok_or_else(|| { + IoError::new( + IoErrorKind::NotFound, + anyhow!("Unknown volume {:?}", self.volume), + ) + })? + .get_file_bytes(&self.entry) + .map_err(|e| IoError::new(IoErrorKind::NotFound, e))?; + + let cursor: usize = self.cursor.try_into().unwrap_or(u32::MAX as usize); + let _start = cursor.min(bytes.len()); + let bytes = &bytes[cursor..]; + + let mut len = 0; + for (source, target) in bytes.iter().zip(buf.iter_mut()) { + *target = *source; + len += 1; + } + + Ok(len) + } +} + +// WebC file is not writable, the FileOpener will return a MemoryFile for writing instead +// This code should never be executed (since writes are redirected to memory instead). +impl Write for WebCFile { + fn write(&mut self, buf: &[u8]) -> Result { + Ok(buf.len()) + } + fn flush(&mut self) -> Result<(), IoError> { + Ok(()) + } +} + +impl Seek for WebCFile { + fn seek(&mut self, pos: SeekFrom) -> Result { + let self_size = self.size(); + match pos { + SeekFrom::Start(s) => { + self.cursor = s.min(self_size); + } + SeekFrom::End(e) => { + let self_size_i64 = self_size.try_into().unwrap_or(i64::MAX); + self.cursor = ((self_size_i64).saturating_add(e)) + .min(self_size_i64) + .try_into() + .unwrap_or(i64::MAX as u64); + } + SeekFrom::Current(c) => { + self.cursor = (self + .cursor + .saturating_add(c.try_into().unwrap_or(i64::MAX as u64))) + .min(self_size); + } + } + Ok(self.cursor) + } +} + +fn get_volume_name_opt>(path: P) -> Option { + use std::path::Component::Normal; + if let Some(Normal(n)) = path.as_ref().components().next() { + if let Some(s) = n.to_str() { + if s.ends_with(':') { + return Some(s.replace(':', "")); + } + } + } + None +} + +fn transform_into_read_dir<'a>(path: &Path, fs_entries: &[FsEntry<'a>]) -> crate::ReadDir { + let entries = fs_entries + .iter() + .map(|e| crate::DirEntry { + path: path.join(&*e.text), + metadata: Ok(crate::Metadata { + ft: translate_file_type(e.fs_type), + accessed: 0, + created: 0, + modified: 0, + len: e.get_len(), + }), + }) + .collect(); + + crate::ReadDir::new(entries) +} + +impl FileSystem for StaticFileSystem { + fn read_dir(&self, path: &Path) -> Result { + let path = normalizes_path(path); + for volume in self.volumes.values() { + let read_dir_result = volume + .read_dir(&path) + .map(|o| transform_into_read_dir(Path::new(&path), o.as_ref())) + .map_err(|_| FsError::EntityNotFound); + + match read_dir_result { + Ok(o) => { + return Ok(o); + } + Err(_) => { + continue; + } + } + } + + self.memory.read_dir(Path::new(&path)) + } + fn create_dir(&self, path: &Path) -> Result<(), FsError> { + let path = normalizes_path(path); + let result = self.memory.create_dir(Path::new(&path)); + result + } + fn remove_dir(&self, path: &Path) -> Result<(), FsError> { + let path = normalizes_path(path); + let result = self.memory.remove_dir(Path::new(&path)); + if self + .volumes + .values() + .find_map(|v| v.get_file_entry(&path).ok()) + .is_some() + { + Ok(()) + } else { + result + } + } + fn rename(&self, from: &Path, to: &Path) -> Result<(), FsError> { + let from = normalizes_path(from); + let to = normalizes_path(to); + let result = self.memory.rename(Path::new(&from), Path::new(&to)); + if self + .volumes + .values() + .find_map(|v| v.get_file_entry(&from).ok()) + .is_some() + { + Ok(()) + } else { + result + } + } + fn metadata(&self, path: &Path) -> Result { + let path = normalizes_path(path); + if let Some(fs_entry) = self + .volumes + .values() + .find_map(|v| v.get_file_entry(&path).ok()) + { + Ok(Metadata { + ft: translate_file_type(FsEntryType::File), + accessed: 0, + created: 0, + modified: 0, + len: fs_entry.get_len(), + }) + } else if let Some(_fs) = self.volumes.values().find_map(|v| v.read_dir(&path).ok()) { + Ok(Metadata { + ft: translate_file_type(FsEntryType::Dir), + accessed: 0, + created: 0, + modified: 0, + len: 0, + }) + } else { + self.memory.metadata(Path::new(&path)) + } + } + fn remove_file(&self, path: &Path) -> Result<(), FsError> { + let path = normalizes_path(path); + let result = self.memory.remove_file(Path::new(&path)); + if self + .volumes + .values() + .find_map(|v| v.get_file_entry(&path).ok()) + .is_some() + { + Ok(()) + } else { + result + } + } + fn new_open_options(&self) -> OpenOptions { + OpenOptions::new(Box::new(WebCFileOpener { + package: self.package.clone(), + volumes: self.volumes.clone(), + memory: self.memory.clone(), + })) + } + fn symlink_metadata(&self, path: &Path) -> Result { + let path = normalizes_path(path); + if let Some(fs_entry) = self + .volumes + .values() + .find_map(|v| v.get_file_entry(&path).ok()) + { + Ok(Metadata { + ft: translate_file_type(FsEntryType::File), + accessed: 0, + created: 0, + modified: 0, + len: fs_entry.get_len(), + }) + } else if self + .volumes + .values() + .find_map(|v| v.read_dir(&path).ok()) + .is_some() + { + Ok(Metadata { + ft: translate_file_type(FsEntryType::Dir), + accessed: 0, + created: 0, + modified: 0, + len: 0, + }) + } else { + self.memory.symlink_metadata(Path::new(&path)) + } + } +} + +fn normalizes_path(path: &Path) -> String { + let path = format!("{}", path.display()); + if !path.starts_with('/') { + format!("/{path}") + } else { + path + } +} + +fn translate_file_type(f: FsEntryType) -> crate::FileType { + crate::FileType { + dir: f == FsEntryType::Dir, + file: f == FsEntryType::File, + symlink: false, + char_device: false, + block_device: false, + socket: false, + fifo: false, + } +} diff --git a/lib/vfs/src/webc_fs.rs b/lib/vfs/src/webc_fs.rs new file mode 100644 index 00000000000..8111157b1b5 --- /dev/null +++ b/lib/vfs/src/webc_fs.rs @@ -0,0 +1,395 @@ +use crate::mem_fs::FileSystem as MemFileSystem; +use crate::{ + FileDescriptor, FileOpener, FileSystem, FsError, Metadata, OpenOptions, OpenOptionsConfig, + ReadDir, VirtualFile, +}; +use anyhow::anyhow; +use std::convert::TryInto; +use std::io::{Error as IoError, ErrorKind as IoErrorKind, Read, Seek, SeekFrom, Write}; +use std::ops::Deref; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use webc::{FsEntry, FsEntryType, OwnedFsEntryFile, WebC}; + +/// Custom file system wrapper to map requested file paths +#[derive(Debug)] +pub struct WebcFileSystem +where + T: std::fmt::Debug + Send + Sync + 'static, +{ + pub package: String, + pub webc: Arc, + pub memory: Arc, +} + +impl<'a, T> WebcFileSystem +where + T: std::fmt::Debug + Send + Sync + 'static, + T: Deref>, +{ + pub fn init(webc: Arc, package: &str) -> Self { + let fs = Self { + package: package.to_string(), + webc: webc.clone(), + memory: Arc::new(MemFileSystem::default()), + }; + for volume in webc.get_volumes_for_package(package) { + for directory in webc.list_directories(&volume) { + let _ = fs.create_dir(Path::new(&directory)); + } + } + fs + } +} + +/// Custom file opener, returns a WebCFile +#[derive(Debug)] +struct WebCFileOpener +where + T: std::fmt::Debug + Send + Sync + 'static, +{ + pub package: String, + pub webc: Arc, + pub memory: Arc, +} + +impl<'a, T> FileOpener for WebCFileOpener +where + T: std::fmt::Debug + Send + Sync + 'static, + T: Deref>, +{ + fn open( + &mut self, + path: &Path, + _conf: &OpenOptionsConfig, + ) -> Result, FsError> { + match get_volume_name_opt(path) { + Some(volume) => { + let file = (*self.webc) + .volumes + .get(&volume) + .ok_or(FsError::EntityNotFound)? + .get_file_entry(&format!("{}", path.display())) + .map_err(|_e| FsError::EntityNotFound)?; + + Ok(Box::new(WebCFile { + package: self.package.clone(), + volume, + webc: self.webc.clone(), + path: path.to_path_buf(), + entry: file, + cursor: 0, + })) + } + None => { + for volume in self.webc.get_volumes_for_package(&self.package) { + let v = match self.webc.volumes.get(&volume) { + Some(s) => s, + None => continue, // error + }; + + let entry = match v.get_file_entry(&format!("{}", path.display())) { + Ok(s) => s, + Err(_) => continue, // error + }; + + return Ok(Box::new(WebCFile { + package: self.package.clone(), + volume: volume.clone(), + webc: self.webc.clone(), + path: path.to_path_buf(), + entry, + cursor: 0, + })); + } + self.memory.new_open_options().open(path) + } + } + } +} + +#[derive(Debug)] +struct WebCFile +where + T: std::fmt::Debug + Send + Sync + 'static, +{ + pub webc: Arc, + #[allow(dead_code)] + pub package: String, + pub volume: String, + #[allow(dead_code)] + pub path: PathBuf, + pub entry: OwnedFsEntryFile, + pub cursor: u64, +} + +impl VirtualFile for WebCFile +where + T: std::fmt::Debug + Send + Sync + 'static, + T: Deref>, +{ + fn last_accessed(&self) -> u64 { + 0 + } + fn last_modified(&self) -> u64 { + 0 + } + fn created_time(&self) -> u64 { + 0 + } + fn size(&self) -> u64 { + self.entry.get_len() + } + fn set_len(&mut self, _new_size: u64) -> Result<(), FsError> { + Ok(()) + } + fn unlink(&mut self) -> Result<(), FsError> { + Ok(()) + } + fn bytes_available(&self) -> Result { + Ok(self.size().try_into().unwrap_or(u32::MAX as usize)) + } + fn sync_to_disk(&self) -> Result<(), FsError> { + Ok(()) + } + fn get_fd(&self) -> Option { + None + } +} + +impl Read for WebCFile +where + T: std::fmt::Debug + Send + Sync + 'static, + T: Deref>, +{ + fn read(&mut self, buf: &mut [u8]) -> Result { + let bytes = self + .webc + .volumes + .get(&self.volume) + .ok_or_else(|| { + IoError::new( + IoErrorKind::NotFound, + anyhow!("Unknown volume {:?}", self.volume), + ) + })? + .get_file_bytes(&self.entry) + .map_err(|e| IoError::new(IoErrorKind::NotFound, e))?; + + let cursor: usize = self.cursor.try_into().unwrap_or(u32::MAX as usize); + let _start = cursor.min(bytes.len()); + let bytes = &bytes[cursor..]; + + let mut len = 0; + for (source, target) in bytes.iter().zip(buf.iter_mut()) { + *target = *source; + len += 1; + } + + Ok(len) + } +} + +// WebC file is not writable, the FileOpener will return a MemoryFile for writing instead +// This code should never be executed (since writes are redirected to memory instead). +impl Write for WebCFile +where + T: std::fmt::Debug + Send + Sync + 'static, +{ + fn write(&mut self, buf: &[u8]) -> Result { + Ok(buf.len()) + } + fn flush(&mut self) -> Result<(), IoError> { + Ok(()) + } +} + +impl Seek for WebCFile +where + T: std::fmt::Debug + Send + Sync + 'static, + T: Deref>, +{ + fn seek(&mut self, pos: SeekFrom) -> Result { + let self_size = self.size(); + match pos { + SeekFrom::Start(s) => { + self.cursor = s.min(self_size); + } + SeekFrom::End(e) => { + let self_size_i64 = self_size.try_into().unwrap_or(i64::MAX); + self.cursor = ((self_size_i64).saturating_add(e)) + .min(self_size_i64) + .try_into() + .unwrap_or(i64::MAX as u64); + } + SeekFrom::Current(c) => { + self.cursor = (self + .cursor + .saturating_add(c.try_into().unwrap_or(i64::MAX as u64))) + .min(self_size); + } + } + Ok(self.cursor) + } +} + +fn get_volume_name_opt>(path: P) -> Option { + use std::path::Component::Normal; + if let Some(Normal(n)) = path.as_ref().components().next() { + if let Some(s) = n.to_str() { + if s.ends_with(':') { + return Some(s.replace(':', "")); + } + } + } + None +} + +#[allow(dead_code)] +fn get_volume_name>(path: P) -> String { + get_volume_name_opt(path).unwrap_or_else(|| "atom".to_string()) +} + +fn transform_into_read_dir<'a>(path: &Path, fs_entries: &[FsEntry<'a>]) -> crate::ReadDir { + let entries = fs_entries + .iter() + .map(|e| crate::DirEntry { + path: path.join(&*e.text), + metadata: Ok(crate::Metadata { + ft: translate_file_type(e.fs_type), + accessed: 0, + created: 0, + modified: 0, + len: e.get_len(), + }), + }) + .collect(); + + crate::ReadDir::new(entries) +} + +impl FileSystem for WebcFileSystem +where + T: std::fmt::Debug + Send + Sync + 'static, + T: Deref>, +{ + fn read_dir(&self, path: &Path) -> Result { + let path = normalizes_path(path); + let read_dir_result = self + .webc + .read_dir(&self.package, &path) + .map(|o| transform_into_read_dir(Path::new(&path), o.as_ref())) + .map_err(|_| FsError::EntityNotFound); + + match read_dir_result { + Ok(o) => Ok(o), + Err(_) => self.memory.read_dir(Path::new(&path)), + } + } + fn create_dir(&self, path: &Path) -> Result<(), FsError> { + let path = normalizes_path(path); + let result = self.memory.create_dir(Path::new(&path)); + result + } + fn remove_dir(&self, path: &Path) -> Result<(), FsError> { + let path = normalizes_path(path); + let result = self.memory.remove_dir(Path::new(&path)); + if self.webc.get_file_entry(&self.package, &path).is_some() { + Ok(()) + } else { + result + } + } + fn rename(&self, from: &Path, to: &Path) -> Result<(), FsError> { + let from = normalizes_path(from); + let to = normalizes_path(to); + let result = self.memory.rename(Path::new(&from), Path::new(&to)); + if self.webc.get_file_entry(&self.package, &from).is_some() { + Ok(()) + } else { + result + } + } + fn metadata(&self, path: &Path) -> Result { + let path = normalizes_path(path); + if let Some(fs_entry) = self.webc.get_file_entry(&self.package, &path) { + Ok(Metadata { + ft: translate_file_type(FsEntryType::File), + accessed: 0, + created: 0, + modified: 0, + len: fs_entry.1.get_len(), + }) + } else if self.webc.read_dir(&self.package, &path).is_ok() { + Ok(Metadata { + ft: translate_file_type(FsEntryType::Dir), + accessed: 0, + created: 0, + modified: 0, + len: 0, + }) + } else { + self.memory.metadata(Path::new(&path)) + } + } + fn remove_file(&self, path: &Path) -> Result<(), FsError> { + let path = normalizes_path(path); + let result = self.memory.remove_file(Path::new(&path)); + if self.webc.get_file_entry(&self.package, &path).is_some() { + Ok(()) + } else { + result + } + } + fn new_open_options(&self) -> OpenOptions { + OpenOptions::new(Box::new(WebCFileOpener { + package: self.package.clone(), + webc: self.webc.clone(), + memory: self.memory.clone(), + })) + } + fn symlink_metadata(&self, path: &Path) -> Result { + let path = normalizes_path(path); + if let Some(fs_entry) = self.webc.get_file_entry(&self.package, &path) { + Ok(Metadata { + ft: translate_file_type(FsEntryType::File), + accessed: 0, + created: 0, + modified: 0, + len: fs_entry.1.get_len(), + }) + } else if self.webc.read_dir(&self.package, &path).is_ok() { + Ok(Metadata { + ft: translate_file_type(FsEntryType::Dir), + accessed: 0, + created: 0, + modified: 0, + len: 0, + }) + } else { + self.memory.symlink_metadata(Path::new(&path)) + } + } +} + +fn normalizes_path(path: &Path) -> String { + let path = format!("{}", path.display()); + if !path.starts_with('/') { + format!("/{path}") + } else { + path + } +} + +fn translate_file_type(f: FsEntryType) -> crate::FileType { + crate::FileType { + dir: f == FsEntryType::Dir, + file: f == FsEntryType::File, + symlink: false, + char_device: false, + block_device: false, + socket: false, + fifo: false, + } +} diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index c8c8220a214..f5b8b6b5851 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -28,6 +28,10 @@ bincode = { version = "1.3", optional = true } chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" +webc = { version = "3.0.1", optional = true, default-features = false, features = ["std", "mmap"] } +serde_cbor = { version = "0.11.2", optional = true } +anyhow = { version = "1.0.66", optional = true } +wasmer-emscripten = { path = "../emscripten", version = "=3.0.0-rc.1", optional = true } [target.'cfg(unix)'.dependencies] libc = { version = "^0.2", default-features = false } @@ -46,6 +50,10 @@ tracing-wasm = "0.2" default = ["sys-default"] wasix = [] +webc_runner = ["webc", "serde_cbor", "anyhow", "serde", "wasmer/compiler", "wasmer/cranelift"] +webc_runner_rt_emscripten = ["wasmer-emscripten"] +webc_runner_rt_wasi = [] + sys = ["wasmer/sys", "wasix", "wasmer-wasi-types/sys"] sys-default = ["wasmer/wat", "wasmer/compiler", "sys", "logging", "host-fs", "sys-poll", "host-vnet" ] sys-poll = [] diff --git a/lib/wasi/src/lib.rs b/lib/wasi/src/lib.rs index 97a79b0d5ed..81a77fc7fcd 100644 --- a/lib/wasi/src/lib.rs +++ b/lib/wasi/src/lib.rs @@ -40,6 +40,10 @@ mod state; mod syscalls; mod utils; +/// Runners for WASI / Emscripten +#[cfg(feature = "webc_runner")] +pub mod runners; + use crate::syscalls::*; pub use crate::state::{ diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs new file mode 100644 index 00000000000..ab568a62dba --- /dev/null +++ b/lib/wasi/src/runners/emscripten.rs @@ -0,0 +1,111 @@ +#![cfg(feature = "webc_runner_rt_wasi")] +//! WebC container support for running Emscripten modules + +use crate::runners::WapmContainer; +use anyhow::anyhow; +use serde::{Deserialize, Serialize}; +use std::error::Error as StdError; +use std::sync::Arc; +use wasmer::{Cranelift, FunctionEnv, Instance, Module, Store}; +use wasmer_emscripten::{ + generate_emscripten_env, is_emscripten_module, run_emscripten_instance, EmEnv, + EmscriptenGlobals, +}; +use webc::{Command, WebCMmap}; + +#[derive(Debug, Default, Clone, PartialEq, PartialOrd, Hash, Serialize, Deserialize)] +pub struct EmscriptenRunner { + args: Vec, +} + +impl EmscriptenRunner { + pub fn set_args(&mut self, args: Vec) { + self.args = args; + } +} + +impl crate::runners::Runner for EmscriptenRunner { + type Output = (); + + fn can_run_command(&self, _: &str, command: &Command) -> Result> { + Ok(command + .runner + .starts_with("https://webc.org/runner/emscripten")) + } + + fn run_command( + &mut self, + command_name: &str, + _command: &Command, + container: &WapmContainer, + ) -> Result> { + let atom_name = container.get_atom_name_for_command("emscripten", command_name)?; + let main_args = container.get_main_args_for_command(command_name); + let atom_bytes = container.get_atom(&container.get_package_name(), &atom_name)?; + + let compiler = Cranelift::default(); + let mut store = Store::new(compiler); + let mut module = Module::new(&store, atom_bytes)?; + module.set_name(&atom_name); + + let (mut globals, env) = + prepare_emscripten_env(&mut store, &module, container.webc.clone(), &atom_name)?; + + exec_module( + &mut store, + &module, + &mut globals, + env, + container.webc.clone(), + &atom_name, + main_args.unwrap_or_default(), + )?; + + Ok(()) + } +} + +fn prepare_emscripten_env( + store: &mut Store, + module: &Module, + _atom: Arc, + name: &str, +) -> Result<(EmscriptenGlobals, FunctionEnv), anyhow::Error> { + if !is_emscripten_module(module) { + return Err(anyhow!("Atom {name:?} is not an emscripten module")); + } + + let env = FunctionEnv::new(store, EmEnv::new()); + let emscripten_globals = EmscriptenGlobals::new(store, &env, module); + let emscripten_globals = emscripten_globals.map_err(|e| anyhow!("{}", e))?; + env.as_mut(store) + .set_data(&emscripten_globals.data, Default::default()); + + Ok((emscripten_globals, env)) +} + +fn exec_module( + store: &mut Store, + module: &Module, + globals: &mut EmscriptenGlobals, + em_env: FunctionEnv, + _atom: Arc, + name: &str, + args: Vec, +) -> Result<(), anyhow::Error> { + let import_object = generate_emscripten_env(store, &em_env, globals); + + let mut instance = Instance::new(store, module, &import_object) + .map_err(|e| anyhow!("Cant instantiate emscripten module {name:?}: {e}"))?; + + run_emscripten_instance( + &mut instance, + em_env.into_mut(store), + globals, + name, + args.iter().map(|arg| arg.as_str()).collect(), + None, + )?; + + Ok(()) +} diff --git a/lib/wasi/src/runners/mod.rs b/lib/wasi/src/runners/mod.rs new file mode 100644 index 00000000000..4cac42921f2 --- /dev/null +++ b/lib/wasi/src/runners/mod.rs @@ -0,0 +1,208 @@ +#![cfg(feature = "webc_runner")] + +use std::error::Error as StdError; +use std::path::PathBuf; +use std::sync::Arc; +use webc::*; + +pub mod emscripten; +pub mod wasi; + +/// Parsed WAPM file, memory-mapped to an on-disk path +#[derive(Debug, Clone)] +pub struct WapmContainer { + /// WebC container + pub webc: Arc, +} + +impl core::ops::Deref for WapmContainer { + type Target = webc::WebC<'static>; + fn deref<'a>(&'a self) -> &WebC<'static> { + &self.webc.webc + } +} + +/// Error that ocurred while parsing the .webc file +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum WebcParseError { + /// Parse error + Parse(webc::Error), +} + +impl From for WebcParseError { + fn from(e: webc::Error) -> Self { + WebcParseError::Parse(e) + } +} + +impl WapmContainer { + /// Parses a .webc container file. Since .webc files + /// can be very large, only file paths are allowed. + pub fn new(path: PathBuf) -> std::result::Result { + let webc = webc::WebCMmap::parse(path, &webc::ParseOptions::default())?; + Ok(Self { + webc: Arc::new(webc), + }) + } + + /// Returns the bytes of a file or a stringified error + pub fn get_file<'b>(&'b self, path: &str) -> Result<&'b [u8], String> { + self.webc + .get_file(&self.webc.get_package_name(), path) + .map_err(|e| e.0) + } + + /// Returns a list of volumes in this container + pub fn get_volumes(&self) -> Vec { + self.webc.volumes.keys().cloned().collect::>() + } + + /// Lookup .wit bindings by name and parse them + pub fn get_bindings( + &self, + bindings: &str, + ) -> std::result::Result { + let bindings = self + .webc + .manifest + .bindings + .iter() + .find(|b| b.name == bindings) + .ok_or_else(|| ParseBindingsError::NoBindings(bindings.to_string()))?; + + T::parse_bindings(self, &bindings.annotations).map_err(ParseBindingsError::ParseBindings) + } +} + +/// Error that happened while parsing .wit bindings +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)] +pub enum ParseBindingsError { + /// No bindings are available for the given lookup key + NoBindings(String), + /// Error happened during parsing + ParseBindings(String), +} + +/// Trait to parse bindings (any kind of bindings) for container .wasm files (usually .wit format) +pub trait Bindings { + /// Function that takes annotations in a free-form `Value` struct and returns the parsed bindings or an error + fn parse_bindings(_: &WapmContainer, _: &serde_cbor::Value) -> Result + where + Self: Sized; +} + +/// WIT bindings +#[derive(Default, Debug, Copy, Clone)] +pub struct WitBindings {} + +impl WitBindings { + /// Unused: creates default wit bindings + pub fn parse(_s: &str) -> Result { + Ok(Self::default()) + } +} + +impl Bindings for WitBindings { + fn parse_bindings( + container: &WapmContainer, + value: &serde_cbor::Value, + ) -> Result { + let value: webc::WitBindingsExtended = + serde_cbor::from_slice(&serde_cbor::to_vec(value).unwrap()) + .map_err(|e| format!("could not parse WitBindings annotations: {e}"))?; + + let mut wit_bindgen_filepath = value.wit.exports; + + for v in container.get_volumes() { + let schema = format!("{v}://"); + if wit_bindgen_filepath.starts_with(&schema) { + wit_bindgen_filepath = wit_bindgen_filepath.replacen(&schema, "", 1); + break; + } + } + + let wit_bindings = container + .get_file(&wit_bindgen_filepath) + .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; + + let wit_bindings_str = std::str::from_utf8(wit_bindings) + .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; + + Self::parse(wit_bindings_str) + } +} + +/// Trait that all runners have to implement +pub trait Runner { + /// The return value of the output of the runner + type Output; + + /// Returns whether the Runner will be able to run the `Command` + fn can_run_command( + &self, + command_name: &str, + command: &Command, + ) -> Result>; + + /// Implementation to run the given command + /// + /// - use `cmd.annotations` to get the metadata for the given command + /// - use `container.get_atom()` to get the + fn run_command( + &mut self, + command_name: &str, + cmd: &Command, + container: &WapmContainer, + ) -> Result>; + + /// Runs the container if the container has an `entrypoint` in the manifest + fn run(&mut self, container: &WapmContainer) -> Result> { + let cmd = match container.webc.webc.manifest.entrypoint.as_ref() { + Some(s) => s, + None => { + let path = format!("{}", container.webc.path.display()); + return Err(Box::new(webc::Error(format!( + "Cannot run {path:?}: not executable (no entrypoint in manifest)" + )))); + } + }; + + self.run_cmd(container, cmd) + } + + /// Runs the given `cmd` on the container + fn run_cmd( + &mut self, + container: &WapmContainer, + cmd: &str, + ) -> Result> { + let path = format!("{}", container.webc.path.display()); + let command_to_exec = container + .webc + .webc + .manifest + .commands + .get(cmd) + .ok_or_else(|| anyhow::anyhow!("{path}: command {cmd:?} not found in manifest"))?; + + let _path = format!("{}", container.webc.path.display()); + + match self.can_run_command(cmd, command_to_exec) { + Ok(true) => {} + Ok(false) => { + return Err(Box::new(webc::Error(format!( + "Cannot run command {cmd:?} with runner {:?}", + command_to_exec.runner + )))); + } + Err(e) => { + return Err(Box::new(webc::Error(format!( + "Cannot run command {cmd:?} with runner {:?}: {e}", + command_to_exec.runner + )))); + } + } + + self.run_command(cmd, command_to_exec, container) + } +} diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs new file mode 100644 index 00000000000..28dd9022986 --- /dev/null +++ b/lib/wasi/src/runners/wasi.rs @@ -0,0 +1,114 @@ +#![cfg(feature = "webc_runner_rt_emscripten")] +//! WebC container support for running WASI modules + +use crate::runners::WapmContainer; +use crate::{WasiFunctionEnv, WasiState}; +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use std::error::Error as StdError; +use std::sync::Arc; +use wasmer::{Cranelift, Instance, Module, Store}; +use wasmer_vfs::webc_fs::WebcFileSystem; +use webc::{Command, WebCMmap}; + +#[derive(Debug, Default, Clone, PartialEq, PartialOrd, Hash, Serialize, Deserialize)] +pub struct WasiRunner { + args: Vec, +} + +impl WasiRunner { + pub fn set_args(&mut self, args: Vec) { + self.args = args; + } +} + +impl crate::runners::Runner for WasiRunner { + type Output = (); + + fn can_run_command( + &self, + _command_name: &str, + command: &Command, + ) -> Result> { + Ok(command.runner.starts_with("https://webc.org/runner/wasi")) + } + + fn run_command( + &mut self, + command_name: &str, + _command: &Command, + container: &WapmContainer, + ) -> Result> { + let atom_name = container.get_atom_name_for_command("wasi", command_name)?; + let atom_bytes = container.get_atom(&container.get_package_name(), &atom_name)?; + + let compiler = Cranelift::default(); + let mut store = Store::new(compiler); + let mut module = Module::new(&store, atom_bytes)?; + module.set_name(&atom_name); + + let env = prepare_webc_env(&mut store, container.webc.clone(), &atom_name, &self.args)?; + + exec_module(&mut store, &module, env)?; + + Ok(()) + } +} + +// https://github.com/tokera-com/ate/blob/42c4ce5a0c0aef47aeb4420cc6dc788ef6ee8804/term-lib/src/eval/exec.rs#L444 +fn prepare_webc_env( + store: &mut Store, + webc: Arc, + command: &str, + args: &[String], +) -> Result { + use webc::FsEntryType; + + let package_name = webc.get_package_name(); + let top_level_dirs = webc + .get_volumes_for_package(&package_name) + .into_iter() + .flat_map(|volume| { + webc.volumes + .get(&volume) + .unwrap() + .header + .top_level + .iter() + .filter(|e| e.fs_type == FsEntryType::Dir) + .map(|e| e.text.to_string()) + }) + .collect::>(); + + let filesystem = Box::new(WebcFileSystem::init(webc, &package_name)); + let mut wasi_env = WasiState::new(command); + wasi_env.set_fs(filesystem); + wasi_env.args(args); + for f_name in top_level_dirs.iter() { + wasi_env.preopen(|p| p.directory(f_name).read(true).write(true).create(true))?; + } + + Ok(wasi_env.finalize(store)?) +} + +pub(crate) fn exec_module( + store: &mut Store, + module: &Module, + wasi_env: crate::WasiFunctionEnv, +) -> Result<(), anyhow::Error> { + let import_object = wasi_env.import_object(store, module)?; + let instance = Instance::new(store, module, &import_object)?; + let memory = instance.exports.get_memory("memory")?; + wasi_env.data_mut(store).set_memory(memory.clone()); + + // If this module exports an _initialize function, run that first. + if let Ok(initialize) = instance.exports.get_function("_initialize") { + initialize + .call(store, &[]) + .with_context(|| "failed to run _initialize function")?; + } + + let _result = instance.exports.get_function("_start")?.call(store, &[])?; + + Ok(()) +} diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index b9601acefde..e48c885fa04 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -9,7 +9,14 @@ publish = false [dev-dependencies] rand = "0.8.5" +tar = "0.4.38" +flate2 = "1.0.24" +target-lexicon = "0.12.4" [dependencies] anyhow = "1" tempfile = "3" + +[features] +default = ["webc_runner"] +webc_runner = [] \ No newline at end of file diff --git a/tests/integration/cli/src/assets.rs b/tests/integration/cli/src/assets.rs index 8fe5dbc79c3..53f15bd7997 100644 --- a/tests/integration/cli/src/assets.rs +++ b/tests/integration/cli/src/assets.rs @@ -63,19 +63,37 @@ pub fn get_wasmer_path() -> PathBuf { ret = PathBuf::from(format!("{}wasmer", WASMER_TARGET_PATH2)); } if !ret.exists() { - if let Some(s) = env!("CARGO_MANIFEST_DIR").split("wasmer").next() { - #[cfg(target_os = "windows")] - { - return std::path::Path::new(&format!("{s}wasmer/target/release/wasmer.exe")) - .to_path_buf(); + match get_repo_root_path() { + Some(s) => { + #[cfg(target_os = "windows")] + { + return s.join("target").join("release").join("wasmer.exe"); + } + #[cfg(not(target_os = "windows"))] + { + return s.join("target").join("release").join("wasmer"); + } } - #[cfg(not(target_os = "windows"))] - { - return std::path::Path::new(&format!("{s}wasmer/target/release/wasmer")) - .to_path_buf(); + None => { + panic!("Could not find wasmer executable path! {:?}", ret); } } - panic!("Could not find wasmer executable path! {:?}", ret); } ret } + +pub fn get_repo_root_path() -> Option { + let mut current_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut counter = 0; + let mut result = None; + 'outer: while counter < 50 { + counter += 1; + if current_dir.join("CHANGELOG.md").exists() && current_dir.join("LICENSE").exists() { + result = Some(current_dir.to_path_buf()); + break 'outer; + } else { + current_dir = current_dir.parent()?; + } + } + result +} diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 140f13b8b7a..c98722789e0 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -1,8 +1,13 @@ //! Basic tests for the `run` subcommand use anyhow::bail; +use std::path::PathBuf; use std::process::Command; -use wasmer_integration_tests_cli::{get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; +use wasmer_integration_tests_cli::{get_repo_root_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; + +fn wasi_test_python_path() -> String { + format!("{}/{}", C_ASSET_PATH, "python-0.1.0.wasmer") +} fn wasi_test_wasm_path() -> String { format!("{}/{}", C_ASSET_PATH, "qjs.wasm") @@ -42,6 +47,191 @@ fn run_wasi_works() -> anyhow::Result<()> { Ok(()) } +#[cfg(feature = "webc_runner")] +fn package_directory(in_dir: &PathBuf, out: &PathBuf) { + use flate2::write::GzEncoder; + use flate2::Compression; + use std::fs::File; + let tar = File::create(out).unwrap(); + let enc = GzEncoder::new(tar, Compression::none()); + let mut a = tar::Builder::new(enc); + a.append_dir_all("", in_dir).unwrap(); + a.finish().unwrap(); +} + +/// TODO: on linux-musl, the packaging of libwasmer.a doesn't work properly +/// Tracked in https://github.com/wasmerio/wasmer/issues/3271 +#[cfg(not(target_env = "musl"))] +#[cfg(feature = "webc_runner")] +#[test] +fn test_wasmer_create_exe_pirita_works() -> anyhow::Result<()> { + let temp_dir = tempfile::TempDir::new()?; + let python_wasmer_path = temp_dir.path().join("python.wasmer"); + std::fs::copy(wasi_test_python_path(), &python_wasmer_path)?; + let python_exe_output_path = temp_dir.path().join("python"); + + let native_target = target_lexicon::HOST; + let root_path = get_repo_root_path().unwrap(); + let package_path = root_path.join("package"); + if !package_path.exists() { + let current_dir = std::env::current_dir().unwrap(); + println!("running make && make build-capi && make package-capi && make package..."); + println!("current dir = {}", current_dir.display()); + println!("setting current dir = {}", root_path.display()); + // make && make build-capi && make package-capi && make package + let mut c1 = std::process::Command::new("make"); + c1.current_dir(&root_path); + let r = c1.output().unwrap(); + if !r.status.success() { + let stdout = String::from_utf8_lossy(&r.stdout); + let stderr = String::from_utf8_lossy(&r.stdout); + println!("make failed: (stdout = {stdout}, stderr = {stderr})"); + } + println!("make ok!"); + let mut c1 = std::process::Command::new("make"); + c1.arg("build-wasmer"); + c1.current_dir(&root_path); + let r = c1.output().unwrap(); + if !r.status.success() { + let stdout = String::from_utf8_lossy(&r.stdout); + let stderr = String::from_utf8_lossy(&r.stdout); + println!("make failed: (stdout = {stdout}, stderr = {stderr})"); + } + println!("make build-wasmer ok!"); + let mut c1 = std::process::Command::new("make"); + c1.arg("build-capi"); + c1.current_dir(&root_path); + let r = c1.output().unwrap(); + if !r.status.success() { + let stdout = String::from_utf8_lossy(&r.stdout); + let stderr = String::from_utf8_lossy(&r.stdout); + println!("make build-capi failed: (stdout = {stdout}, stderr = {stderr})"); + } + println!("make build-capi ok!"); + + let mut c1 = std::process::Command::new("make"); + c1.arg("build-wasmer"); + c1.current_dir(&root_path); + let r = c1.output().unwrap(); + if !r.status.success() { + let stdout = String::from_utf8_lossy(&r.stdout); + let stderr = String::from_utf8_lossy(&r.stdout); + println!("make build-wasmer failed: (stdout = {stdout}, stderr = {stderr})"); + } + println!("make build-wasmer ok!"); + + let mut c1 = std::process::Command::new("make"); + c1.arg("package-capi"); + c1.current_dir(&root_path); + let r = c1.output().unwrap(); + if !r.status.success() { + let stdout = String::from_utf8_lossy(&r.stdout); + let stderr = String::from_utf8_lossy(&r.stdout); + println!("make package-capi: (stdout = {stdout}, stderr = {stderr})"); + } + println!("make package-capi ok!"); + + let mut c1 = std::process::Command::new("make"); + c1.arg("package"); + c1.current_dir(&root_path); + let r = c1.output().unwrap(); + if !r.status.success() { + let stdout = String::from_utf8_lossy(&r.stdout); + let stderr = String::from_utf8_lossy(&r.stdout); + println!("make package failed: (stdout = {stdout}, stderr = {stderr})"); + } + println!("make package ok!"); + } + if !package_path.exists() { + panic!("package path {} does not exist", package_path.display()); + } + let tmp_targz_path = tempfile::TempDir::new()?; + let tmp_targz_path = tmp_targz_path.path().join("link.tar.gz"); + println!("compiling to target {native_target}"); + println!( + "packaging /package to .tar.gz: {}", + tmp_targz_path.display() + ); + package_directory(&package_path, &tmp_targz_path); + println!("packaging done"); + + let mut cmd = Command::new(get_wasmer_path()); + cmd.arg("create-exe"); + cmd.arg(&python_wasmer_path); + cmd.arg("--tarball"); + cmd.arg(&tmp_targz_path); + cmd.arg("--target"); + cmd.arg(format!("{native_target}")); + cmd.arg("-o"); + cmd.arg(&python_exe_output_path); + + println!("running: {cmd:?}"); + + let output = cmd.output()?; + + if !output.status.success() { + let stdout = std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"); + + bail!( + "running wasmer create-exe {} failed with: stdout: {}\n\nstderr: {}", + python_wasmer_path.display(), + stdout, + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); + } + + let output = Command::new(&python_exe_output_path) + .arg("-c") + .arg("print(\"hello\")") + .output()?; + + let stdout = std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"); + + if !stdout.ends_with("hello\n") { + bail!( + "1 running python.wasmer failed with: stdout: {}\n\nstderr: {}", + stdout, + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); + } + + Ok(()) +} + +#[cfg(feature = "webc_runner")] +#[test] +fn test_wasmer_run_pirita_works() -> anyhow::Result<()> { + let temp_dir = tempfile::TempDir::new()?; + let python_wasmer_path = temp_dir.path().join("python.wasmer"); + std::fs::copy(wasi_test_python_path(), &python_wasmer_path)?; + + let output = Command::new(get_wasmer_path()) + .arg("run") + .arg(python_wasmer_path) + .arg("--") + .arg("-c") + .arg("print(\"hello\")") + .output()?; + + let stdout = std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"); + + if !stdout.ends_with("hello\n") { + bail!( + "1 running python.wasmer failed with: stdout: {}\n\nstderr: {}", + stdout, + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); + } + + Ok(()) +} + #[test] fn test_wasmer_run_works_with_dir() -> anyhow::Result<()> { let temp_dir = tempfile::TempDir::new()?; diff --git a/tests/integration/ios/Cargo.toml b/tests/integration/ios/Cargo.toml index 6d6874fd9e8..19562479a22 100644 --- a/tests/integration/ios/Cargo.toml +++ b/tests/integration/ios/Cargo.toml @@ -6,3 +6,7 @@ description = "iOS integration tests" repository = "https://github.com/wasmerio/wasmer" edition = "2018" publish = false + +[features] +default = ["webc_runner"] +webc_runner = [] \ No newline at end of file