diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 61959d7fb3c..69e911f3791 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -275,14 +275,14 @@ jobs: target: aarch64-unknown-linux-gnu - name: Build cross image run: | - docker build -t wasmer/aarch64 /home/runner/work/wasmer/wasmer/.github/cross-linux-aarch64/ + docker build -t wasmer/aarch64 ${GITHUB_WORKSPACE}/.github/cross-linux-aarch64/ env: CROSS_DOCKER_IN_DOCKER: true - name: Build Wasmer binary run: | make build-wasmer env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig @@ -293,7 +293,7 @@ jobs: run: | make package-capi-headless env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig @@ -305,7 +305,7 @@ jobs: run: | make build-capi env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig @@ -315,7 +315,7 @@ jobs: run: | make distribution env: - CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v /home/runner/work/wasmer/wasmer:/project -w /project wasmer/aarch64 cross + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross CROSS_DOCKER_IN_DOCKER: true CARGO_TARGET: --target aarch64-unknown-linux-gnu PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig diff --git a/Cargo.lock b/Cargo.lock index 128e98b66d8..cce10a65a67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30275ad0ad84ec1c06dde3b3f7d23c6006b7d76d61a85e7060b426b747eff70d" +[[package]] +name = "any_ascii" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e" + [[package]] name = "anyhow" version = "1.0.66" @@ -99,6 +105,19 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "async-compression" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.58" @@ -496,6 +515,15 @@ dependencies = [ "windows-sys 0.33.0", ] +[[package]] +name = "cpufeatures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +dependencies = [ + "libc", +] + [[package]] name = "cranelift-bforest" version = "0.86.1" @@ -1004,6 +1032,28 @@ dependencies = [ "serde", ] +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1085,6 +1135,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.25" @@ -1253,6 +1309,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "graphql-introspection-query" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "610aac641dbd2a457ad4cef34aa2827dae3f035fd214cb38c2d62d8543f3973f" +dependencies = [ + "serde", +] + [[package]] name = "graphql-introspection-query" version = "0.2.0" @@ -1262,6 +1327,16 @@ dependencies = [ "serde", ] +[[package]] +name = "graphql-parser" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5613c31f18676f164112732202124f373bb2103ff017b3b85ca954ea6a66ada" +dependencies = [ + "combine", + "failure", +] + [[package]] name = "graphql-parser" version = "0.4.0" @@ -1272,15 +1347,45 @@ dependencies = [ "thiserror", ] +[[package]] +name = "graphql_client" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bb4f09181e4f80018d01c612125b07e0156f3753bfac37055fe2a25e031ca8" +dependencies = [ + "doc-comment", + "graphql_query_derive 0.9.0", + "serde", + "serde_json", +] + [[package]] name = "graphql_client" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fc16d75d169fddb720d8f1c7aed6413e329e1584079b9734ff07266a193f5bc" dependencies = [ - "graphql_query_derive", + "graphql_query_derive 0.11.0", + "serde", + "serde_json", +] + +[[package]] +name = "graphql_client_codegen" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e304c223c809b3bff4614018f8e6d9edb176b31d64ed9ea48b6ae8b1a03abb9" +dependencies = [ + "failure", + "graphql-introspection-query 0.1.0", + "graphql-parser 0.2.3", + "heck 0.3.3", + "lazy_static", + "proc-macro2", + "quote", "serde", "serde_json", + "syn", ] [[package]] @@ -1289,8 +1394,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f290ecfa3bea3e8a157899dc8a1d96ee7dd6405c18c8ddd213fc58939d18a0e9" dependencies = [ - "graphql-introspection-query", - "graphql-parser", + "graphql-introspection-query 0.2.0", + "graphql-parser 0.4.0", "heck 0.4.0", "lazy_static", "proc-macro2", @@ -1300,13 +1405,25 @@ dependencies = [ "syn", ] +[[package]] +name = "graphql_query_derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1f6b14d5ce549227aa9e649cd9d36d008b91021275a8e0a67d71cef815adc2f" +dependencies = [ + "failure", + "graphql_client_codegen 0.9.0", + "proc-macro2", + "syn", +] + [[package]] name = "graphql_query_derive" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a755cc59cda2641ea3037b4f9f7ef40471c329f55c1fa2db6fa0bb7ae6c1f7ce" dependencies = [ - "graphql_client_codegen", + "graphql_client_codegen 0.11.0", "proc-macro2", "syn", ] @@ -1523,6 +1640,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.3.0" @@ -1665,6 +1793,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1677,6 +1816,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" +[[package]] +name = "lexical-sort" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c09e4591611e231daf4d4c685a66cb0410cc1e502027a20ae55f2bb9e997207a" +dependencies = [ + "any_ascii", +] + [[package]] name = "libc" version = "0.2.136" @@ -1779,6 +1927,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "memchr" version = "2.5.0" @@ -2053,6 +2207,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +[[package]] +name = "path-clean" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd" + [[package]] name = "percent-encoding" version = "2.2.0" @@ -2069,6 +2229,40 @@ dependencies = [ "ucd-trie", ] +[[package]] +name = "pest_derive" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b75706b9642ebcb34dab3bc7750f811609a0eb1dd8b88c2d15bf628c1c65b2" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f9272122f5979a6511a749af9db9bfc810393f63119970d7085fed1c4ea0db" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8717927f9b79515e565a64fe46c38b8cd0427e64c40680b14a7365ab09ac8d" +dependencies = [ + "once_cell", + "pest", + "sha1 0.10.5", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -2081,6 +2275,14 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pirita" +version = "0.1.0" +dependencies = [ + "webc", + "webc-runner", +] + [[package]] name = "pkg-config" version = "0.3.25" @@ -2280,6 +2482,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -2288,7 +2503,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -2298,9 +2513,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", ] +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -2353,6 +2583,15 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2453,6 +2692,7 @@ version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" dependencies = [ + "async-compression", "base64", "bytes", "encoding_rs", @@ -2478,6 +2718,8 @@ dependencies = [ "serde_urlencoded", "tokio", "tokio-rustls", + "tokio-socks", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -2855,12 +3097,34 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sha2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -2971,7 +3235,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1", + "sha1 0.6.1", "syn", ] @@ -3004,6 +3268,18 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + [[package]] name = "tar" version = "0.4.38" @@ -3015,6 +3291,16 @@ dependencies = [ "xattr", ] +[[package]] +name = "tar" +version = "0.4.38" +source = "git+https://github.com/fschutt/tar-rs?rev=04ded46840bb195f6c14ca85c5a6d25b61ca349c#04ded46840bb195f6c14ca85c5a6d25b61ca349c" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.11.2" @@ -3027,6 +3313,16 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -3244,6 +3540,18 @@ dependencies = [ "webpki 0.22.0", ] +[[package]] +name = "tokio-socks" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0" +dependencies = [ + "either", + "futures-util", + "thiserror", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.4" @@ -3465,8 +3773,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna", + "idna 0.3.0", "percent-encoding", + "serde", +] + +[[package]] +name = "validator" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f07b0a1390e01c0fc35ebb26b28ced33c9a3808f7f9fbe94d3cc01e233bfeed5" +dependencies = [ + "idna 0.2.3", + "lazy_static", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", ] [[package]] @@ -3517,6 +3841,59 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wapm-resolve-url" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wapm-cli?branch=download-pirita#306712983d020da772062f9654ef16ebfff29647" +dependencies = [ + "anyhow", + "graphql_client 0.9.0", + "reqwest", + "serde", + "serde_json", + "thiserror", + "url", + "whoami", +] + +[[package]] +name = "wapm-targz-to-pirita" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "flate2", + "json5", + "rand 0.8.5", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "sha2", + "tar 0.4.38 (git+https://github.com/fschutt/tar-rs?rev=04ded46840bb195f6c14ca85c5a6d25b61ca349c)", + "validator", + "webc", +] + +[[package]] +name = "wapm-toml" +version = "0.1.0" +source = "git+https://github.com/wasmerio/wapm-cli?branch=download-pirita#306712983d020da772062f9654ef16ebfff29647" +dependencies = [ + "anyhow", + "indexmap", + "semver 1.0.14", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "serde_yaml", + "thiserror", + "toml", + "validator", + "wapm-resolve-url", +] + [[package]] name = "wapm-toml" version = "0.2.0" @@ -3692,18 +4069,43 @@ dependencies = [ "tracing", "wasm-bindgen", "wasm-bindgen-test", - "wasmer-compiler", - "wasmer-compiler-cranelift", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-compiler-cranelift 3.0.0-rc.1", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", - "wasmer-derive", - "wasmer-types", - "wasmer-vm", + "wasmer-derive 3.0.0-rc.1", + "wasmer-types 3.0.0-rc.1", + "wasmer-vm 3.0.0-rc.1", "wasmparser 0.83.0", "wat", "winapi", ] +[[package]] +name = "wasmer" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6eea8f768e7b4313e1db8f8f4d20cccf0999102d53a2c27e1d88ecdf0b638d" +dependencies = [ + "bytes", + "cfg-if 1.0.0", + "indexmap", + "js-sys", + "more-asserts", + "serde", + "serde-wasm-bindgen", + "target-lexicon 0.12.4", + "thiserror", + "wasm-bindgen", + "wasmer-compiler 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-compiler-cranelift 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-derive 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-types 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-vm 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wat", + "winapi", +] + [[package]] name = "wasmer-bin-fuzz" version = "0.0.0" @@ -3711,9 +4113,9 @@ dependencies = [ "anyhow", "libfuzzer-sys", "wasm-smith", - "wasmer", - "wasmer-compiler", - "wasmer-compiler-cranelift", + "wasmer 3.0.0-rc.1", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-compiler-cranelift 3.0.0-rc.1", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", "wasmer-middlewares", @@ -3732,18 +4134,19 @@ dependencies = [ "lazy_static", "libc", "paste", + "pirita", "thiserror", "typetag", - "wasmer", - "wasmer-compiler", - "wasmer-compiler-cranelift", + "wasmer 3.0.0-rc.1", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-compiler-cranelift 3.0.0-rc.1", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", - "wasmer-emscripten", + "wasmer-emscripten 3.0.0-rc.1", "wasmer-inline-c", "wasmer-middlewares", - "wasmer-types", - "wasmer-wasi", + "wasmer-types 3.0.0-rc.1", + "wasmer-wasi 3.0.0-rc.1", ] [[package]] @@ -3763,10 +4166,10 @@ dependencies = [ "blake3", "criterion", "hex", - "rand", + "rand 0.8.5", "tempfile", "thiserror", - "wasmer", + "wasmer 3.0.0-rc.1", "wasmer-compiler-singlepass", ] @@ -3796,30 +4199,33 @@ dependencies = [ "fern", "http_req", "log", + "pirita", "prettytable-rs", "regex", "serde_json", "spinner", "target-lexicon 0.12.4", + "tempdir", "tempfile", "toml", "unix_mode", "url", "walkdir", - "wapm-toml", - "wasmer", + "wapm-targz-to-pirita", + "wapm-toml 0.2.0", + "wasmer 3.0.0-rc.1", "wasmer-cache", - "wasmer-compiler", - "wasmer-compiler-cranelift", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-compiler-cranelift 3.0.0-rc.1", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", - "wasmer-emscripten", + "wasmer-emscripten 3.0.0-rc.1", "wasmer-object", "wasmer-registry", - "wasmer-types", - "wasmer-vfs", - "wasmer-vm", - "wasmer-wasi", + "wasmer-types 3.0.0-rc.1", + "wasmer-vfs 3.0.0-rc.1", + "wasmer-vm 3.0.0-rc.1", + "wasmer-wasi 3.0.0-rc.1", "wasmer-wasi-experimental-io-devices", "wasmer-wast", ] @@ -3844,8 +4250,32 @@ dependencies = [ "smallvec", "thiserror", "wasmer-object", - "wasmer-types", - "wasmer-vm", + "wasmer-types 3.0.0-rc.1", + "wasmer-vm 3.0.0-rc.1", + "wasmparser 0.83.0", + "winapi", +] + +[[package]] +name = "wasmer-compiler" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6aac0e272385c4e29cbd217f874df59362053a5bda95c340d0c90c2cf9e56ca" +dependencies = [ + "backtrace", + "cfg-if 1.0.0", + "enum-iterator", + "enumset", + "lazy_static", + "leb128", + "memmap2", + "more-asserts", + "region", + "rustc-demangle", + "smallvec", + "thiserror", + "wasmer-types 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-vm 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", "wasmparser 0.83.0", "winapi", ] @@ -3866,10 +4296,10 @@ dependencies = [ "target-lexicon 0.12.4", "tempfile", "unix_mode", - "wasmer-compiler", - "wasmer-compiler-cranelift", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-compiler-cranelift 3.0.0-rc.1", "wasmer-compiler-singlepass", - "wasmer-types", + "wasmer-types 3.0.0-rc.1", ] [[package]] @@ -3887,8 +4317,27 @@ dependencies = [ "smallvec", "target-lexicon 0.12.4", "tracing", - "wasmer-compiler", - "wasmer-types", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-types 3.0.0-rc.1", +] + +[[package]] +name = "wasmer-compiler-cranelift" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4930b0384f8754129eabcfbd2ac33eaea976aef7a573a2c73728df7a31bab54e" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "gimli", + "more-asserts", + "rayon", + "smallvec", + "target-lexicon 0.12.4", + "tracing", + "wasmer-compiler 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-types 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3908,9 +4357,9 @@ dependencies = [ "semver 1.0.14", "smallvec", "target-lexicon 0.12.4", - "wasmer-compiler", - "wasmer-types", - "wasmer-vm", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-types 3.0.0-rc.1", + "wasmer-vm 3.0.0-rc.1", ] [[package]] @@ -3927,8 +4376,8 @@ dependencies = [ "rayon", "smallvec", "target-lexicon 0.12.4", - "wasmer-compiler", - "wasmer-types", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-types 3.0.0-rc.1", ] [[package]] @@ -3940,12 +4389,40 @@ dependencies = [ "proc-macro2", "quote", "syn", - "wasmer", + "wasmer 3.0.0-rc.1", +] + +[[package]] +name = "wasmer-derive" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f1bc5a30f2b49dc52fe5ac9487b22b69985032f9a9b7b6d7dc27c56fca6430" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "wasmer-emscripten" +version = "3.0.0-rc.1" +dependencies = [ + "byteorder", + "getrandom", + "lazy_static", + "libc", + "log", + "time", + "wasmer 3.0.0-rc.1", + "wasmer-types 3.0.0-rc.1", ] [[package]] name = "wasmer-emscripten" version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728fcca56d350752266116db58e2cb27b059c4a5ee58fc0cca928157c67cd31" dependencies = [ "byteorder", "getrandom", @@ -3953,8 +4430,8 @@ dependencies = [ "libc", "log", "time", - "wasmer", - "wasmer-types", + "wasmer 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-types 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3990,7 +4467,7 @@ name = "wasmer-integration-tests-cli" version = "3.0.0-rc.1" dependencies = [ "anyhow", - "rand", + "rand 0.8.5", "tempfile", ] @@ -4002,9 +4479,9 @@ version = "3.0.0-rc.1" name = "wasmer-middlewares" version = "3.0.0-rc.1" dependencies = [ - "wasmer", - "wasmer-types", - "wasmer-vm", + "wasmer 3.0.0-rc.1", + "wasmer-types 3.0.0-rc.1", + "wasmer-vm 3.0.0-rc.1", ] [[package]] @@ -4013,7 +4490,7 @@ version = "3.0.0-rc.1" dependencies = [ "object 0.28.4", "thiserror", - "wasmer-types", + "wasmer-types 3.0.0-rc.1", ] [[package]] @@ -4023,16 +4500,16 @@ dependencies = [ "anyhow", "dirs 4.0.0", "flate2", - "graphql_client", + "graphql_client 0.11.0", "reqwest", "semver 1.0.14", "serde", "serde_json", - "tar", + "tar 0.4.38 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", "toml", "url", - "wapm-toml", + "wapm-toml 0.2.0", "whoami", ] @@ -4052,12 +4529,37 @@ dependencies = [ "thiserror", ] +[[package]] +name = "wasmer-types" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0065ac636258dd9277d69210ad881df5f7973491e693a7e877b0e4389afe5a2c" +dependencies = [ + "enum-iterator", + "enumset", + "indexmap", + "more-asserts", + "rkyv", + "target-lexicon 0.12.4", + "thiserror", +] + +[[package]] +name = "wasmer-vbus" +version = "3.0.0-rc.1" +dependencies = [ + "thiserror", + "wasmer-vfs 3.0.0-rc.1", +] + [[package]] name = "wasmer-vbus" version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3873de544e32d21953ce526c28d478e23c75fa7e574eb0104800302529ae0e4" dependencies = [ "thiserror", - "wasmer-vfs", + "wasmer-vfs 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4072,6 +4574,18 @@ dependencies = [ "typetag", ] +[[package]] +name = "wasmer-vfs" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02ae1e172fd3d57e25b339eb0db78d7401366c9091730cb5faf4fa603b5153cc" +dependencies = [ + "libc", + "slab", + "thiserror", + "tracing", +] + [[package]] name = "wasmer-vm" version = "3.0.0-rc.1" @@ -4091,17 +4605,52 @@ dependencies = [ "scopeguard", "serde", "thiserror", - "wasmer-types", + "wasmer-types 3.0.0-rc.1", "winapi", ] +[[package]] +name = "wasmer-vm" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "366dc013296afba61854d4729e360367dd1fb6651266ef1f904059dbdc1d838a" +dependencies = [ + "backtrace", + "cc", + "cfg-if 1.0.0", + "corosensei", + "enum-iterator", + "indexmap", + "lazy_static", + "libc", + "mach", + "memoffset", + "more-asserts", + "region", + "scopeguard", + "thiserror", + "wasmer-types 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi", +] + +[[package]] +name = "wasmer-vnet" +version = "3.0.0-rc.1" +dependencies = [ + "bytes", + "thiserror", + "wasmer-vfs 3.0.0-rc.1", +] + [[package]] name = "wasmer-vnet" version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a127fd94dd7b3fbba68c2446bab779f7b7dde3de8fa8dfd6242c5cfb2624f1" dependencies = [ "bytes", "thiserror", - "wasmer-vfs", + "wasmer-vfs 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4123,12 +4672,36 @@ dependencies = [ "typetag", "wasm-bindgen", "wasm-bindgen-test", - "wasmer", - "wasmer-vbus", - "wasmer-vfs", - "wasmer-vnet", - "wasmer-wasi-local-networking", - "wasmer-wasi-types", + "wasmer 3.0.0-rc.1", + "wasmer-vbus 3.0.0-rc.1", + "wasmer-vfs 3.0.0-rc.1", + "wasmer-vnet 3.0.0-rc.1", + "wasmer-wasi-local-networking 3.0.0-rc.1", + "wasmer-wasi-types 3.0.0-rc.1", + "winapi", +] + +[[package]] +name = "wasmer-wasi" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f3bd1aef78b748e2be1fb8ac04638910d85b59d38ca2a43aaeb3e336d9be56" +dependencies = [ + "bytes", + "cfg-if 1.0.0", + "derivative", + "generational-arena", + "getrandom", + "libc", + "thiserror", + "tracing", + "wasm-bindgen", + "wasmer 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-vbus 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-vfs 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-vnet 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-wasi-local-networking 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-wasi-types 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi", ] @@ -4142,17 +4715,29 @@ dependencies = [ "serde", "tracing", "typetag", - "wasmer-wasi", + "wasmer-wasi 3.0.0-rc.1", +] + +[[package]] +name = "wasmer-wasi-local-networking" +version = "3.0.0-rc.1" +dependencies = [ + "bytes", + "tracing", + "wasmer-vfs 3.0.0-rc.1", + "wasmer-vnet 3.0.0-rc.1", ] [[package]] name = "wasmer-wasi-local-networking" version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75d5318cd6f3a51832b05c2aaf977177f51ece7ff1615d156d10da5d94df225e" dependencies = [ "bytes", "tracing", - "wasmer-vfs", - "wasmer-vnet", + "wasmer-vfs 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-vnet 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4163,9 +4748,26 @@ dependencies = [ "pretty_assertions", "serde", "time", - "wasmer", - "wasmer-derive", - "wasmer-types", + "wasmer 3.0.0-rc.1", + "wasmer-derive 3.0.0-rc.1", + "wasmer-types 3.0.0-rc.1", + "wasmer-wit-bindgen-gen-core", + "wasmer-wit-bindgen-gen-rust-wasm", + "wasmer-wit-bindgen-rust", + "wasmer-wit-parser", +] + +[[package]] +name = "wasmer-wasi-types" +version = "3.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0777021b8eba997fe2e0bc315de2c0f897d9f22d91dd8822a3eca26ba7652f9b" +dependencies = [ + "byteorder", + "time", + "wasmer 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-derive 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-types 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", "wasmer-wit-bindgen-gen-core", "wasmer-wit-bindgen-gen-rust-wasm", "wasmer-wit-bindgen-rust", @@ -4180,9 +4782,9 @@ dependencies = [ "serde", "tempfile", "thiserror", - "wasmer", - "wasmer-vfs", - "wasmer-wasi", + "wasmer 3.0.0-rc.1", + "wasmer-vfs 3.0.0-rc.1", + "wasmer-wasi 3.0.0-rc.1", "wast 38.0.1", ] @@ -4272,16 +4874,16 @@ dependencies = [ "tracing", "tracing-subscriber", "wasi-test-generator", - "wasmer", + "wasmer 3.0.0-rc.1", "wasmer-cache", - "wasmer-compiler", - "wasmer-compiler-cranelift", + "wasmer-compiler 3.0.0-rc.1", + "wasmer-compiler-cranelift 3.0.0-rc.1", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", - "wasmer-emscripten", + "wasmer-emscripten 3.0.0-rc.1", "wasmer-middlewares", - "wasmer-types", - "wasmer-wasi", + "wasmer-types 3.0.0-rc.1", + "wasmer-wasi 3.0.0-rc.1", "wasmer-wast", ] @@ -4432,6 +5034,62 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webc" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "indexmap", + "leb128", + "lexical-sort", + "memchr", + "memmap2", + "path-clean", + "rand 0.8.5", + "serde", + "serde_cbor", + "serde_json", + "sha2", + "url", + "walkdir", +] + +[[package]] +name = "webc-runner" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures-util", + "lazy_static", + "libc", + "log", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "tokio", + "url", + "wapm-resolve-url", + "wapm-toml 0.1.0", + "wasmer 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-compiler 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-emscripten 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-vfs 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-wasi 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webc", + "webc-vfs", +] + +[[package]] +name = "webc-vfs" +version = "0.1.0" +dependencies = [ + "anyhow", + "wasmer-vfs 3.0.0-rc.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webc", +] + [[package]] name = "webpki" version = "0.21.4" diff --git a/Makefile b/Makefile index 557ea873a19..9a8eb00456c 100644 --- a/Makefile +++ b/Makefile @@ -367,6 +367,12 @@ build-wasmer: build-wasmer-debug: $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "debug" --bin wasmer +build-wasmer-pirita: + $(CARGO_BINARY) build $(CARGO_TARGET) --release --manifest-path lib/cli/Cargo.toml $(compiler_features) --features="pirita_file" --bin wasmer + +build-wasmer-pirita-debug: + $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/cli/Cargo.toml $(compiler_features) --features "pirita_file,debug" --bin wasmer + bench: $(CARGO_BINARY) bench $(CARGO_TARGET) $(compiler_features) @@ -428,6 +434,10 @@ build-capi: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,compiler,wasi,middlewares $(capi_compiler_features) +build-capi-pirita: capi-setup + RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ + --no-default-features --features wat,compiler,wasi,middlewares,pirita_file $(capi_compiler_features) + build-capi-singlepass: capi-setup RUSTFLAGS="${RUSTFLAGS}" $(CARGO_BINARY) build $(CARGO_TARGET) --manifest-path lib/c-api/Cargo.toml --release \ --no-default-features --features wat,compiler,singlepass,wasi,middlewares diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index 1c896654b8e..48494a8d624 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -39,6 +39,12 @@ thiserror = "1" typetag = { version = "0.1", optional = true } paste = "1.0" +[dependencies.pirita] +path = "../../../pirita/crates/pirita" +default-features = false +features = ["webc_runner"] +optional = true + [dev-dependencies] field-offset = "0.3.3" @@ -90,6 +96,7 @@ wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] static-artifact-load = ["wasmer-compiler/static-artifact-load"] static-artifact-create = ["wasmer-compiler/static-artifact-create"] +pirita_file = ["pirita"] # Deprecated features. jit = ["compiler"] diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index b83dc3fd243..34e423d981a 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -8,12 +8,15 @@ use super::{ instance::wasm_instance_t, module::wasm_module_t, store::{wasm_store_t, StoreRef}, + types::wasm_byte_vec_t, }; use crate::error::update_last_error; use std::convert::TryInto; use std::ffi::CStr; use std::os::raw::c_char; use std::slice; +#[cfg(feature = "pirita_file")] +use wasmer_api::{AsStoreMut, Imports, Module}; use std::sync::{Arc, Mutex}; use std::{ convert::TryFrom, @@ -813,6 +816,135 @@ pub unsafe extern "C" fn wasi_config_overwrite_stderr( .stderr(Box::from_raw(stderr_overwrite)); } + +#[repr(C)] +pub struct wasi_filesystem_t { + ptr: *const c_char, + size: usize, +} + +#[no_mangle] +pub unsafe extern "C" fn wasi_filesystem_init_static_memory( + volume_bytes: Option<&wasm_byte_vec_t>, +) -> Option> { + let volume_bytes = volume_bytes.as_ref()?; + Some(Box::new(wasi_filesystem_t { + ptr: { + let ptr = (volume_bytes.data.as_ref()?) as *const _ as *const c_char; + if ptr.is_null() { + return None; + } + ptr + }, + size: volume_bytes.size, + })) +} + +#[no_mangle] +pub unsafe extern "C" fn wasi_filesystem_delete(ptr: *mut wasi_filesystem_t) { + let _ = Box::from_raw(ptr); +} + +/// Initializes the `imports` with an import object that links to +/// the custom file system +#[cfg(feature = "pirita_file")] +#[no_mangle] +pub unsafe extern "C" fn wasi_env_with_filesystem( + config: Box, + store: Option<&mut wasm_store_t>, + module: Option<&wasm_module_t>, + fs: Option<&wasi_filesystem_t>, + imports: Option<&mut wasm_extern_vec_t>, + package: *const c_char, +) -> Option> { + wasi_env_with_filesystem_inner(config, store, module, fs, imports, package) +} + +#[cfg(feature = "pirita_file")] +unsafe fn wasi_env_with_filesystem_inner( + config: Box, + store: Option<&mut wasm_store_t>, + module: Option<&wasm_module_t>, + fs: Option<&wasi_filesystem_t>, + imports: Option<&mut wasm_extern_vec_t>, + package: *const c_char, +) -> Option> { + let store = &mut store?.inner; + let fs = fs.as_ref()?; + let package_str = CStr::from_ptr(package); + let package = package_str.to_str().unwrap_or(""); + let module = &module.as_ref()?.inner; + let imports = imports?; + + let (wasi_env, import_object) = prepare_webc_env( + config, + &mut store.store_mut(), + module, + std::mem::transmute(fs.ptr), // cast wasi_filesystem_t.ptr as &'static [u8] + fs.size, + package, + )?; + + imports_set_buffer(&store, module, import_object, imports)?; + + Some(Box::new(wasi_env_t { + inner: wasi_env, + store: store.clone(), + })) +} + +#[cfg(feature = "pirita_file")] +fn prepare_webc_env( + config: Box, + store: &mut impl AsStoreMut, + module: &Module, + bytes: &'static u8, + len: usize, + package_name: &str, +) -> Option<(WasiFunctionEnv, Imports)> { + use pirita::FsEntryType; + use pirita::StaticFileSystem; + + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + let volumes = pirita::PiritaFile::parse_volumes_from_fileblock(slice).ok()?; + let top_level_dirs = volumes + .into_iter() + .flat_map(|(_, volume)| { + volume + .header + .top_level + .iter() + .cloned() + .filter(|e| e.fs_type == FsEntryType::Dir) + .map(|e| e.text.to_string()) + .collect::>() + .into_iter() + }) + .collect::>(); + + let filesystem = Box::new(StaticFileSystem::init(slice, &package_name)?); + let mut wasi_env = config.state_builder; + + if !config.inherit_stdout { + wasi_env.stdout(Box::new(Pipe::new())); + } + + if !config.inherit_stderr { + wasi_env.stderr(Box::new(Pipe::new())); + } + + wasi_env.set_fs(filesystem); + + for f_name in top_level_dirs.iter() { + wasi_env + .preopen(|p| p.directory(f_name).read(true).write(true).create(true)) + .ok()?; + } + let env = wasi_env.finalize(store).ok()?; + let import_object = env.import_object(store, &module).ok()?; + Some((env, import_object)) +} + #[allow(non_camel_case_types)] pub struct wasi_env_t { /// cbindgen:ignore diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 7eb728cbfc6..7a8d5230c69 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -54,6 +54,7 @@ cfg-if = "1.0" fern = { version = "0.6", features = ["colored"], optional = true } log = { version = "0.4", optional = true } tempfile = "3" +tempdir = "0.3.7" http_req = { version="^0.8", default-features = false, features = ["rust-tls"], optional = true } dirs = { version = "4.0", optional = true } serde_json = { version = "1.0", optional = true } @@ -71,6 +72,18 @@ chrono = { version = "^0.4", default-features = false, features = [ "std", "cloc [target.'cfg(target_os = "linux")'.dependencies] unix_mode = "0.1.3" +[dependencies.wapm-targz-to-pirita] +path = "../../../pirita/crates/wapm-targz-to-pirita" +default-features = false +features = [] +optional = true + +[dependencies.pirita] +path = "../../../pirita/crates/pirita" +default-features = false +features = ["mmap", "webc_runner", "wasi", "emscripten"] +optional = true + [features] # Don't add the compiler features in default, please add them on the Makefile # since we might want to autoconfigure them depending on the availability on the host. @@ -148,6 +161,8 @@ enable-serde = [ "wasmer-wasi/enable-serde", ] +pirita_file = ["pirita", "wapm-targz-to-pirita"] + http = [ "http_req", "dirs", diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index a3b51f0836a..566bf8861f5 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -4,13 +4,15 @@ use super::ObjectFormat; use crate::store::CompilerOptions; use anyhow::{Context, Result}; use clap::Parser; +#[cfg(feature = "pirita_file")] +use pirita::{ParseOptions, PiritaFileMmap}; use std::env; use std::fs; use std::fs::File; use std::io::prelude::*; use std::io::BufWriter; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; @@ -20,26 +22,27 @@ use wasmer_object::{emit_serialized, get_object_for_target}; #[cfg(feature = "static-artifact-create")] pub type PrefixerFn = Box String + Send>; -const WASMER_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_create_exe_main.c"); -const WASMER_DESERIALIZE_HEADER: &str = include_str!("wasmer_deserialize_module.h"); +const WASMER_MAIN_C_SOURCE: &str = include_str!("wasmer_create_exe_main.c"); +#[cfg(feature = "static-artifact-create")] +const WASMER_STATIC_MAIN_C_SOURCE: &str = include_str!("wasmer_static_create_exe_main.c"); #[derive(Debug, Clone)] -struct CrossCompile { +pub(crate) struct CrossCompile { /// Cross-compilation library path. - library_path: Option, + pub(crate) library_path: Option, /// Cross-compilation tarball library path. - tarball: Option, + pub(crate) tarball: Option, /// Specify `zig` binary path - zig_binary_path: Option, + pub(crate) zig_binary_path: Option, } -struct CrossCompileSetup { - target: Triple, - zig_binary_path: PathBuf, - library: PathBuf, - working_dir: PathBuf, +#[derive(Debug)] +pub(crate) struct CrossCompileSetup { + pub(crate) target: Triple, + pub(crate) zig_binary_path: PathBuf, + pub(crate) library: PathBuf, } #[derive(Debug, Parser)] @@ -80,6 +83,12 @@ pub struct CreateExe { #[clap(long = "zig-binary-path")] zig_binary_path: Option, + /// When compiling .webc files in separate steps + /// (create-obj, then create-exe on the resulting dir) + /// the create-exe step needs the path to the .webc file again + #[clap(long = "webc-volume-path")] + webc_volume_path: Option, + /// Object format options /// /// This flag accepts two options: `symbols` or `serialized`. @@ -111,11 +120,6 @@ pub struct CreateExe { impl CreateExe { /// Runs logic for the `compile` subcommand pub fn execute(&self) -> Result<()> { - let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); - let working_dir = tempfile::tempdir()?; - let starting_cd = env::current_dir()?; - let output_path = starting_cd.join(&self.output); - /* Making library_path, tarball zig_binary_path flags require that target_triple flag * is set cannot be encoded with structopt, so we have to perform cli flag validation * manually here */ @@ -155,152 +159,99 @@ impl CreateExe { }) .unwrap_or_default(); - env::set_current_dir(&working_dir)?; + let starting_cd = env::current_dir()?; + let wasm_module_path = starting_cd.join(&self.path); + let output_path = starting_cd.join(&self.output); + let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); - let cross_compilation: Option = if let Some(mut cross_subc) = - cross_compile.or_else(|| { - if self.target_triple.is_some() { - Some(CrossCompile { - library_path: None, - tarball: None, - zig_binary_path: None, - }) - } else { - None - } - }) { - if let ObjectFormat::Serialized = object_format { - return Err(anyhow!( - "Cross-compilation with serialized object format is not implemented." - )); - } + let cross_compilation = Self::get_cross_compile_setup( + cross_compile, + self.target_triple.clone(), + object_format, + &starting_cd, + )?; - let target = if let Some(target_triple) = self.target_triple.clone() { - target_triple - } else { - return Err(anyhow!( - "To cross-compile an executable, you must specify a target triple with --target" - )); - }; - if let Some(tarball_path) = cross_subc.tarball.as_mut() { - if tarball_path.is_relative() { - *tarball_path = starting_cd.join(&tarball_path); - if !tarball_path.exists() { - return Err(anyhow!( - "Tarball path `{}` does not exist.", - tarball_path.display() - )); - } else if tarball_path.is_dir() { - return Err(anyhow!( - "Tarball path `{}` is a directory.", - tarball_path.display() - )); - } - } + #[cfg(feature = "pirita_file")] + { + let working_dir = tempdir::TempDir::new("testpirita")?; + let working_dir = working_dir.path().to_path_buf(); + + if let Ok(pirita) = + PiritaFileMmap::parse(wasm_module_path.clone(), &ParseOptions::default()) + { + return self.create_exe_pirita( + &pirita, + target, + cross_compilation, + &working_dir, + output_path, + object_format, + ); } - let zig_binary_path = - find_zig_binary(cross_subc.zig_binary_path.as_ref().and_then(|p| { - if p.is_absolute() { - p.canonicalize().ok() - } else { - starting_cd.join(p).canonicalize().ok() - } - }))?; - let library = if let Some(v) = cross_subc.library_path.clone() { - v - } else { - { - let libwasmer_path = if self - .target_triple - .clone() - .unwrap_or(Triple::host()) - .operating_system - == wasmer_types::OperatingSystem::Windows - { - "lib/wasmer.lib" - } else { - "lib/libwasmer.a" - }; - let libwasmer_headless_path = if self - .target_triple - .clone() - .unwrap_or(Triple::host()) - .operating_system - == wasmer_types::OperatingSystem::Windows - { - "lib/wasmer.lib" - } else { - "lib/libwasmer-headless.a" - }; - let filename = if let Some(local_tarball) = cross_subc.tarball { - let files = untar(local_tarball)?; - files.clone().into_iter().find(|f| f.contains(libwasmer_headless_path)).or_else(|| - files.into_iter().find(|f| f.contains(libwasmer_path))).ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the provided tarball path.", target)})? - } else { - #[cfg(feature = "http")] - { - let release = http_fetch::get_latest_release()?; - let tarball = http_fetch::download_release(release, target.clone())?; - let files = untar(tarball)?; - files.clone().into_iter().find(|f| f.contains(libwasmer_headless_path)).or_else(|| - files.into_iter().find(|f| f.contains(libwasmer_path))).ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})? - } - #[cfg(not(feature = "http"))] - return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); - }; - filename.into() - } - }; - Some(CrossCompileSetup { - target, - zig_binary_path, - library, - working_dir: working_dir.path().to_path_buf(), - }) - } else { - None - }; + } let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + // Object is likely a file created from create-obj + #[cfg(feature = "pirita_file")] + if self.path.is_dir() { + let pirita_volume_path = self.webc_volume_path.clone() + .ok_or(anyhow::anyhow!("If compiling using a directory (created by create-obj), you need to also specify the webc path again using --webc-volume-path because the volumes.o is not part of the directory"))?; + + let file = PiritaFileMmap::parse(pirita_volume_path.clone(), &ParseOptions::default()) + .map_err(|e| { + anyhow::anyhow!( + "could not parse {} as webc: {e}", + pirita_volume_path.display() + ) + })?; + + let volume_bytes = file.get_volumes_as_fileblock(); + + return self.link_exe_from_dir( + volume_bytes.as_slice(), + &store, + &target, + cross_compilation, + &self.path, + output_path, + object_format, + ); + } + println!("Compiler: {}", compiler_type.to_string()); println!("Target: {}", target.triple()); println!("Format: {:?}", object_format); #[cfg(not(windows))] - let wasm_object_path = working_dir.path().join("wasm.o"); + let wasm_object_path = PathBuf::from("wasm.o"); #[cfg(windows)] - let wasm_object_path = working_dir.path().join("wasm.obj"); - - let wasm_module_path = starting_cd.join(&self.path); - - let static_defs_header_path: PathBuf = working_dir.path().join("static_defs.h"); + let wasm_object_path = PathBuf::from("wasm.obj"); if let Some(header_path) = self.header.as_ref() { /* In this case, since a header file is given, the input file is expected to be an * object created with `create-obj` subcommand */ let header_path = starting_cd.join(&header_path); - std::fs::copy(&header_path, &static_defs_header_path) + std::fs::copy(&header_path, Path::new("static_defs.h")) .context("Could not access given header file")?; - let object_file_path = wasm_module_path; if let Some(setup) = cross_compilation.as_ref() { self.compile_zig( output_path, - object_file_path, - static_defs_header_path, + wasm_module_path, + std::path::Path::new("static_defs.h").into(), setup, + &[], + None, + None, )?; } else { self.link( - static_defs_header_path, - LinkCode { - object_paths: vec![object_file_path, "main_obj.obj".into()], - output_path, - working_dir: working_dir.path().to_path_buf(), - ..Default::default() - }, + output_path, + wasm_module_path, + std::path::Path::new("static_defs.h").into(), + &[], + None, + None, )?; } } else { @@ -310,50 +261,15 @@ impl CreateExe { .context("failed to compile Wasm")?; let bytes = module.serialize()?; let mut obj = get_object_for_target(target.triple())?; - emit_serialized(&mut obj, &bytes, target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple(), "WASMER_MODULE")?; let mut writer = BufWriter::new(File::create(&wasm_object_path)?); obj.write_stream(&mut writer) .map_err(|err| anyhow::anyhow!(err.to_string()))?; writer.flush()?; drop(writer); - // Write down header file that includes deserialize function - { - let mut writer = BufWriter::new(File::create(&static_defs_header_path)?); - writer.write_all(WASMER_DESERIALIZE_HEADER.as_bytes())?; - writer.flush()?; - } - - // write C src to disk - let c_src_path: PathBuf = working_dir.path().join("wasmer_main.c"); - #[cfg(not(windows))] - let c_src_obj: PathBuf = working_dir.path().join("wasmer_main.o"); - #[cfg(windows)] - let c_src_obj: PathBuf = working_dir.path().join("wasmer_main.obj"); - { - let mut c_src_file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&c_src_path) - .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; - } - run_c_compile( - &c_src_path, - &c_src_obj, - static_defs_header_path, - self.target_triple.clone(), - ) - .context("Failed to compile C source code")?; - LinkCode { - object_paths: vec![c_src_obj, wasm_object_path], - output_path, - additional_libraries: self.libraries.clone(), - target: self.target_triple.clone(), - ..Default::default() - } - .run() - .context("Failed to link objects together")?; + let cli_given_triple = self.target_triple.clone(); + self.compile_c(wasm_object_path, cli_given_triple, output_path)?; } #[cfg(not(feature = "static-artifact-create"))] ObjectFormat::Symbols => { @@ -380,31 +296,33 @@ impl CreateExe { ); // Write object file with functions let object_file_path: std::path::PathBuf = - working_dir.path().join("functions.o"); + std::path::Path::new("functions.o").into(); let mut writer = BufWriter::new(File::create(&object_file_path)?); obj.write_stream(&mut writer) .map_err(|err| anyhow::anyhow!(err.to_string()))?; writer.flush()?; // Write down header file that includes pointer arrays and the deserialize function - let mut writer = BufWriter::new(File::create(&static_defs_header_path)?); + let mut writer = BufWriter::new(File::create("static_defs.h")?); writer.write_all(header_file_src.as_bytes())?; writer.flush()?; if let Some(setup) = cross_compilation.as_ref() { self.compile_zig( output_path, object_file_path, - static_defs_header_path, + std::path::Path::new("static_defs.h").into(), setup, + &[], + None, + None, )?; } else { self.link( - static_defs_header_path, - LinkCode { - object_paths: vec![object_file_path, "main_obj.obj".into()], - output_path, - working_dir: working_dir.path().to_path_buf(), - ..Default::default() - }, + output_path, + object_file_path, + std::path::Path::new("static_defs.h").into(), + &[], + None, + None, )?; } } @@ -427,27 +345,192 @@ impl CreateExe { Ok(()) } + pub(crate) fn get_cross_compile_setup( + cross_compile: Option, + target_triple: Option, + object_format: ObjectFormat, + starting_cd: &PathBuf, + ) -> Result, anyhow::Error> { + if let Some(mut cross_subc) = cross_compile.or_else(|| { + if target_triple.is_some() { + Some(CrossCompile { + library_path: None, + tarball: None, + zig_binary_path: None, + }) + } else { + None + } + }) { + if let ObjectFormat::Serialized = object_format { + return Err(anyhow!( + "Cross-compilation with serialized object format is not implemented." + )); + } + + let target = if let Some(target_triple) = target_triple.clone() { + target_triple + } else { + return Err(anyhow!( + "To cross-compile an executable, you must specify a target triple with --target" + )); + }; + if let Some(tarball_path) = cross_subc.tarball.as_mut() { + if tarball_path.is_relative() { + *tarball_path = starting_cd.join(&tarball_path); + if !tarball_path.exists() { + return Err(anyhow!( + "Tarball path `{}` does not exist.", + tarball_path.display() + )); + } else if tarball_path.is_dir() { + return Err(anyhow!( + "Tarball path `{}` is a directory.", + tarball_path.display() + )); + } + } + } + let zig_binary_path = + find_zig_binary(cross_subc.zig_binary_path.as_ref().and_then(|p| { + if p.is_absolute() { + p.canonicalize().ok() + } else { + starting_cd.join(p).canonicalize().ok() + } + }))?; + let library = if let Some(v) = cross_subc.library_path.clone() { + v.clone().canonicalize().unwrap_or(v.clone()) + } else { + { + let libwasmer_path = if target_triple + .clone() + .unwrap_or(Triple::host()) + .operating_system + == wasmer_types::OperatingSystem::Windows + { + "lib/wasmer.lib" + } else { + "lib/libwasmer.a" + }; + let tarball_dir; + let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() { + let target_file_path = local_tarball + .parent() + .and_then(|parent| Some(parent.join(local_tarball.file_stem()?))) + .unwrap_or(local_tarball.clone()); + + let target_file_path = target_file_path + .parent() + .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) + .unwrap_or(target_file_path.clone()); + + let _ = std::fs::create_dir_all(&target_file_path); + let files = untar(local_tarball.clone(), target_file_path.clone())?; + tarball_dir = target_file_path + .clone() + .canonicalize() + .unwrap_or(target_file_path.clone()); + files.into_iter().find(|f| f.contains(libwasmer_path)).ok_or_else(|| { + anyhow!("Could not find libwasmer for {} target in the provided tarball path.", target)})? + } else { + #[cfg(feature = "http")] + { + let release = http_fetch::get_latest_release()?; + let tarball = http_fetch::download_release(release, target.clone())?; + let target_file_path = tarball + .parent() + .and_then(|parent| Some(parent.join(tarball.file_stem()?))) + .unwrap_or(tarball.clone()); + + let target_file_path = target_file_path + .parent() + .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) + .unwrap_or(target_file_path.clone()); + + tarball_dir = target_file_path + .clone() + .canonicalize() + .unwrap_or(target_file_path.clone()); + let files = untar(tarball.clone(), target_file_path.clone())?; + files.into_iter().find(|f| f.contains(libwasmer_path)).ok_or_else(|| { + anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})? + } + #[cfg(not(feature = "http"))] + return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); + }; + tarball_dir.join(&filename) + } + }; + let ccs = CrossCompileSetup { + target, + zig_binary_path, + library, + }; + println!("{:#?}", ccs); + Ok(Some(ccs)) + } else { + Ok(None) + } + } + + fn compile_c( + &self, + wasm_object_path: PathBuf, + target_triple: Option, + output_path: PathBuf, + ) -> anyhow::Result<()> { + let tempdir = tempdir::TempDir::new("compile-c")?; + let tempdir_path = tempdir.path(); + + // write C src to disk + let c_src_path = tempdir_path.clone().join("wasmer_main.c"); + #[cfg(not(windows))] + let c_src_obj = tempdir_path.clone().join("wasmer_main.o"); + #[cfg(windows)] + let c_src_obj = tempdir_path.clone().join("wasmer_main.obj"); + + std::fs::write( + &c_src_path, + WASMER_MAIN_C_SOURCE + .replace("// WASI_DEFINES", "#define WASI") + .as_bytes(), + )?; + + run_c_compile(&c_src_path, &c_src_obj, target_triple.clone()) + .context("Failed to compile C source code")?; + LinkCode { + object_paths: vec![c_src_obj, wasm_object_path], + output_path, + additional_libraries: self.libraries.clone(), + target: target_triple, + ..Default::default() + } + .run() + .context("Failed to link objects together")?; + + Ok(()) + } + fn compile_zig( &self, output_path: PathBuf, object_path: PathBuf, - mut header_path: PathBuf, + mut header_code_path: PathBuf, setup: &CrossCompileSetup, + pirita_atoms: &[String], + pirita_main_atom: Option<&str>, + pirita_volume_path: Option, ) -> anyhow::Result<()> { - debug_assert!( - header_path.is_absolute(), - "compile_zig() called with relative header file path {}", - header_path.display() - ); + let tempdir = tempdir::TempDir::new("wasmer-static-compile-zig")?; + let tempdir_path = tempdir.path(); + let c_src_path = tempdir_path.join("wasmer_main.c"); let CrossCompileSetup { ref target, ref zig_binary_path, ref library, - ref working_dir, } = setup; - let c_src_path = working_dir.join("wasmer_main.c"); let mut libwasmer_path = library.to_path_buf(); - println!("Library Path: {}", libwasmer_path.display()); /* Cross compilation is only possible with zig */ println!("Using zig binary: {}", zig_binary_path.display()); @@ -461,17 +544,20 @@ impl CreateExe { .unwrap() .to_string(); libwasmer_path.pop(); - { - let mut c_src_file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&c_src_path) - .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; + + if let Some(entrypoint) = pirita_main_atom.as_ref() { + let c_code = Self::generate_pirita_wasmer_main_c_static(&pirita_atoms, entrypoint); + std::fs::write(&c_src_path, c_code)?; + } else { + std::fs::write(&c_src_path, WASMER_STATIC_MAIN_C_SOURCE)?; } - if !header_path.is_dir() { - header_path.pop(); + if !header_code_path.is_dir() { + header_code_path.pop(); + } + + if header_code_path.display().to_string().is_empty() { + header_code_path = std::env::current_dir()?; } /* Compile main function */ @@ -480,30 +566,25 @@ impl CreateExe { include_dir.pop(); include_dir.push("include"); - let compiler_cmd = match std::process::Command::new("cc").output() { - Ok(_) => "cc", - Err(_) => "gcc", - }; - let mut cmd = Command::new(zig_binary_path); let mut cmd_mut: &mut Command = cmd - .arg(compiler_cmd) - .arg("-w") - .arg("-fgnu-inline-asm") - .arg("-fsanitize=undefined") - .arg("-fsanitize-trap=undefined") + .arg("cc") .arg("-target") .arg(&zig_triple) .arg(&format!("-L{}", libwasmer_path.display())) .arg(&format!("-l:{}", lib_filename)) .arg(&format!("-I{}", include_dir.display())) - .arg(&format!("-I{}", header_path.display())); + .arg(&format!("-I{}", header_code_path.display())); if !zig_triple.contains("windows") { cmd_mut = cmd_mut.arg("-lunwind"); } + cmd_mut = cmd_mut.arg(&object_path).arg(&c_src_path); + + if let Some(volume_obj) = pirita_volume_path.as_ref() { + cmd_mut = cmd_mut.arg(volume_obj.clone()); + } + cmd_mut - .arg(&object_path) - .arg(&c_src_path) .arg("-o") .arg(&output_path) .output() @@ -518,14 +599,462 @@ impl CreateExe { Ok(()) } + // Write the volumes.o file + #[cfg(feature = "pirita_file")] + fn write_volume_obj( + volume_bytes: &[u8], + target: &Target, + output_path: &Path, + ) -> anyhow::Result { + #[cfg(not(windows))] + let volume_object_path = output_path.clone().join("volumes.o"); + #[cfg(windows)] + let volume_object_path = output_path.clone().join("volumes.obj"); + + let mut volumes_object = get_object_for_target(&target.triple())?; + emit_serialized( + &mut volumes_object, + volume_bytes, + target.triple(), + "VOLUMES", + )?; + + let mut writer = BufWriter::new(File::create(&volume_object_path)?); + volumes_object + .write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + drop(writer); + + Ok(volume_object_path.clone()) + } + + #[cfg(feature = "pirita_file")] + pub(crate) fn create_objs_pirita( + store: &Store, + file: &PiritaFileMmap, + target: &Target, + output_path: &Path, + object_format: ObjectFormat, + ) -> anyhow::Result<()> { + if !output_path.is_dir() { + return Err(anyhow::anyhow!( + "Expected {} to be an output directory, not a file", + output_path.display() + )); + } + + std::fs::create_dir_all(output_path)?; + std::fs::create_dir_all(output_path.clone().join("atoms"))?; + + let atom_to_run = file + .manifest + .entrypoint + .as_ref() + .and_then(|s| file.get_atom_name_for_command("wasi", s).ok()); + if let Some(atom_to_run) = atom_to_run.as_ref() { + std::fs::write(output_path.clone().join("entrypoint"), atom_to_run)?; + } + + for (atom_name, atom_bytes) in file.get_all_atoms() { + std::fs::create_dir_all(output_path.clone().join("atoms"))?; + + #[cfg(not(windows))] + let object_path = output_path + .clone() + .join("atoms") + .join(&format!("{atom_name}.o")); + #[cfg(windows)] + let object_path = output_path + .clone() + .join("atoms") + .join(&format!("{atom_name}.obj")); + + std::fs::create_dir_all(output_path.clone().join("atoms").join(&atom_name))?; + + let header_path = output_path + .clone() + .join("atoms") + .join(&atom_name) + .join(&format!("static_defs.h")); + + match object_format { + ObjectFormat::Serialized => { + let module = Module::new(&store, &atom_bytes) + .context(format!("Failed to compile atom {atom_name:?} to wasm"))?; + let bytes = module.serialize()?; + let mut obj = get_object_for_target(target.triple())?; + let atom_name_uppercase = atom_name.to_uppercase(); + emit_serialized(&mut obj, &bytes, target.triple(), &atom_name_uppercase)?; + + let mut writer = BufWriter::new(File::create(&object_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + drop(writer); + } + #[cfg(feature = "static-artifact-create")] + ObjectFormat::Symbols => { + let engine = store.engine(); + let engine_inner = engine.inner(); + let compiler = engine_inner.compiler()?; + let features = engine_inner.features(); + let tunables = store.tunables(); + let prefixer: Option = None; + let (module_info, obj, metadata_length, symbol_registry) = + Artifact::generate_object( + compiler, + &atom_bytes, + prefixer, + &target, + tunables, + features, + )?; + + let header_file_src = crate::c_gen::staticlib_header::generate_header_file( + &module_info, + &*symbol_registry, + metadata_length, + ); + + let mut writer = BufWriter::new(File::create(&object_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + + let mut writer = BufWriter::new(File::create(&header_path)?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + } + #[cfg(not(feature = "static-artifact-create"))] + ObjectFormat::Symbols => { + return Err(anyhow!("Objects cannot be compiled in format \"symbols\" without static-artifact-create feature")); + } + } + } + + Ok(()) + } + + #[cfg(feature = "pirita_file")] + fn link_exe_from_dir( + &self, + volume_bytes: &[u8], + _store: &Store, + target: &Target, + cross_compilation: Option, + working_dir: &Path, + output_path: PathBuf, + object_format: ObjectFormat, + ) -> anyhow::Result<()> { + let tempdir = tempdir::TempDir::new("link-exe-from-dir")?; + let tempdir_path = tempdir.path(); + + let entrypoint = std::fs::read_to_string(working_dir.clone().join("entrypoint")) + .map_err(|_| anyhow::anyhow!("file has no entrypoint to run"))?; + + if !working_dir.clone().join("atoms").exists() { + return Err(anyhow::anyhow!("file has no atoms to compile")); + } + + let mut atom_names = Vec::new(); + for obj in std::fs::read_dir(working_dir.clone().join("atoms"))? { + let path = obj?.path(); + if !path.is_dir() { + if let Some(s) = path.file_stem() { + atom_names.push( + s.to_str() + .ok_or(anyhow::anyhow!("wrong atom name"))? + .to_string(), + ); + } + } + } + + match object_format { + ObjectFormat::Serialized => { + let mut link_objects: Vec = Vec::new(); + + let volume_object_path = + Self::write_volume_obj(volume_bytes, target, &tempdir_path)?; + + link_objects.push(volume_object_path); + + #[cfg(not(windows))] + let c_src_obj = working_dir.clone().join("wasmer_main.o"); + #[cfg(windows)] + let c_src_obj = working_dir.clone().join("wasmer_main.obj"); + + for obj in std::fs::read_dir(working_dir.clone().join("atoms"))? { + let path = obj?.path(); + if !path.is_dir() { + link_objects.push(path.to_path_buf()); + } + } + + let c_code = Self::generate_pirita_wasmer_main_c(&atom_names, &entrypoint); + + let c_src_path = working_dir.clone().join("wasmer_main.c"); + + std::fs::write(&c_src_path, c_code.as_bytes()) + .context("Failed to open C source code file")?; + + if let Some(setup) = cross_compilation { + let CrossCompileSetup { + ref target, + ref zig_binary_path, + ref library, + } = setup; + + let mut libwasmer_path = library.to_path_buf(); + let zig_triple = triple_to_zig_triple(target); + + // Cross compilation is only possible with zig + println!("Library Path: {}", libwasmer_path.display()); + println!("Using zig binary: {}", zig_binary_path.display()); + eprintln!("Using zig target triple: {}", &zig_triple); + + let lib_filename = libwasmer_path + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + + /* Compile main function */ + let compilation = { + libwasmer_path.pop(); + let mut include_dir = libwasmer_path.clone(); + include_dir.pop(); + include_dir.push("include"); + eprintln!("include dir: {}", include_dir.display()); + let mut cmd = Command::new(zig_binary_path); + let mut cmd_mut: &mut Command = cmd + .arg("cc") + .arg("--verbose") + .arg("-target") + .arg(&zig_triple) + .arg(&format!("-L{}", libwasmer_path.display())) + .arg(&format!("-l:{}", lib_filename)) + .arg(&format!("-I{}", include_dir.display())); + if !zig_triple.contains("windows") { + cmd_mut = cmd_mut.arg("-lunwind"); + } + cmd_mut + .args(link_objects.into_iter()) + .arg(&c_src_path) + .arg("-o") + .arg(&output_path) + .output() + .context("Could not execute `zig`")? + }; + if !compilation.status.success() { + return Err(anyhow::anyhow!(String::from_utf8_lossy( + &compilation.stderr + ) + .to_string())); + } + } else { + run_c_compile(c_src_path.as_path(), &c_src_obj, self.target_triple.clone()) + .context("Failed to compile C source code")?; + + link_objects.push(c_src_obj.clone()); + LinkCode { + object_paths: link_objects, + output_path, + additional_libraries: self.libraries.clone(), + target: self.target_triple.clone(), + ..Default::default() + } + .run() + .context("Failed to link objects together")?; + } + } + ObjectFormat::Symbols => { + let object_file_path = working_dir + .clone() + .join("atoms") + .join(&format!("{entrypoint}.o")); + let static_defs_file_path = working_dir + .clone() + .join("atoms") + .join(&entrypoint) + .join("static_defs.h"); + let volumes_obj_path = Self::write_volume_obj(volume_bytes, target, &tempdir_path)?; + + if let Some(setup) = cross_compilation.as_ref() { + self.compile_zig( + output_path, + object_file_path, + static_defs_file_path, + setup, + &atom_names, + Some(&entrypoint), + Some(volumes_obj_path), + )?; + } else { + self.link( + output_path, + object_file_path, + static_defs_file_path, + &atom_names, + Some(&entrypoint), + Some(volumes_obj_path), + )?; + } + } + } + + Ok(()) + } + + fn normalize_atom_name(s: &str) -> String { + s.chars() + .filter_map(|c| { + if char::is_alphabetic(c) { + Some(c) + } else if c == '-' { + Some('_') + } else { + None + } + }) + .collect() + } + + fn generate_pirita_wasmer_main_c_static(atom_names: &[String], atom_to_run: &str) -> String { + let mut c_code_to_instantiate = String::new(); + let mut deallocate_module = String::new(); + + let atom_to_run = Self::normalize_atom_name(&atom_to_run); + + for atom_name in atom_names.iter() { + let atom_name = Self::normalize_atom_name(&atom_name); + + c_code_to_instantiate.push_str(&format!( + " + + wasm_module_t *atom_{atom_name} = wasmer_static_module_new(store, \"{atom_name}\"); + + if (!atom_{atom_name}) {{ + fprintf(stderr, \"Failed to create module from atom \\\"{atom_name}\\\"\\n\"); + print_wasmer_error(); + return -1; + }} + " + )); + deallocate_module.push_str(&format!("wasm_module_delete(atom_{atom_name});")); + } + + c_code_to_instantiate.push_str(&format!("wasm_module_t *module = atom_{atom_to_run};")); + + let c_code = WASMER_STATIC_MAIN_C_SOURCE + .replace("#define WASI", "#define WASI\r\n#define WASI_PIRITA") + .replace("// INSTANTIATE_MODULES", &c_code_to_instantiate) + .replace("##atom-name##", &atom_to_run) + .replace("wasm_module_delete(module);", &deallocate_module); + + c_code + } + + #[cfg(feature = "pirita_file")] + fn generate_pirita_wasmer_main_c(atom_names: &[String], atom_to_run: &str) -> String { + let mut c_code_to_add = String::new(); + let mut c_code_to_instantiate = String::new(); + let mut deallocate_module = String::new(); + + for atom_name in atom_names.iter() { + let atom_name = Self::normalize_atom_name(atom_name); + let atom_name_uppercase = atom_name.to_uppercase(); + + c_code_to_add.push_str(&format!( + " + extern size_t {atom_name_uppercase}_LENGTH asm(\"{atom_name_uppercase}_LENGTH\"); + extern char {atom_name_uppercase}_DATA asm(\"{atom_name_uppercase}_DATA\"); + " + )); + + c_code_to_instantiate.push_str(&format!(" + wasm_byte_vec_t atom_{atom_name}_byte_vec = {{ + .size = {atom_name_uppercase}_LENGTH, + .data = &{atom_name_uppercase}_DATA, + }}; + wasm_module_t *atom_{atom_name} = wasm_module_deserialize(store, &atom_{atom_name}_byte_vec); + + if (!atom_{atom_name}) {{ + fprintf(stderr, \"Failed to create module from atom \\\"{atom_name}\\\"\\n\"); + print_wasmer_error(); + return -1; + }} + ")); + deallocate_module.push_str(&format!("wasm_module_delete(atom_{atom_name});")); + } + + c_code_to_instantiate.push_str(&format!("wasm_module_t *module = atom_{atom_to_run};")); + + let c_code = WASMER_MAIN_C_SOURCE + .replace("#define WASI", "#define WASI\r\n#define WASI_PIRITA") + .replace("// DECLARE_MODULES", &c_code_to_add) + .replace("// INSTANTIATE_MODULES", &c_code_to_instantiate) + .replace("##atom-name##", &atom_to_run) + .replace("wasm_module_delete(module);", &deallocate_module); + + c_code + } + + #[cfg(feature = "pirita_file")] + fn create_exe_pirita( + &self, + file: &PiritaFileMmap, + target: Target, + cross_compilation: Option, + working_dir: &Path, + output_path: PathBuf, + object_format: ObjectFormat, + ) -> anyhow::Result<()> { + let _ = std::fs::create_dir_all(&working_dir); + let (store, _) = self.compiler.get_store_for_target(target.clone())?; + + Self::create_objs_pirita(&store, file, &target, working_dir, object_format)?; + + let volumes_obj = file.get_volumes_as_fileblock(); + self.link_exe_from_dir( + volumes_obj.as_slice(), + &store, + &target, + cross_compilation, + working_dir, + output_path, + object_format, + )?; + + Ok(()) + } + #[cfg(feature = "static-artifact-create")] - fn link(&self, mut header_path: PathBuf, linkcode: LinkCode) -> anyhow::Result<()> { - debug_assert!( - header_path.is_absolute(), - "link() called with relative header file path {}", - header_path.display() - ); - let c_src_path: PathBuf = linkcode.working_dir.join("wasmer_main.c"); + fn link( + &self, + output_path: PathBuf, + object_path: PathBuf, + mut header_code_path: PathBuf, + pirita_atoms: &[String], + pirita_main_atom: Option<&str>, + pirita_volume_path: Option, + ) -> anyhow::Result<()> { + let tempdir = tempdir::TempDir::new("wasmer-static-compile")?; + let tempdir_path = tempdir.path(); + + let mut object_paths = vec![object_path, "main_obj.obj".into()]; + if let Some(volume_obj) = pirita_volume_path.as_ref() { + object_paths.push(volume_obj.to_path_buf()); + } + + let linkcode = LinkCode { + object_paths, + output_path, + ..Default::default() + }; + let c_src_path = tempdir_path.join("wasmer_main.c"); let mut libwasmer_path = get_libwasmer_path()? .canonicalize() .context("Failed to find libwasmer")?; @@ -539,27 +1068,43 @@ impl CreateExe { .unwrap() .to_string(); libwasmer_path.pop(); - { - let mut c_src_file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&c_src_path) - .context("Failed to open C source code file")?; - c_src_file.write_all(WASMER_MAIN_C_SOURCE)?; + + if let Some(entrypoint) = pirita_main_atom.as_ref() { + let c_code = Self::generate_pirita_wasmer_main_c_static(&pirita_atoms, entrypoint); + std::fs::write(&c_src_path, c_code)?; + } else { + std::fs::write(&c_src_path, WASMER_STATIC_MAIN_C_SOURCE)?; + } + + if !header_code_path.is_dir() { + header_code_path.pop(); } - if !header_path.is_dir() { - header_path.pop(); + if header_code_path.display().to_string().is_empty() { + header_code_path = std::env::current_dir()?; } + let wasmer_include_dir = get_wasmer_include_directory()?; + let wasmer_h_path = wasmer_include_dir.join("wasmer.h"); + if !wasmer_h_path.exists() { + return Err(anyhow::anyhow!( + "Could not find wasmer.h in {}", + wasmer_include_dir.display() + )); + } + let wasm_h_path = wasmer_include_dir.join("wasm.h"); + if !wasm_h_path.exists() { + return Err(anyhow::anyhow!( + "Could not find wasm.h in {}", + wasmer_include_dir.display() + )); + } + std::fs::copy(wasmer_h_path, header_code_path.join("wasmer.h"))?; + std::fs::copy(wasm_h_path, header_code_path.join("wasm.h"))?; + /* Compile main function */ let compilation = { - let compiler_cmd = match Command::new("cc").output() { - Ok(_) => "cc", - Err(_) => "gcc", - }; - - Command::new(compiler_cmd) + Command::new("cc") .arg("-c") .arg(&c_src_path) .arg(if linkcode.optimization_flag.is_empty() { @@ -568,7 +1113,6 @@ impl CreateExe { linkcode.optimization_flag.as_str() }) .arg(&format!("-L{}", libwasmer_path.display())) - .arg(&format!("-I{}", get_wasmer_include_directory()?.display())) .arg(&format!("-l:{}", lib_filename)) //.arg("-lwasmer") // Add libraries required per platform. @@ -583,7 +1127,7 @@ impl CreateExe { .arg("-ldl") .arg("-lm") .arg("-pthread") - .arg(&format!("-I{}", header_path.display())) + .arg(&format!("-I{}", header_code_path.display())) .arg("-v") .arg("-o") .arg("main_obj.obj") @@ -600,6 +1144,11 @@ impl CreateExe { } } +#[test] +fn test_normalize_atom_name() { + assert_eq!(CreateExe::normalize_atom_name("atom-name-with-dash"), "atom_name_with_dash".to_string()); +} + fn triple_to_zig_triple(target_triple: &Triple) -> String { let arch = match target_triple.architecture { wasmer_types::Architecture::X86_64 => "x86_64".into(), @@ -637,22 +1186,34 @@ fn get_wasmer_dir() -> anyhow::Result { fn get_wasmer_include_directory() -> anyhow::Result { let mut path = get_wasmer_dir()?; + if path.clone().join("wasmer.h").exists() { + return Ok(path); + } path.push("include"); + if !path.clone().join("wasmer.h").exists() { + println!( + "wasmer.h does not exist in {}, will probably default to the system path", + path.canonicalize().unwrap().display() + ); + } Ok(path) } /// path to the static libwasmer fn get_libwasmer_path() -> anyhow::Result { - let mut path = get_wasmer_dir()?; - path.push("lib"); + let path = get_wasmer_dir()?; // TODO: prefer headless Wasmer if/when it's a separate library. #[cfg(not(windows))] - path.push("libwasmer.a"); + let libwasmer_static_name = "libwasmer.a"; #[cfg(windows)] - path.push("wasmer.lib"); + let libwasmer_static_name = "libwasmer.lib"; - Ok(path) + if path.exists() && path.join(libwasmer_static_name).exists() { + Ok(path.join(libwasmer_static_name)) + } else { + Ok(path.join("lib").join(libwasmer_static_name)) + } } /// path to library tarball cache dir @@ -667,15 +1228,8 @@ fn get_libwasmer_cache_path() -> anyhow::Result { fn run_c_compile( path_to_c_src: &Path, output_name: &Path, - mut header_path: PathBuf, target: Option, ) -> anyhow::Result<()> { - debug_assert!( - header_path.is_absolute(), - "run_c_compile() called with relative header file path {}", - header_path.display() - ); - #[cfg(not(windows))] let c_compiler = "cc"; // We must use a C++ compiler on Windows because wasm.h uses `static_assert` @@ -683,17 +1237,14 @@ fn run_c_compile( #[cfg(windows)] let c_compiler = "clang++"; - if !header_path.is_dir() { - header_path.pop(); - } - let mut command = Command::new(c_compiler); let command = command + .arg("-Wall") .arg("-O2") .arg("-c") .arg(path_to_c_src) - .arg(&format!("-I{}", header_path.display())) - .arg(&format!("-I{}", get_wasmer_include_directory()?.display())); + .arg("-I") + .arg(get_wasmer_include_directory()?); let command = if let Some(target) = target { command.arg("-target").arg(format!("{}", target)) @@ -702,6 +1253,13 @@ fn run_c_compile( }; let output = command.arg("-o").arg(output_name).output()?; + eprintln!( + "run_c_compile: stdout: {}\n\nstderr: {}", + std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"), + std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes") + ); if !output.status.success() { bail!( @@ -732,8 +1290,6 @@ struct LinkCode { libwasmer_path: PathBuf, /// The target to link the executable for. target: Option, - /// Working directory - working_dir: PathBuf, } impl Default for LinkCode { @@ -750,7 +1306,6 @@ impl Default for LinkCode { output_path: PathBuf::from("a.out"), libwasmer_path: get_libwasmer_path().unwrap(), target: None, - working_dir: env::current_dir().expect("could not get current dir from environment"), } } } @@ -767,6 +1322,7 @@ impl LinkCode { ); let mut command = Command::new(&self.linker_path); let command = command + .arg("-Wall") .arg(&self.optimization_flag) .args( self.object_paths @@ -795,7 +1351,8 @@ impl LinkCode { .iter() .map(|lib| format!("-l{}", lib)); let command = command.args(link_against_extra_libs); - let output = command.arg("-o").arg(&self.output_path).output()?; + let command = command.arg("-o").arg(&self.output_path); + let output = command.output()?; if !output.status.success() { bail!( @@ -989,12 +1546,36 @@ mod http_fetch { } Ok(response) }); + match super::get_libwasmer_cache_path() { + Ok(mut cache_path) => { + cache_path.push(&filename); + if !cache_path.exists() { + if let Err(err) = std::fs::copy(&filename, &cache_path) { + eprintln!( + "Could not store tarball to cache path `{}`: {}", + cache_path.display(), + err + ); + } else { + eprintln!( + "Cached tarball to cache path `{}`.", + cache_path.display() + ); + } + } + } + Err(err) => { + eprintln!( + "Could not determine cache path for downloaded binaries.: {}", + err + ); + } + } let _response = download_thread .join() .expect("Could not join downloading thread"); match super::get_libwasmer_cache_path() { Ok(mut cache_path) => { - let _ = std::fs::create_dir_all(&cache_path); cache_path.push(&filename); if !cache_path.exists() { if let Err(err) = std::fs::copy(&filename, &cache_path) { @@ -1025,7 +1606,7 @@ mod http_fetch { } } -fn untar(tarball: std::path::PathBuf) -> Result> { +fn untar(tarball: std::path::PathBuf, target: std::path::PathBuf) -> Result> { let files = std::process::Command::new("tar") .arg("-tf") .arg(&tarball) @@ -1041,9 +1622,12 @@ fn untar(tarball: std::path::PathBuf) -> Result> { .map(|s| s.to_string()) .collect::>(); + let _ = std::fs::create_dir_all(&target); let _output = std::process::Command::new("tar") .arg("-xf") .arg(&tarball) + .arg("-C") + .arg(&target) .output() .expect("failed to execute process"); Ok(files) @@ -1112,4 +1696,4 @@ fn find_zig_binary(path: Option) -> Result { } else { Ok(retval) } -} +} \ No newline at end of file diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs index 79995fd6f15..a938446f31c 100644 --- a/lib/cli/src/commands/create_obj.rs +++ b/lib/cli/src/commands/create_obj.rs @@ -1,19 +1,26 @@ -//! Create a compiled standalone object file for a given Wasm file. +#![allow(dead_code)] +//! Create a standalone native executable for a given Wasm file. use super::ObjectFormat; -use crate::{commands::PrefixerFn, store::CompilerOptions}; +use crate::{ + commands::{CrossCompile, PrefixerFn}, + store::CompilerOptions, +}; use anyhow::{Context, Result}; use clap::Parser; +#[cfg(feature = "pirita_file")] +use pirita::{ParseOptions, PiritaFileMmap}; use std::env; use std::fs; use std::fs::File; use std::io::prelude::*; use std::io::BufWriter; use std::path::PathBuf; +use std::process::Command; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; -const WASMER_SERIALIZED_HEADER: &[u8] = include_bytes!("wasmer_deserialize_module.h"); +const WASMER_SERIALIZED_HEADER: &[u8] = include_bytes!("wasmer_create_exe.h"); #[derive(Debug, Parser)] /// The options for the `wasmer create-exe` subcommand @@ -83,16 +90,27 @@ impl CreateObj { Target::new(target_triple.clone(), features) }) .unwrap_or_default(); - let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + + let starting_cd = env::current_dir()?; + let wasm_module_path = starting_cd.join(&self.path); + let output_path = starting_cd.join(&self.output); let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); + #[cfg(feature = "pirita_file")] + { + if let Ok(pirita) = + PiritaFileMmap::parse(wasm_module_path.clone(), &ParseOptions::default()) + { + return self.execute_pirita(&pirita, target, output_path, object_format); + } + } + + let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + println!("Compiler: {}", compiler_type.to_string()); println!("Target: {}", target.triple()); println!("Format: {:?}", object_format); - let starting_cd = env::current_dir()?; - - let output_path = starting_cd.join(&self.output); let header_output = self.header_output.clone().unwrap_or_else(|| { let mut retval = self.output.clone(); retval.set_extension("h"); @@ -100,7 +118,6 @@ impl CreateObj { }); let header_output_path = starting_cd.join(&header_output); - let wasm_module_path = starting_cd.join(&self.path); match object_format { ObjectFormat::Serialized => { @@ -108,7 +125,7 @@ impl CreateObj { .context("failed to compile Wasm")?; let bytes = module.serialize()?; let mut obj = get_object_for_target(target.triple())?; - emit_serialized(&mut obj, &bytes, target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple(), "WASMER_MODULE")?; let mut writer = BufWriter::new(File::create(&output_path)?); obj.write_stream(&mut writer) .map_err(|err| anyhow::anyhow!(err.to_string()))?; @@ -150,17 +167,98 @@ impl CreateObj { self.output.display(), header_output.display(), ); - eprintln!("\n---\n"); - eprintln!( - r#"To use, link the object file to your executable and call the `wasmer_object_module_new` function defined in the header file. For example, in the C language: - #include "{}" - - wasm_module_t *module = wasmer_object_module_new(store, "my_module_name"); - "#, - header_output.display(), - ); + Ok(()) + } + #[cfg(feature = "pirita_file")] + fn execute_pirita( + &self, + file: &PiritaFileMmap, + target: Target, + output_path: PathBuf, + object_format: ObjectFormat, + ) -> Result<()> { + use wasmer_object::Object; + if output_path.exists() { + if output_path.is_dir() { + wapm_targz_to_pirita::nuke_dir(&output_path)?; + } + } else { + let _ = std::fs::create_dir_all(&output_path)?; + } + println!( + "outputting create-obj to directory {}", + output_path.display() + ); + let (store, _) = self.compiler.get_store_for_target(target.clone())?; + crate::commands::create_exe::CreateExe::create_objs_pirita( + &store, + file, + &target, + &output_path, + object_format, + )?; Ok(()) } } + +fn link( + output_path: PathBuf, + object_path: PathBuf, + header_code_path: PathBuf, +) -> anyhow::Result<()> { + let libwasmer_path = get_libwasmer_path()? + .canonicalize() + .context("Failed to find libwasmer")?; + println!( + "link output {:?}", + Command::new("cc") + .arg(&header_code_path) + .arg(&format!("-L{}", libwasmer_path.display())) + //.arg(&format!("-I{}", header_code_path.display())) + .arg("-pie") + .arg("-o") + .arg("header_obj.o") + .output()? + ); + //ld -relocatable a.o b.o -o c.o + + println!( + "link output {:?}", + Command::new("ld") + .arg("-relocatable") + .arg(&object_path) + .arg("header_obj.o") + .arg("-o") + .arg(&output_path) + .output()? + ); + + Ok(()) +} + +/// path to the static libwasmer +fn get_libwasmer_path() -> anyhow::Result { + let mut path = get_wasmer_dir()?; + path.push("lib"); + + // TODO: prefer headless Wasmer if/when it's a separate library. + #[cfg(not(windows))] + path.push("libwasmer.a"); + #[cfg(windows)] + path.push("wasmer.lib"); + + Ok(path) +} +fn get_wasmer_dir() -> anyhow::Result { + Ok(PathBuf::from( + env::var("WASMER_DIR") + .or_else(|e| { + option_env!("WASMER_INSTALL_PREFIX") + .map(str::to_string) + .ok_or(e) + }) + .context("Trying to read env var `WASMER_DIR`")?, + )) +} diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 569b37394bc..f464ff040d3 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -216,6 +216,14 @@ impl Run { } fn inner_execute(&self) -> Result<()> { + #[cfg(feature = "pirita_file")] + { + if let Some(pf) = pirita::PiritaContainer::load_mmap(self.path.clone()) { + return pf + .run(&self.command_name.clone().unwrap_or_default()) + .map_err(|e| anyhow!("Could not run PiritaFile: {e}")); + } + } let (mut store, module) = self.get_store_module()?; #[cfg(feature = "emscripten")] { diff --git a/lib/cli/src/commands/wasmer_create_exe.h b/lib/cli/src/commands/wasmer_create_exe.h new file mode 100644 index 00000000000..98705d4fc6f --- /dev/null +++ b/lib/cli/src/commands/wasmer_create_exe.h @@ -0,0 +1,25 @@ +#include "wasmer.h" +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); +extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); + +wasm_module_t* wasmer_module_new(wasm_store_t* store, const char* wasm_name) { + wasm_byte_vec_t module_byte_vec = { + .size = WASMER_MODULE_LENGTH, + .data = (const char*)&WASMER_MODULE_DATA, + }; + wasm_module_t* module = wasm_module_deserialize(store, &module_byte_vec); + + return module; +} + +#ifdef __cplusplus +} +#endif diff --git a/lib/cli/src/commands/wasmer_create_exe_main.c b/lib/cli/src/commands/wasmer_create_exe_main.c index 3fe98f62c45..4fe6a7f9402 100644 --- a/lib/cli/src/commands/wasmer_create_exe_main.c +++ b/lib/cli/src/commands/wasmer_create_exe_main.c @@ -1,16 +1,24 @@ + #include "wasmer.h" -#include "static_defs.h" +//#include "my_wasm.h" + #include #include #include #define own -// TODO: make this define templated so that the Rust code can toggle it on/off -#define WASI -extern wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) asm("wasmer_object_module_new"); +#define WASI +#ifdef WASI_PIRITA +extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH"); +extern char VOLUMES_DATA asm("VOLUMES_DATA"); +// DECLARE_MODULES +#else +extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); +extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); +#endif static void print_wasmer_error() { int error_len = wasmer_last_error_length(); @@ -93,18 +101,54 @@ int main(int argc, char *argv[]) { wasm_engine_t *engine = wasm_engine_new_with_config(config); wasm_store_t *store = wasm_store_new(engine); - wasm_module_t *module = wasmer_object_module_new(store, "module"); +#ifdef WASI_PIRITA + // INSTANTIATE_MODULES +#else + wasm_byte_vec_t module_byte_vec = { + .size = WASMER_MODULE_LENGTH, + .data = &WASMER_MODULE_DATA, + }; + wasm_module_t *module = wasm_module_deserialize(store, &module_byte_vec); if (!module) { fprintf(stderr, "Failed to create module\n"); print_wasmer_error(); return -1; } +#endif // We have now finished the memory buffer book keeping and we have a valid // Module. -#ifdef WASI +#ifdef WASI_PIRITA + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasm_byte_vec_t volume_bytes = { + .size = VOLUMES_LENGTH, + .data = &VOLUMES_DATA, + }; + + wasi_filesystem_t* filesystem = wasi_filesystem_init_static_memory(&volume_bytes); + if (!filesystem) { + printf("Error parsing filesystem from bytes\n"); + return 1; + } + + wasm_extern_vec_t imports; + wasi_env_t* wasi_env = wasi_env_with_filesystem( + wasi_config, + store, + module, + filesystem, + &imports, + "##atom-name##" + ); + if (!wasi_env) { + printf("Error setting filesystem\n"); + return 1; + } +#else wasi_config_t *wasi_config = wasi_config_new(argv[0]); handle_arguments(wasi_config, argc, argv); @@ -114,7 +158,6 @@ int main(int argc, char *argv[]) { print_wasmer_error(); return 1; } -#endif wasm_importtype_vec_t import_types; wasm_module_imports(module, &import_types); @@ -132,6 +175,7 @@ int main(int argc, char *argv[]) { return 1; } +#endif #endif wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); @@ -179,6 +223,9 @@ int main(int argc, char *argv[]) { // TODO: handle non-WASI start (maybe with invoke?) +#ifdef WASI_PIRITA + wasi_filesystem_delete(filesystem); +#endif #ifdef WASI wasi_env_delete(wasi_env); wasm_extern_vec_delete(&exports); diff --git a/lib/cli/src/commands/wasmer_deserialize_module.h b/lib/cli/src/commands/wasmer_deserialize_module.h index f0ef229aa43..98705d4fc6f 100644 --- a/lib/cli/src/commands/wasmer_deserialize_module.h +++ b/lib/cli/src/commands/wasmer_deserialize_module.h @@ -10,7 +10,7 @@ extern "C" { extern size_t WASMER_MODULE_LENGTH asm("WASMER_MODULE_LENGTH"); extern char WASMER_MODULE_DATA asm("WASMER_MODULE_DATA"); -wasm_module_t* wasmer_object_module_new(wasm_store_t* store, const char* module_name) { +wasm_module_t* wasmer_module_new(wasm_store_t* store, const char* wasm_name) { wasm_byte_vec_t module_byte_vec = { .size = WASMER_MODULE_LENGTH, .data = (const char*)&WASMER_MODULE_DATA, diff --git a/lib/cli/src/commands/wasmer_static_create_exe_main.c b/lib/cli/src/commands/wasmer_static_create_exe_main.c new file mode 100644 index 00000000000..67507005e1f --- /dev/null +++ b/lib/cli/src/commands/wasmer_static_create_exe_main.c @@ -0,0 +1,232 @@ +#include "wasmer.h" +#include "static_defs.h" +#include +#include +#include + +#define own + +// TODO: make this define templated so that the Rust code can toggle it on/off +#define WASI + +#ifdef WASI_PIRITA +extern size_t VOLUMES_LENGTH asm("VOLUMES_LENGTH"); +extern char VOLUMES_DATA asm("VOLUMES_DATA"); +#endif + +extern wasm_module_t* wasmer_module_new(wasm_store_t* store) asm("wasmer_module_new"); +extern wasm_module_t* wasmer_static_module_new(wasm_store_t* store,const char* wasm_name) asm("wasmer_static_module_new"); + + +static void print_wasmer_error() { + int error_len = wasmer_last_error_length(); + printf("Error len: `%d`\n", error_len); + char *error_str = (char *)malloc(error_len); + wasmer_last_error_message(error_str, error_len); + printf("%s\n", error_str); + free(error_str); +} + +#ifdef WASI +static void pass_mapdir_arg(wasi_config_t *wasi_config, char *mapdir) { + int colon_location = strchr(mapdir, ':') - mapdir; + if (colon_location == 0) { + // error malformed argument + fprintf(stderr, "Expected mapdir argument of the form alias:directory\n"); + exit(-1); + } + + char *alias = (char *)malloc(colon_location + 1); + memcpy(alias, mapdir, colon_location); + alias[colon_location] = '\0'; + + int dir_len = strlen(mapdir) - colon_location; + char *dir = (char *)malloc(dir_len + 1); + memcpy(dir, &mapdir[colon_location + 1], dir_len); + dir[dir_len] = '\0'; + + wasi_config_mapdir(wasi_config, alias, dir); + free(alias); + free(dir); +} + +// We try to parse out `--dir` and `--mapdir` ahead of time and process those +// specially. All other arguments are passed to the guest program. +static void handle_arguments(wasi_config_t *wasi_config, int argc, + char *argv[]) { + for (int i = 1; i < argc; ++i) { + // We probably want special args like `--dir` and `--mapdir` to not be + // passed directly + if (strcmp(argv[i], "--dir") == 0) { + // next arg is a preopen directory + if ((i + 1) < argc) { + i++; + wasi_config_preopen_dir(wasi_config, argv[i]); + } else { + fprintf(stderr, "--dir expects a following argument specifying which " + "directory to preopen\n"); + exit(-1); + } + } else if (strcmp(argv[i], "--mapdir") == 0) { + // next arg is a mapdir + if ((i + 1) < argc) { + i++; + pass_mapdir_arg(wasi_config, argv[i]); + } else { + fprintf(stderr, + "--mapdir expects a following argument specifying which " + "directory to preopen in the form alias:directory\n"); + exit(-1); + } + } else if (strncmp(argv[i], "--dir=", strlen("--dir=")) == 0) { + // this arg is a preopen dir + char *dir = argv[i] + strlen("--dir="); + wasi_config_preopen_dir(wasi_config, dir); + } else if (strncmp(argv[i], "--mapdir=", strlen("--mapdir=")) == 0) { + // this arg is a mapdir + char *mapdir = argv[i] + strlen("--mapdir="); + pass_mapdir_arg(wasi_config, mapdir); + } else { + // guest argument + wasi_config_arg(wasi_config, argv[i]); + } + } +} +#endif + +int main(int argc, char *argv[]) { + wasm_config_t *config = wasm_config_new(); + wasm_engine_t *engine = wasm_engine_new_with_config(config); + wasm_store_t *store = wasm_store_new(engine); + + #ifdef WASI_PIRITA + // INSTANTIATE_MODULES + #else + wasm_module_t *module = wasmer_static_module_new(store, "module"); + #endif + + if (!module) { + fprintf(stderr, "Failed to create module\n"); + print_wasmer_error(); + return -1; + } + + // We have now finished the memory buffer book keeping and we have a valid + // Module. + +#ifdef WASI_PIRITA + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasm_byte_vec_t volume_bytes = { + .size = VOLUMES_LENGTH, + .data = &VOLUMES_DATA, + }; + + wasi_filesystem_t* filesystem = wasi_filesystem_init_static_memory(&volume_bytes); + if (!filesystem) { + printf("Error parsing filesystem from bytes\n"); + return 1; + } + + wasm_extern_vec_t imports; + wasi_env_t* wasi_env = wasi_env_with_filesystem( + wasi_config, + store, + module, + filesystem, + &imports, + "##atom-name##" + ); + if (!wasi_env) { + printf("Error setting filesystem\n"); + return 1; + } +#else + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasi_env_t *wasi_env = wasi_env_new(store, wasi_config); + if (!wasi_env) { + fprintf(stderr, "Error building WASI env!\n"); + print_wasmer_error(); + return 1; + } + + wasm_importtype_vec_t import_types; + wasm_module_imports(module, &import_types); + + wasm_extern_vec_t imports; + wasm_extern_vec_new_uninitialized(&imports, import_types.size); + wasm_importtype_vec_delete(&import_types); + +#ifdef WASI + bool get_imports_result = wasi_get_imports(store, wasi_env, module, &imports); + + if (!get_imports_result) { + fprintf(stderr, "Error getting WASI imports!\n"); + print_wasmer_error(); + + return 1; + } +#endif +#endif + + wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); + + if (!instance) { + fprintf(stderr, "Failed to create instance\n"); + print_wasmer_error(); + return -1; + } + +#ifdef WASI + // Read the exports. + wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + wasm_memory_t* mem = NULL; + for (size_t i = 0; i < exports.size; i++) { + mem = wasm_extern_as_memory(exports.data[i]); + if (mem) { + break; + } + } + + if (!mem) { + fprintf(stderr, "Failed to create instance: Could not find memory in exports\n"); + print_wasmer_error(); + return -1; + } + wasi_env_set_memory(wasi_env, mem); + + own wasm_func_t *start_function = wasi_get_start_function(instance); + if (!start_function) { + fprintf(stderr, "`_start` function not found\n"); + print_wasmer_error(); + return -1; + } + + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_EMPTY_VEC; + own wasm_trap_t *trap = wasm_func_call(start_function, &args, &results); + if (trap) { + fprintf(stderr, "Trap is not NULL: TODO:\n"); + return -1; + } +#endif + + // TODO: handle non-WASI start (maybe with invoke?) + +#ifdef WASI_PIRITA + wasi_filesystem_delete(filesystem); +#endif +#ifdef WASI + wasi_env_delete(wasi_env); + wasm_extern_vec_delete(&exports); +#endif + wasm_instance_delete(instance); + wasm_module_delete(module); + wasm_store_delete(store); + wasm_engine_delete(engine); + return 0; +} diff --git a/lib/object/src/module.rs b/lib/object/src/module.rs index 40cb005af2c..34caee5db4d 100644 --- a/lib/object/src/module.rs +++ b/lib/object/src/module.rs @@ -401,7 +401,7 @@ pub fn emit_compilation( /// # symbol_registry: impl SymbolRegistry, /// # ) -> Result<(), ObjectError> { /// let mut object = get_object_for_target(&triple)?; -/// emit_compilation(&mut object, compilation, &symbol_registry, &triple)?; +/// emit_serialized(&mut object, bytes, &triple, "WASMER_MODULE")?; /// # Ok(()) /// # } /// ``` @@ -409,11 +409,12 @@ pub fn emit_serialized( obj: &mut Object, sercomp: &[u8], triple: &Triple, + object_name: &str, ) -> Result<(), ObjectError> { obj.set_mangling(object::write::Mangling::None); //let module_name = module.compile_info.module.name.clone(); - let len_name = "WASMER_MODULE_LENGTH"; - let data_name = "WASMER_MODULE_DATA"; + let len_name = format!("{}_LENGTH", object_name); + let data_name = format!("{}_DATA", object_name); //let metadata_name = "WASMER_MODULE_METADATA"; let align = match triple.architecture {