diff --git a/Cargo.lock b/Cargo.lock index af1a36a1a31..a003295db11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -151,9 +151,9 @@ checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e" [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95" [[package]] name = "arbitrary" @@ -211,7 +211,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -280,7 +280,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -377,7 +377,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -499,7 +499,7 @@ checksum = "523363cbe1df49b68215efdf500b103ac3b0fb4836aed6d15689a076eadb8fff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -520,9 +520,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" dependencies = [ "serde", ] @@ -608,7 +608,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.79", + "syn 2.0.82", "tempfile", "toml 0.8.19", ] @@ -753,7 +753,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1300,7 +1300,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.79", + "syn 2.0.82", "thiserror", ] @@ -1324,7 +1324,7 @@ dependencies = [ "cynic-codegen", "darling 0.20.10", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1372,7 +1372,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1394,7 +1394,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core 0.20.10", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1465,7 +1465,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1509,7 +1509,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1597,7 +1597,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1761,7 +1761,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -1782,7 +1782,7 @@ dependencies = [ "darling 0.20.10", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2027,7 +2027,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2100,7 +2100,7 @@ checksum = "b0e085ded9f1267c32176b40921b9754c474f7dd96f7e808d4a982e48aa1e854" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2394,6 +2394,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hexdump" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf31ab66ed8145a1c7427bd8e9b42a6131bd74ccf444f69b9e620c2e73ded832" +dependencies = [ + "arrayvec 0.5.2", +] + [[package]] name = "hmac" version = "0.12.1" @@ -2409,7 +2418,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "fnv", "itoa", ] @@ -2420,7 +2429,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "fnv", "itoa", ] @@ -2431,7 +2440,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "http 1.1.0", ] @@ -2441,7 +2450,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "http 1.1.0", "http-body", @@ -2492,7 +2501,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-channel", "futures-util", "http 1.1.0", @@ -2530,7 +2539,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "http-body-util", "hyper", "hyper-util", @@ -2561,7 +2570,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-channel", "futures-util", "http 1.1.0", @@ -2709,7 +2718,7 @@ checksum = "9dd28cfd4cfba665d47d31c08a6ba637eed16770abca2eccbbc3ca831fef1e44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -2758,6 +2767,7 @@ dependencies = [ "console", "lazy_static", "linked-hash-map", + "regex", "serde", "similar", ] @@ -2957,7 +2967,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if 1.0.0", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -3090,7 +3100,7 @@ dependencies = [ "proc-macro2", "quote", "regex-syntax 0.8.5", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3188,7 +3198,7 @@ version = "5.0.0-rc.1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3431,7 +3441,7 @@ checksum = "1bb5c1d8184f13f7d0ccbeeca0def2f9a181bce2624302793005f5ca8aa62e5e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3693,7 +3703,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3736,7 +3746,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3859,7 +3869,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -3900,7 +3910,7 @@ checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4009,7 +4019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4063,7 +4073,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", "version_check", "yansi", ] @@ -4105,7 +4115,7 @@ checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4131,7 +4141,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -4149,7 +4159,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "rand", "ring", "rustc-hash 2.0.0", @@ -4284,7 +4294,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4385,7 +4395,7 @@ checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "async-compression", "base64", - "bytes 1.7.2", + "bytes 1.8.0", "futures-channel", "futures-core", "futures-util", @@ -4450,7 +4460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "395027076c569819ea6035ee62e664f5e03d74e281744f55261dd1afd939212b" dependencies = [ "bytecheck 0.8.0", - "bytes 1.7.2", + "bytes 1.8.0", "hashbrown 0.14.5", "indexmap 2.6.0", "munge", @@ -4470,7 +4480,7 @@ checksum = "09cb82b74b4810f07e460852c32f522e979787691b0b7b7439fe473e49d49b2f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4790,7 +4800,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4828,7 +4838,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4921,9 +4931,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.211" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "1ac55e59090389fb9f0dd9e0f3c09615afed1d19094284d0b200441f13550793" dependencies = [ "serde_derive", ] @@ -4960,13 +4970,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.211" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "54be4f245ce16bc58d57ef2716271d0d4519e0f6defa147f6e081005bcb278ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -4977,14 +4987,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -5070,7 +5080,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -5110,7 +5120,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "memmap2 0.6.2", ] @@ -5284,7 +5294,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -5306,9 +5316,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" dependencies = [ "proc-macro2", "quote", @@ -5431,7 +5441,7 @@ checksum = "5999e24eaa32083191ba4e425deb75cdf25efefabe5aaccb7446dd0d4122a3f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -5473,7 +5483,7 @@ checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -5604,7 +5614,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", - "bytes 1.7.2", + "bytes 1.8.0", "libc", "mio 1.0.2", "pin-project-lite", @@ -5694,7 +5704,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -5755,7 +5765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" dependencies = [ "bincode", - "bytes 1.7.2", + "bytes 1.8.0", "educe", "futures-core", "futures-sink", @@ -5899,7 +5909,7 @@ version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ - "bytes 1.7.2", + "bytes 1.8.0", "futures-core", "futures-sink", "pin-project-lite", @@ -5984,7 +5994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "bitflags 2.6.0", - "bytes 1.7.2", + "bytes 1.8.0", "futures-util", "http 1.1.0", "http-body", @@ -6028,7 +6038,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6101,7 +6111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04659ddb06c87d233c566112c1c9c5b9e98256d9af50ec3bc9c8327f873a7568" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6155,7 +6165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", - "bytes 1.7.2", + "bytes 1.8.0", "data-encoding", "http 1.1.0", "httparse", @@ -6317,7 +6327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5c400339a9d1d17be34257d0b407e91d64af335e5b4fa49f4bf28467fc8d635" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6327,7 +6337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a02e67ac9634b10da9e4aa63a29a7920b8f1395eafef1ea659b2dd76dda96906" dependencies = [ "anyhow", - "bytes 1.7.2", + "bytes 1.8.0", "camino", "log 0.4.22", "once_cell", @@ -6348,7 +6358,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.79", + "syn 2.0.82", "toml 0.5.11", "uniffi_meta", ] @@ -6360,7 +6370,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583bab49f2bdf5681f9732f8b67a7e555ad920dbb5427be21450217bf1818189" dependencies = [ "anyhow", - "bytes 1.7.2", + "bytes 1.8.0", "siphasher", "uniffi_checksum_derive", ] @@ -6498,7 +6508,7 @@ version = "0.18.0" dependencies = [ "anyhow", "async-trait", - "bytes 1.7.2", + "bytes 1.8.0", "dashmap 6.1.0", "derivative", "dunce", @@ -6521,6 +6531,7 @@ dependencies = [ "tracing", "tracing-test", "typetag", + "wasmer-package", "webc", ] @@ -6529,7 +6540,7 @@ name = "virtual-mio" version = "0.5.0" dependencies = [ "async-trait", - "bytes 1.7.2", + "bytes 1.8.0", "derivative", "futures 0.3.31", "mio 1.0.2", @@ -6548,7 +6559,7 @@ dependencies = [ "base64", "bincode", "bytecheck 0.6.12", - "bytes 1.7.2", + "bytes 1.8.0", "derivative", "futures-util", "hyper", @@ -6767,7 +6778,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", "wasm-bindgen-shared", ] @@ -6801,7 +6812,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6835,7 +6846,7 @@ checksum = "c97b2ef2c8d627381e51c071c2ab328eac606d3f69dd82bcbca20a9e389d95f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -6923,7 +6934,7 @@ version = "5.0.0-rc.1" dependencies = [ "anyhow", "bindgen", - "bytes 1.7.2", + "bytes 1.8.0", "cfg-if 1.0.0", "cmake", "derivative", @@ -6983,7 +6994,8 @@ dependencies = [ "tracing", "url", "uuid", - "wasmer-config 0.9.0", + "wasmer-config", + "wasmer-package", "webc", ] @@ -7104,7 +7116,7 @@ dependencies = [ "anyhow", "assert_cmd 2.0.16", "async-trait", - "bytes 1.7.2", + "bytes 1.8.0", "bytesize", "cargo_metadata", "cfg-if 1.0.0", @@ -7182,7 +7194,7 @@ dependencies = [ "wasmer-compiler-cranelift", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", - "wasmer-config 0.9.0", + "wasmer-config", "wasmer-emscripten", "wasmer-object", "wasmer-package", @@ -7200,7 +7212,7 @@ name = "wasmer-compiler" version = "5.0.0-rc.1" dependencies = [ "backtrace", - "bytes 1.7.2", + "bytes 1.8.0", "cfg-if 1.0.0", "enum-iterator", "enumset", @@ -7309,28 +7321,6 @@ dependencies = [ "wasmer-types", ] -[[package]] -name = "wasmer-config" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644b7e3547bd7e796d92220f60bf57734914254c6cee56607e80177a3e8a28da" -dependencies = [ - "anyhow", - "bytesize", - "ciborium", - "derive_builder", - "hex", - "indexmap 2.6.0", - "schemars", - "semver 1.0.23", - "serde", - "serde_json", - "serde_yaml 0.9.34+deprecated", - "thiserror", - "toml 0.8.19", - "url", -] - [[package]] name = "wasmer-config" version = "0.9.0" @@ -7449,7 +7439,7 @@ dependencies = [ "base64", "bincode", "bytecheck 0.6.12", - "bytes 1.7.2", + "bytes 1.8.0", "derivative", "lz4_flex", "num_enum", @@ -7489,10 +7479,27 @@ dependencies = [ name = "wasmer-package" version = "0.1.0" dependencies = [ + "anyhow", + "bytes 1.8.0", + "cfg-if 1.0.0", + "ciborium", + "flate2", + "hexdump", + "insta", "pretty_assertions", + "regex", + "semver 1.0.23", + "serde", + "serde_json", + "sha2", + "shared-buffer", + "tar", "tempfile", + "thiserror", "toml 0.8.19", - "wasmer-config 0.9.0", + "ureq", + "url", + "wasmer-config", "webc", ] @@ -7538,7 +7545,7 @@ dependencies = [ "toml 0.5.11", "tracing", "url", - "wasmer-config 0.9.0", + "wasmer-config", "wasmer-wasm-interface", "wasmparser 0.216.0", "webc", @@ -7634,7 +7641,7 @@ dependencies = [ "bincode", "blake3", "bytecheck 0.6.12", - "bytes 1.7.2", + "bytes 1.8.0", "cfg-if 1.0.0", "chrono", "cooked-waker", @@ -7695,9 +7702,10 @@ dependencies = [ "wasm-bindgen-futures", "wasm-bindgen-test", "wasmer", - "wasmer-config 0.9.0", + "wasmer-config", "wasmer-emscripten", "wasmer-journal", + "wasmer-package", "wasmer-types", "wasmer-wasix-types", "wcgi 0.2.0", @@ -8009,17 +8017,16 @@ dependencies = [ [[package]] name = "webc" -version = "6.1.0" +version = "7.0.0-rc.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdea84cf234555864ca9b7a5084c1a99dbdf2d148035f62a09b19ce5606532c1" +checksum = "56c452dd6074574080dea17d228caa5efc24f1e29650895c61333b0900734c20" dependencies = [ "anyhow", "base64", - "bytes 1.7.2", + "bytes 1.8.0", "cfg-if 1.0.0", "ciborium", "document-features", - "flate2", "ignore", "indexmap 1.9.3", "leb128", @@ -8028,17 +8035,12 @@ dependencies = [ "once_cell", "path-clean", "rand", - "semver 1.0.23", "serde", "serde_json", "sha2", "shared-buffer", - "tar", - "tempfile", "thiserror", - "toml 0.8.19", "url", - "wasmer-config 0.8.0", ] [[package]] @@ -8402,7 +8404,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] @@ -8422,7 +8424,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.82", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 83c9ec1cece..5fa441231af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,11 +93,12 @@ version = "5.0.0-rc.1" [workspace.dependencies] # Repo-local crates +wasmer-package = { version = "0.1.0", path = "lib/package" } wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.1.0", default-features = false, features = ["package"] } +webc = { version = "7.0.0-rc.1" } shared-buffer = "0.1.4" # Third-party crates diff --git a/lib/backend-api/Cargo.toml b/lib/backend-api/Cargo.toml index a47af3ae1ef..6c5888a746c 100644 --- a/lib/backend-api/Cargo.toml +++ b/lib/backend-api/Cargo.toml @@ -16,6 +16,7 @@ rust-version.workspace = true [dependencies] # Wasmer dependencies. wasmer-config = { version = "0.9.0", path = "../config" } +wasmer-package.workspace = true webc.workspace = true diff --git a/lib/backend-api/src/query.rs b/lib/backend-api/src/query.rs index 9032c788b90..67f5761a612 100644 --- a/lib/backend-api/src/query.rs +++ b/lib/backend-api/src/query.rs @@ -8,6 +8,8 @@ use time::OffsetDateTime; use tracing::Instrument; use url::Url; use wasmer_config::package::PackageIdent; +use wasmer_package::utils::from_bytes; +use webc::Container; use crate::{ types::{self, *}, @@ -374,7 +376,7 @@ pub async fn fetch_webc_package( client: &WasmerClient, ident: &PackageIdent, default_registry: &Url, -) -> Result { +) -> Result { let url = match ident { PackageIdent::Named(n) => Url::parse(&format!( "{default_registry}/{}:{}", @@ -398,7 +400,7 @@ pub async fn fetch_webc_package( .bytes() .await?; - webc::compat::Container::from_bytes(data).context("failed to parse webc package") + from_bytes(data).context("failed to parse webc package") } /// Fetch app templates. diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 47541ff02fa..929232561e3 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -119,7 +119,7 @@ wasmer-compiler-cranelift = { version = "=5.0.0-rc.1", path = "../compiler-crane wasmer-compiler-singlepass = { version = "=5.0.0-rc.1", path = "../compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=5.0.0-rc.1", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=5.0.0-rc.1", path = "../emscripten" } -wasmer-package = { version = "=0.1.0", path = "../package" } +wasmer-package.workspace = true wasmer-vm = { version = "=5.0.0-rc.1", path = "../vm", optional = true } wasmer-wasix = { path = "../wasix", version = "=0.29.0", features = [ diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 4058eda65a6..f4151d21dc5 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -14,11 +14,11 @@ use tar::Archive; use wasmer::sys::Artifact; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; +use wasmer_package::utils::from_disk; use wasmer_types::{compilation::symbols::ModuleMetadataSymbolRegistry, ModuleInfo}; -use webc::{ - compat::{Container, Volume as WebcVolume}, - PathSegments, -}; +use webc::Container; +use webc::PathSegments; +use webc::{Metadata, Volume as WebcVolume}; use self::utils::normalize_atom_name; use crate::{ @@ -245,7 +245,7 @@ impl CreateExe { }; std::fs::create_dir_all(&tempdir)?; - let atoms = if let Ok(pirita) = Container::from_disk(&input_path) { + let atoms = if let Ok(pirita) = from_disk(&input_path) { // pirita file compile_pirita_into_directory( &pirita, @@ -516,14 +516,14 @@ fn serialize_volume_to_webc_v1(volume: &WebcVolume) -> Vec { path.push(segment); match meta { - webc::compat::Metadata::Dir { .. } => { + Metadata::Dir { .. } => { files.insert( webc::v1::DirOrFile::Dir(path.to_string().into()), Vec::new(), ); read_dir(volume, path, files); } - webc::compat::Metadata::File { .. } => { + Metadata::File { .. } => { if let Some((contents, _)) = volume.read_file(&*path) { files.insert( webc::v1::DirOrFile::File(path.to_string().into()), diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs index 03d47158e53..444f224c26c 100644 --- a/lib/cli/src/commands/create_obj.rs +++ b/lib/cli/src/commands/create_obj.rs @@ -6,6 +6,7 @@ use std::{env, path::PathBuf}; use anyhow::{Context, Result}; use clap::Parser; use wasmer::*; +use wasmer_package::utils::from_disk; use crate::store::CompilerOptions; @@ -83,7 +84,7 @@ impl CreateObj { println!("Compiler: {}", compiler_type); println!("Target: {}", target.triple()); - let atoms = if let Ok(webc) = webc::compat::Container::from_disk(&input_path) { + let atoms = if let Ok(webc) = from_disk(&input_path) { crate::commands::create_exe::compile_pirita_into_directory( &webc, &output_directory_path, diff --git a/lib/cli/src/commands/gen_c_header.rs b/lib/cli/src/commands/gen_c_header.rs index 142c41a111a..144ffd84f53 100644 --- a/lib/cli/src/commands/gen_c_header.rs +++ b/lib/cli/src/commands/gen_c_header.rs @@ -4,10 +4,13 @@ use anyhow::{Context, Error}; use bytes::Bytes; use clap::Parser; use wasmer_compiler::Artifact; +use wasmer_package::package::WasmerPackageError; +use wasmer_package::utils::from_bytes; use wasmer_types::{ compilation::symbols::ModuleMetadataSymbolRegistry, CpuFeature, MetadataHeader, Triple, }; -use webc::{compat::SharedBytes, Container, DetectError}; +use webc::{compat::SharedBytes, DetectError}; +use webc::{Container, ContainerError}; use crate::store::CompilerOptions; @@ -60,9 +63,11 @@ impl GenCHeader { None => crate::commands::PrefixMapCompilation::hash_for_bytes(&file), }; - let atom = match Container::from_bytes(file.clone()) { + let atom = match from_bytes(file.clone()) { Ok(webc) => self.get_atom(&webc)?, - Err(webc::compat::ContainerError::Detect(DetectError::InvalidMagic { .. })) => { + Err(WasmerPackageError::ContainerError(ContainerError::Detect( + DetectError::InvalidMagic { .. }, + ))) => { // we've probably got a WebAssembly file file.into() } diff --git a/lib/cli/src/commands/package/build.rs b/lib/cli/src/commands/package/build.rs index 28860967e07..3d1e06ac131 100644 --- a/lib/cli/src/commands/package/build.rs +++ b/lib/cli/src/commands/package/build.rs @@ -5,7 +5,7 @@ use dialoguer::console::{style, Emoji}; use indicatif::ProgressBar; use sha2::Digest; use wasmer_config::package::PackageHash; -use webc::wasmer_package::Package; +use wasmer_package::package::Package; use crate::utils::load_package_manifest; @@ -54,12 +54,10 @@ impl PackageBuild { manifest_path.display() ) }; - let pkg = webc::wasmer_package::Package::from_manifest(manifest_path.clone()).context( - format!( - "While parsing the manifest (loaded from {})", - manifest_path.canonicalize()?.display() - ), - )?; + let pkg = Package::from_manifest(manifest_path.clone()).context(format!( + "While parsing the manifest (loaded from {})", + manifest_path.canonicalize()?.display() + ))?; let data = pkg.serialize().context("While validating the package")?; let hash = sha2::Sha256::digest(&data).into(); let pkg_hash = PackageHash::from_sha256_bytes(hash); @@ -181,6 +179,8 @@ impl PackageBuild { #[cfg(test)] mod tests { + use wasmer_package::utils::from_disk; + use super::*; /// Download a package from the dev registry. @@ -215,6 +215,6 @@ description = "hello" cmd.execute().unwrap(); - webc::Container::from_disk(path.join("wasmer-hello-0.1.0.webc")).unwrap(); + from_disk(path.join("wasmer-hello-0.1.0.webc")).unwrap(); } } diff --git a/lib/cli/src/commands/package/common/mod.rs b/lib/cli/src/commands/package/common/mod.rs index 76f3e2c0bf6..9572655a9b7 100644 --- a/lib/cli/src/commands/package/common/mod.rs +++ b/lib/cli/src/commands/package/common/mod.rs @@ -13,7 +13,7 @@ use std::{ }; use wasmer_api::WasmerClient; use wasmer_config::package::{Manifest, NamedPackageIdent, PackageHash}; -use webc::wasmer_package::Package; +use wasmer_package::package::Package; pub mod macros; pub mod wait; diff --git a/lib/cli/src/commands/package/download.rs b/lib/cli/src/commands/package/download.rs index 97cc9650a4a..8c81d9b6cca 100644 --- a/lib/cli/src/commands/package/download.rs +++ b/lib/cli/src/commands/package/download.rs @@ -5,6 +5,7 @@ use dialoguer::console::{style, Emoji}; use indicatif::{ProgressBar, ProgressStyle}; use tempfile::NamedTempFile; use wasmer_config::package::{PackageIdent, PackageSource}; +use wasmer_package::utils::from_disk; use crate::config::WasmerEnv; @@ -243,7 +244,7 @@ impl PackageDownload { step_num += 1; - webc::compat::Container::from_disk(tmpfile.path()) + from_disk(tmpfile.path()) .context("could not parse downloaded file as a package - invalid download?")?; } @@ -302,6 +303,6 @@ mod tests { cmd.execute().unwrap(); - webc::compat::Container::from_disk(out_path).unwrap(); + from_disk(out_path).unwrap(); } } diff --git a/lib/cli/src/commands/package/push.rs b/lib/cli/src/commands/package/push.rs index 7c2628949ea..0c42ce10e27 100644 --- a/lib/cli/src/commands/package/push.rs +++ b/lib/cli/src/commands/package/push.rs @@ -9,7 +9,7 @@ use is_terminal::IsTerminal; use std::path::{Path, PathBuf}; use wasmer_api::WasmerClient; use wasmer_config::package::{Manifest, PackageHash}; -use webc::wasmer_package::Package; +use wasmer_package::package::Package; /// Push a package to the registry. /// diff --git a/lib/cli/src/commands/package/unpack.rs b/lib/cli/src/commands/package/unpack.rs index 1992743aeef..08e579d4bbf 100644 --- a/lib/cli/src/commands/package/unpack.rs +++ b/lib/cli/src/commands/package/unpack.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use anyhow::Context; use dialoguer::console::{style, Emoji}; use indicatif::ProgressBar; +use wasmer_package::utils::from_disk; /// Extract contents of a webc image to a directory. /// @@ -67,7 +68,7 @@ impl PackageUnpack { PACKAGE_EMOJI )); - let pkg = webc::compat::Container::from_disk(&self.package_path).with_context(|| { + let pkg = from_disk(&self.package_path).with_context(|| { format!( "could not open package at '{}'", self.package_path.display() diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index aa1f9d52b71..c4db7821297 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -32,6 +32,7 @@ use wasmer::{ #[cfg(feature = "compiler")] use wasmer_compiler::ArtifactBuild; use wasmer_config::package::PackageSource as PackageSpecifier; +use wasmer_package::utils::from_disk; use wasmer_registry::{wasmer_env::WasmerEnv, Package}; use wasmer_types::ModuleHash; #[cfg(feature = "journal")] @@ -53,7 +54,8 @@ use wasmer_wasix::{ }, Runtime, WasiError, }; -use webc::{metadata::Manifest, Container}; +use webc::metadata::Manifest; +use webc::Container; use crate::{ commands::run::wasi::Wasi, common::HashAlgorithm, error::PrettyError, logging::Output, @@ -761,7 +763,7 @@ impl ExecutableTarget { }) } TargetOnDisk::LocalWebc => { - let container = Container::from_disk(path)?; + let container = from_disk(path)?; pb.set_message("Resolving dependencies"); let inner_runtime = runtime.clone(); diff --git a/lib/package/Cargo.toml b/lib/package/Cargo.toml index c1fab6cdf07..602d8f75a10 100644 --- a/lib/package/Cargo.toml +++ b/lib/package/Cargo.toml @@ -16,7 +16,25 @@ rust-version.workspace = true webc.workspace = true wasmer-config = { version = "0.9.0", path = "../config" } toml = "0.8.0" +bytes = "1.8.0" +sha2 = "0.10.8" +shared-buffer.workspace = true +serde_json = "1.0.132" +anyhow = "1.0.90" +thiserror = "1.0.64" +cfg-if = "1.0.0" +ciborium = "0.2.2" +semver = "1.0.23" +url = "2.5.2" +serde = "1.0.211" +insta = { version = "1", features = ["filters", "yaml"] } +flate2 = "1.0.34" +tar = "0.4.42" +tempfile = "3.12.0" [dev-dependencies] pretty_assertions.workspace = true tempfile = "3.12.0" +regex = "1.11.0" +ureq = "2.10.1" +hexdump = "0.1.2" diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs index 22edafa39c1..a6f0b03618c 100644 --- a/lib/package/src/convert/webc_to_package.rs +++ b/lib/package/src/convert/webc_to_package.rs @@ -2,14 +2,13 @@ use std::path::Path; use wasmer_config::package::ModuleReference; +use webc::Container; + use super::ConversionError; /// Convert a webc image into a directory with a wasmer.toml file that can /// be used for generating a new pacakge. -pub fn webc_to_package_dir( - webc: &webc::Container, - target_dir: &Path, -) -> Result<(), ConversionError> { +pub fn webc_to_package_dir(webc: &Container, target_dir: &Path) -> Result<(), ConversionError> { let mut pkg_manifest = wasmer_config::package::Manifest::new_empty(); let manifest = webc.manifest(); @@ -227,6 +226,8 @@ mod tests { use pretty_assertions::assert_eq; + use crate::{package::Package, utils::from_bytes}; + use super::*; // Build a webc from a pacakge directory, and then restore the directory @@ -277,10 +278,9 @@ main-args = ["/mounted/script.py"] ) .unwrap(); - let pkg = webc::wasmer_package::Package::from_manifest(dir_input.join("wasmer.toml")) - .unwrap(); + let pkg = Package::from_manifest(dir_input.join("wasmer.toml")).unwrap(); let raw = pkg.serialize().unwrap(); - webc::Container::from_bytes(raw).unwrap() + from_bytes(raw).unwrap() }; let dir_output = dir.join("output"); diff --git a/lib/package/src/lib.rs b/lib/package/src/lib.rs index b5b67213dac..2d4ab3b1976 100644 --- a/lib/package/src/lib.rs +++ b/lib/package/src/lib.rs @@ -1 +1,7 @@ +#[macro_use] +#[cfg(test)] +mod macros; + pub mod convert; +pub mod package; +pub mod utils; diff --git a/lib/package/src/macros.rs b/lib/package/src/macros.rs new file mode 100644 index 00000000000..cfdb764c0c6 --- /dev/null +++ b/lib/package/src/macros.rs @@ -0,0 +1,129 @@ +//! Macros and abstractions used during testing. + +use bytes::{Bytes, BytesMut}; + +use webc::Version; + +/// Construct a sequence of bytes using one or more items that implement the +/// [`ToBytes`] trait. +macro_rules! bytes { + ($($item:expr),* $(,)?) => { + { + #[allow(unused_mut)] + let mut buffer: ::bytes::BytesMut = ::bytes::BytesMut::new(); + $( + #[allow(clippy::identity_op)] + $crate::macros::ToBytes::to_bytes(&$item, &mut buffer); + )* + buffer.freeze() + } + }; +} + +/// Write something to a byte buffer. +pub(crate) trait ToBytes { + fn to_bytes(&self, buffer: &mut BytesMut); +} + +impl ToBytes for webc::v3::Tag { + fn to_bytes(&self, buffer: &mut BytesMut) { + self.as_u8().to_bytes(buffer); + } +} + +impl ToBytes for webc::v3::Timestamps { + fn to_bytes(&self, buffer: &mut BytesMut) { + // TODO: This impl is used in the bytes macro. This macro + // is not public, so this unwrap should be fine for now. + // But a better solution will make `ToBytes` fallible. + self.write_to(buffer).unwrap() + } +} + +impl ToBytes for [u8] { + fn to_bytes(&self, buffer: &mut BytesMut) { + buffer.extend_from_slice(self); + } +} + +impl ToBytes for [u8; N] { + fn to_bytes(&self, buffer: &mut BytesMut) { + buffer.extend_from_slice(self); + } +} + +impl ToBytes for str { + fn to_bytes(&self, buffer: &mut BytesMut) { + self.as_bytes().to_bytes(buffer); + } +} + +impl ToBytes for &T { + fn to_bytes(&self, buffer: &mut BytesMut) { + (**self).to_bytes(buffer); + } +} + +impl ToBytes for u8 { + fn to_bytes(&self, buffer: &mut BytesMut) { + [*self].to_bytes(buffer); + } +} + +impl ToBytes for Vec { + fn to_bytes(&self, buffer: &mut BytesMut) { + self.as_slice().to_bytes(buffer); + } +} + +impl ToBytes for Bytes { + fn to_bytes(&self, buffer: &mut BytesMut) { + self.as_ref().to_bytes(buffer); + } +} + +impl ToBytes for Version { + fn to_bytes(&self, buffer: &mut BytesMut) { + self.0.to_bytes(buffer); + } +} + +impl ToBytes for BytesMut { + fn to_bytes(&self, buffer: &mut BytesMut) { + self.as_ref().to_bytes(buffer); + } +} + +macro_rules! impl_to_bytes_le { + ($( $type:ty ),* $(,)?) => { + $( + impl ToBytes for $type { + fn to_bytes(&self, buffer: &mut BytesMut) { + self.to_le_bytes().to_bytes(buffer); + } + } + )* + }; +} + +impl_to_bytes_le!(u16, u32); + +macro_rules! assert_bytes_eq { + ($lhs:expr, $rhs:expr, $msg:literal $( $tokens:tt)*) => {{ + let lhs = &$lhs[..]; + let rhs = &$rhs[..]; + if lhs != rhs { + let lhs: Vec<_> = hexdump::hexdump_iter(lhs) + .map(|line| line.to_string()) + .collect(); + let rhs: Vec<_> = hexdump::hexdump_iter(rhs) + .map(|line| line.to_string()) + .collect::>(); + + pretty_assertions::assert_eq!(lhs.join("\n"), rhs.join("\n"), $msg $($tokens)*); + } + }}; + ($lhs:expr, $rhs:expr) => { + assert_bytes_eq!($lhs, $rhs, "{} != {}", stringify!($lhs), stringify!($rhs)); + }; +} diff --git a/lib/package/src/package/manifest.rs b/lib/package/src/package/manifest.rs new file mode 100644 index 00000000000..b9beb9ed687 --- /dev/null +++ b/lib/package/src/package/manifest.rs @@ -0,0 +1,1400 @@ +use std::{ + collections::{BTreeMap, HashMap}, + path::{Path, PathBuf}, +}; + +use ciborium::Value; +use semver::VersionReq; +use sha2::Digest; +use shared_buffer::{MmapError, OwnedBuffer}; +use url::Url; +#[allow(deprecated)] +use wasmer_config::package::{CommandV1, CommandV2, Manifest as WasmerManifest, Package}; +use webc::{ + indexmap::{self, IndexMap}, + metadata::AtomSignature, + sanitize_path, +}; + +use webc::metadata::{ + annotations::{ + Atom as AtomAnnotation, Emscripten, FileSystemMapping, FileSystemMappings, + VolumeSpecificPath, Wapm, Wasi, + }, + Atom, Binding, Command, Manifest as WebcManifest, UrlOrManifest, WaiBindings, WitBindings, +}; + +use super::{FsVolume, Strictness}; + +const METADATA_VOLUME: &str = FsVolume::METADATA; + +/// Errors that may occur when converting from a [`wasmer_config::package::Manifest`] to +/// a [`crate::metadata::Manifest`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum ManifestError { + /// A dependency specification had a syntax error. + #[error("The dependency, \"{_0}\", isn't in the \"namespace/name\" format")] + InvalidDependency(String), + /// Unable to serialize an annotation. + #[error("Unable to serialize the \"{key}\" annotation")] + SerializeCborAnnotation { + /// Which annotation was being serialized? + key: String, + /// The underlying error. + #[source] + error: ciborium::value::Error, + }, + /// Specified an unknown atom kind. + #[error("Unknown atom kind, \"{_0}\"")] + UnknownAtomKind(String), + /// A module was specified more than once. + #[error("Duplicate module, \"{_0}\"")] + DuplicateModule(String), + /// Unable to read a module's `source`. + #[error("Unable to read the \"{module}\" module's file from \"{}\"", path.display())] + ReadAtomFile { + /// The name of the module. + module: String, + /// The path that was read. + path: PathBuf, + /// The underlying error. + #[source] + error: std::io::Error, + }, + /// A command was specified more than once. + #[error("Duplicate command, \"{_0}\"")] + DuplicateCommand(String), + /// An unknown runner kind was specified. + #[error("Unknown runner kind, \"{_0}\"")] + UnknownRunnerKind(String), + /// An error occurred while merging user-defined annotations in with + /// automatically generated ones. + #[error("Unable to merge in user-defined \"{key}\" annotations for the \"{command}\" command")] + #[non_exhaustive] + MergeAnnotations { + /// The command annotations were being merged for. + command: String, + /// The annotation that was being merged. + key: String, + }, + /// A command uses a non-existent module. + #[error("The \"{command}\" command uses a non-existent module, \"{module}\"")] + InvalidModuleReference { + /// The command name. + command: String, + /// The module name. + module: String, + }, + /// A command references a module from an undeclared dependency. + #[error("The \"{command}\" command references the undeclared dependency \"{dependency}\"")] + UndeclaredCommandDependency { + /// The command name. + command: String, + /// The dependency name. + dependency: String, + }, + /// Unable to deserialize custom annotations from the `wasmer.toml` + /// manifest. + #[error("Unable to deserialize custom annotations from the wasmer.toml manifest")] + WasmerTomlAnnotations { + /// The underlying error. + #[source] + error: Box, + }, + /// The `wasmer.toml` file references a file outside of its base directory. + #[error("\"{}\" is outside of \"{}\"", path.display(), base_dir.display())] + OutsideBaseDirectory { + /// The file that was referenced. + path: PathBuf, + /// The base directory. + base_dir: PathBuf, + }, + /// The manifest references a file that doesn't exist. + #[error("The \"{}\" doesn't exist (base dir: {})", path.display(), base_dir.display())] + MissingFile { + /// The file that was referenced. + path: PathBuf, + /// The base directory. + base_dir: PathBuf, + }, + /// File based commands are not supported for in-memory package creation + #[error("File based commands are not supported for in-memory package creation")] + FileNotSupported, +} + +/// take a `wasmer.toml` manifest and convert it to the `*.webc` equivalent. +pub(crate) fn wasmer_manifest_to_webc( + manifest: &WasmerManifest, + base_dir: &Path, + strictness: Strictness, +) -> Result<(WebcManifest, BTreeMap), ManifestError> { + let use_map = transform_dependencies(&manifest.dependencies)?; + + // Note: We need to clone the [fs] table because the wasmer-toml crate has + // already upgraded to indexmap v2.0, but the webc crate needs to stay at + // 1.9.2 for backwards compatibility reasons. + let fs: IndexMap = manifest.fs.clone().into_iter().collect(); + + let package = + transform_package_annotations(manifest.package.as_ref(), &fs, base_dir, strictness)?; + let (atoms, atom_files) = transform_atoms(manifest, base_dir)?; + let commands = transform_commands(manifest, base_dir)?; + let bindings = transform_bindings(manifest, base_dir)?; + + let manifest = WebcManifest { + origin: None, + use_map, + package, + atoms, + commands, + bindings, + entrypoint: entrypoint(manifest), + }; + + Ok((manifest, atom_files)) +} + +/// take a `wasmer.toml` manifest and convert it to the `*.webc` equivalent. +pub(crate) fn in_memory_wasmer_manifest_to_webc( + manifest: &WasmerManifest, + atoms: &BTreeMap, OwnedBuffer)>, +) -> Result<(WebcManifest, BTreeMap), ManifestError> { + let use_map = transform_dependencies(&manifest.dependencies)?; + + // Note: We need to clone the [fs] table because the wasmer-toml crate has + // already upgraded to indexmap v2.0, but the webc crate needs to stay at + // 1.9.2 for backwards compatibility reasons. + let fs: IndexMap = manifest.fs.clone().into_iter().collect(); + + let package = transform_in_memory_package_annotations(manifest.package.as_ref(), &fs)?; + let (atoms, atom_files) = transform_in_memory_atoms(atoms)?; + let commands = transform_in_memory_commands(manifest)?; + let bindings = transform_in_memory_bindings(manifest)?; + + let manifest = WebcManifest { + origin: None, + use_map, + package, + atoms, + commands, + bindings, + entrypoint: entrypoint(manifest), + }; + + Ok((manifest, atom_files)) +} + +fn transform_package_annotations( + package: Option<&wasmer_config::package::Package>, + fs: &IndexMap, + base_dir: &Path, + strictness: Strictness, +) -> Result, ManifestError> { + transform_package_annotations_shared(package, fs, |package| { + transform_package_meta_to_annotations(package, base_dir, strictness) + }) +} + +fn transform_in_memory_package_annotations( + package: Option<&wasmer_config::package::Package>, + fs: &IndexMap, +) -> Result, ManifestError> { + transform_package_annotations_shared(package, fs, |package| { + transform_in_memory_package_meta_to_annotations(package) + }) +} + +fn transform_package_annotations_shared( + package: Option<&wasmer_config::package::Package>, + fs: &IndexMap, + transform_package_meta_to_annotations: impl Fn(&Package) -> Result, +) -> Result, ManifestError> { + let mut annotations = IndexMap::new(); + + if let Some(wasmer_package) = package { + let wapm = transform_package_meta_to_annotations(wasmer_package)?; + insert_annotation(&mut annotations, Wapm::KEY, wapm)?; + } + + let fs = get_fs_table(fs); + + if !fs.is_empty() { + insert_annotation(&mut annotations, FileSystemMappings::KEY, fs)?; + } + + Ok(annotations) +} + +fn transform_dependencies( + original_dependencies: &HashMap, +) -> Result, ManifestError> { + let mut dependencies = IndexMap::new(); + + for (dep, version) in original_dependencies { + let (namespace, package_name) = extract_dependency_parts(dep) + .ok_or_else(|| ManifestError::InvalidDependency(dep.clone()))?; + + // Note: the wasmer.toml format forces you to go through a registry for + // all dependencies. There's no way to specify a URL-based dependency. + let dependency_specifier = + UrlOrManifest::RegistryDependentUrl(format!("{namespace}/{package_name}@{version}")); + + dependencies.insert(dep.clone(), dependency_specifier); + } + + Ok(dependencies) +} + +fn extract_dependency_parts(dep: &str) -> Option<(&str, &str)> { + let (namespace, package_name) = dep.split_once('/')?; + + fn invalid_char(c: char) -> bool { + !matches!(c, 'a'..='z' | 'A'..='Z' | '_' | '-' | '0'..='9') + } + + if namespace.contains(invalid_char) || package_name.contains(invalid_char) { + None + } else { + Some((namespace, package_name)) + } +} + +type Atoms = BTreeMap; + +fn transform_atoms( + manifest: &WasmerManifest, + base_dir: &Path, +) -> Result<(IndexMap, Atoms), ManifestError> { + let mut atom_entries = BTreeMap::new(); + + for module in &manifest.modules { + let name = &module.name; + let path = base_dir.join(&module.source); + let file = open_file(&path).map_err(|error| ManifestError::ReadAtomFile { + module: name.clone(), + path, + error, + })?; + + atom_entries.insert(name.clone(), (module.kind.clone(), file)); + } + + transform_atoms_shared(&atom_entries) +} + +fn transform_in_memory_atoms( + atoms: &BTreeMap, OwnedBuffer)>, +) -> Result<(IndexMap, Atoms), ManifestError> { + transform_atoms_shared(atoms) +} + +fn transform_atoms_shared( + atoms: &BTreeMap, OwnedBuffer)>, +) -> Result<(IndexMap, Atoms), ManifestError> { + let mut atom_files = BTreeMap::new(); + let mut metadata = IndexMap::new(); + + for (name, (kind, content)) in atoms.iter() { + let atom = Atom { + kind: atom_kind(kind.as_ref().map(|s| s.as_str()))?, + signature: atom_signature(content), + }; + + if metadata.contains_key(name) { + return Err(ManifestError::DuplicateModule(name.clone())); + } + + metadata.insert(name.clone(), atom); + atom_files.insert(name.clone(), content.clone()); + } + + Ok((metadata, atom_files)) +} + +fn atom_signature(atom: &[u8]) -> String { + let hash: [u8; 32] = sha2::Sha256::digest(atom).into(); + AtomSignature::Sha256(hash).to_string() +} + +/// Map the "kind" field in a `[module]` to the corresponding URI. +fn atom_kind(kind: Option<&str>) -> Result { + const WASM_ATOM_KIND: &str = "https://webc.org/kind/wasm"; + const TENSORFLOW_SAVED_MODEL_KIND: &str = "https://webc.org/kind/tensorflow-SavedModel"; + + let url = match kind { + Some("wasm") | None => WASM_ATOM_KIND.parse().expect("Should never fail"), + Some("tensorflow-SavedModel") => TENSORFLOW_SAVED_MODEL_KIND + .parse() + .expect("Should never fail"), + Some(other) => { + if let Ok(url) = Url::parse(other) { + // if it is a valid URL, pass that through as-is + url + } else { + return Err(ManifestError::UnknownAtomKind(other.to_string())); + } + } + }; + + Ok(url) +} + +/// Try to open a file, preferring mmap and falling back to [`std::fs::read()`] +/// if mapping fails. +fn open_file(path: &Path) -> Result { + match OwnedBuffer::mmap(path) { + Ok(b) => return Ok(b), + Err(MmapError::Map(_)) => { + // Unable to mmap the atom file. Falling back to std::fs::read() + } + Err(MmapError::FileOpen { error, .. }) => { + return Err(error); + } + } + + let bytes = std::fs::read(path)?; + + Ok(OwnedBuffer::from_bytes(bytes)) +} + +fn insert_annotation( + annotations: &mut IndexMap, + key: impl Into, + value: impl serde::Serialize, +) -> Result<(), ManifestError> { + let key = key.into(); + + match ciborium::value::Value::serialized(&value) { + Ok(value) => { + annotations.insert(key, value); + Ok(()) + } + Err(error) => Err(ManifestError::SerializeCborAnnotation { key, error }), + } +} + +fn get_fs_table(fs: &IndexMap) -> FileSystemMappings { + if fs.is_empty() { + return FileSystemMappings::default(); + } + + // When wapm-targz-to-pirita creates the final webc all files will be + // merged into one "atom" volume, but we want to map each directory + // separately. + let mut entries = Vec::new(); + for (guest, host) in fs { + let volume_name = host + .to_str() + .expect("failed to convert path to string") + .to_string(); + + let volume_name = sanitize_path(volume_name); + + let mapping = FileSystemMapping { + from: None, + volume_name, + host_path: None, + mount_path: sanitize_path(guest), + }; + entries.push(mapping); + } + + FileSystemMappings(entries) +} + +fn transform_package_meta_to_annotations( + package: &wasmer_config::package::Package, + base_dir: &Path, + strictness: Strictness, +) -> Result { + fn metadata_file( + path: Option<&PathBuf>, + base_dir: &Path, + strictness: Strictness, + ) -> Result, ManifestError> { + let path = match path { + Some(p) => p, + None => return Ok(None), + }; + + let absolute_path = base_dir.join(path); + + // Touch the file to make sure it actually exists + if !absolute_path.exists() { + match strictness.missing_file(path, base_dir) { + Ok(_) => return Ok(None), + Err(e) => { + return Err(e); + } + } + } + + match base_dir.join(path).strip_prefix(base_dir) { + Ok(without_prefix) => Ok(Some(VolumeSpecificPath { + volume: METADATA_VOLUME.to_string(), + path: sanitize_path(without_prefix), + })), + Err(_) => match strictness.outside_base_directory(path, base_dir) { + Ok(_) => Ok(None), + Err(e) => Err(e), + }, + } + } + + transform_package_meta_to_annotations_shared(package, |path| { + metadata_file(path, base_dir, strictness) + }) +} + +fn transform_in_memory_package_meta_to_annotations( + package: &wasmer_config::package::Package, +) -> Result { + transform_package_meta_to_annotations_shared(package, |path| { + Ok(path.map(|readme_file| VolumeSpecificPath { + volume: METADATA_VOLUME.to_string(), + path: sanitize_path(readme_file), + })) + }) +} + +fn transform_package_meta_to_annotations_shared( + package: &wasmer_config::package::Package, + volume_specific_path: impl Fn(Option<&PathBuf>) -> Result, ManifestError>, +) -> Result { + let mut wapm = Wapm::new( + package.name.clone(), + package.version.clone().map(|v| v.to_string()), + package.description.clone(), + ); + + wapm.license = package.license.clone(); + wapm.license_file = volume_specific_path(package.license_file.as_ref())?; + wapm.readme = volume_specific_path(package.readme.as_ref())?; + wapm.repository = package.repository.clone(); + wapm.homepage = package.homepage.clone(); + wapm.private = package.private; + + Ok(wapm) +} + +fn transform_commands( + manifest: &WasmerManifest, + base_dir: &Path, +) -> Result, ManifestError> { + trasform_commands_shared( + manifest, + |cmd| transform_command_v1(cmd, manifest), + |cmd| transform_command_v2(cmd, base_dir), + ) +} + +fn transform_in_memory_commands( + manifest: &WasmerManifest, +) -> Result, ManifestError> { + trasform_commands_shared( + manifest, + |cmd| transform_command_v1(cmd, manifest), + transform_in_memory_command_v2, + ) +} + +#[allow(deprecated)] +fn trasform_commands_shared( + manifest: &WasmerManifest, + transform_command_v1: impl Fn(&CommandV1) -> Result, + transform_command_v2: impl Fn(&CommandV2) -> Result, +) -> Result, ManifestError> { + let mut commands = IndexMap::new(); + + for command in &manifest.commands { + let cmd = match command { + wasmer_config::package::Command::V1(cmd) => transform_command_v1(cmd)?, + wasmer_config::package::Command::V2(cmd) => transform_command_v2(cmd)?, + }; + + // If a command uses a module from a dependency, then ensure that + // the dependency is declared. + match command.get_module() { + wasmer_config::package::ModuleReference::CurrentPackage { .. } => {} + wasmer_config::package::ModuleReference::Dependency { dependency, .. } => { + if !manifest.dependencies.contains_key(dependency) { + return Err(ManifestError::UndeclaredCommandDependency { + command: command.get_name().to_string(), + dependency: dependency.to_string(), + }); + } + } + } + + match commands.entry(command.get_name().to_string()) { + indexmap::map::Entry::Occupied(_) => { + return Err(ManifestError::DuplicateCommand( + command.get_name().to_string(), + )); + } + indexmap::map::Entry::Vacant(entry) => { + entry.insert(cmd); + } + } + } + + Ok(commands) +} + +#[allow(deprecated)] +fn transform_command_v1( + cmd: &wasmer_config::package::CommandV1, + manifest: &WasmerManifest, +) -> Result { + // Note: a key difference between CommandV1 and CommandV2 is that v1 uses + // a module's "abi" field to figure out which runner to use, whereas v2 has + // a dedicated "runner" field and ignores module.abi. + let runner = match &cmd.module { + wasmer_config::package::ModuleReference::CurrentPackage { module } => { + let module = manifest + .modules + .iter() + .find(|m| m.name == module.as_str()) + .ok_or_else(|| ManifestError::InvalidModuleReference { + command: cmd.name.clone(), + module: cmd.module.to_string(), + })?; + + RunnerKind::from_name(module.abi.to_str())? + } + wasmer_config::package::ModuleReference::Dependency { .. } => { + // Note: We don't have any visibility into dependencies (this code + // doesn't do resolution), so we blindly assume it's a WASI command. + // That should be fine because people shouldn't use the CommandV1 + // syntax any more. + RunnerKind::Wasi + } + }; + + let mut annotations = IndexMap::new(); + // Splitting by whitespace isn't really correct, but proper shell splitting + // would require a dependency and CommandV1 won't be used any more, anyway. + let main_args = cmd + .main_args + .as_deref() + .map(|args| args.split_whitespace().map(String::from).collect()); + runner.runner_specific_annotations( + &mut annotations, + &cmd.module, + cmd.package.clone(), + main_args, + )?; + + Ok(Command { + runner: runner.uri().to_string(), + annotations, + }) +} + +fn transform_command_v2( + cmd: &wasmer_config::package::CommandV2, + base_dir: &Path, +) -> Result { + transform_command_v2_shared(cmd, || { + cmd.get_annotations(base_dir) + .map_err(|error| ManifestError::WasmerTomlAnnotations { + error: error.into(), + }) + }) +} + +fn transform_in_memory_command_v2( + cmd: &wasmer_config::package::CommandV2, +) -> Result { + transform_command_v2_shared(cmd, || { + cmd.annotations + .as_ref() + .map(|a| match a { + wasmer_config::package::CommandAnnotations::File(_) => { + Err(ManifestError::FileNotSupported) + } + wasmer_config::package::CommandAnnotations::Raw(v) => Ok(toml_to_cbor_value(v)), + }) + .transpose() + }) +} + +fn transform_command_v2_shared( + cmd: &wasmer_config::package::CommandV2, + custom_annotations: impl Fn() -> Result, ManifestError>, +) -> Result { + let runner = RunnerKind::from_name(&cmd.runner)?; + let mut annotations = IndexMap::new(); + + runner.runner_specific_annotations(&mut annotations, &cmd.module, None, None)?; + + let custom_annotations = custom_annotations()?; + + if let Some(ciborium::Value::Map(custom_annotations)) = custom_annotations { + for (key, value) in custom_annotations { + if let ciborium::Value::Text(key) = key { + match annotations.entry(key) { + indexmap::map::Entry::Occupied(mut entry) => { + merge_cbor(entry.get_mut(), value).map_err(|_| { + ManifestError::MergeAnnotations { + command: cmd.name.clone(), + key: entry.key().clone(), + } + })?; + } + indexmap::map::Entry::Vacant(entry) => { + entry.insert(value); + } + } + } + } + } + + Ok(Command { + runner: runner.uri().to_string(), + annotations, + }) +} + +fn toml_to_cbor_value(val: &toml::value::Value) -> ciborium::Value { + match val { + toml::Value::String(s) => ciborium::Value::Text(s.clone()), + toml::Value::Integer(i) => ciborium::Value::Integer(ciborium::value::Integer::from(*i)), + toml::Value::Float(f) => ciborium::Value::Float(*f), + toml::Value::Boolean(b) => ciborium::Value::Bool(*b), + toml::Value::Datetime(d) => ciborium::Value::Text(format!("{}", d)), + toml::Value::Array(sq) => { + ciborium::Value::Array(sq.iter().map(toml_to_cbor_value).collect()) + } + toml::Value::Table(m) => ciborium::Value::Map( + m.iter() + .map(|(k, v)| (ciborium::Value::Text(k.clone()), toml_to_cbor_value(v))) + .collect(), + ), + } +} + +fn merge_cbor(original: &mut Value, addition: Value) -> Result<(), ()> { + match (original, addition) { + (Value::Map(left), Value::Map(right)) => { + for (k, v) in right { + if let Some(entry) = left.iter_mut().find(|lk| lk.0 == k) { + merge_cbor(&mut entry.1, v)?; + } else { + left.push((k, v)); + } + } + } + (Value::Array(left), Value::Array(right)) => { + left.extend(right); + } + // Primitives that have the same values are fine + (Value::Bool(left), Value::Bool(right)) if *left == right => {} + (Value::Bytes(left), Value::Bytes(right)) if *left == right => {} + (Value::Float(left), Value::Float(right)) if *left == right => {} + (Value::Integer(left), Value::Integer(right)) if *left == right => {} + (Value::Text(left), Value::Text(right)) if *left == right => {} + // null can be overwritten + (original @ Value::Null, value) => { + *original = value; + } + (_original, Value::Null) => {} + // Oh well, we tried... + (_left, _right) => { + return Err(()); + } + } + + Ok(()) +} + +#[derive(Debug, Clone, PartialEq)] +enum RunnerKind { + Wasi, + Wcgi, + Emscripten, + Wasm4, + Other(Url), +} + +impl RunnerKind { + fn from_name(name: &str) -> Result { + match name { + "wasi" | "wasi@unstable_" | webc::metadata::annotations::WASI_RUNNER_URI => { + Ok(RunnerKind::Wasi) + } + "generic" => { + // This is what you get with a CommandV1 and abi = "none" + Ok(RunnerKind::Wasi) + } + "wcgi" | webc::metadata::annotations::WCGI_RUNNER_URI => Ok(RunnerKind::Wcgi), + "emscripten" | webc::metadata::annotations::EMSCRIPTEN_RUNNER_URI => { + Ok(RunnerKind::Emscripten) + } + "wasm4" | webc::metadata::annotations::WASM4_RUNNER_URI => Ok(RunnerKind::Wasm4), + other => { + if let Ok(other) = Url::parse(other) { + Ok(RunnerKind::Other(other)) + } else if let Ok(other) = format!("https://webc.org/runner/{other}").parse() { + // fall back to something under webc.org + Ok(RunnerKind::Other(other)) + } else { + Err(ManifestError::UnknownRunnerKind(other.to_string())) + } + } + } + } + + fn uri(&self) -> &str { + match self { + RunnerKind::Wasi => webc::metadata::annotations::WASI_RUNNER_URI, + RunnerKind::Wcgi => webc::metadata::annotations::WCGI_RUNNER_URI, + RunnerKind::Emscripten => webc::metadata::annotations::EMSCRIPTEN_RUNNER_URI, + RunnerKind::Wasm4 => webc::metadata::annotations::WASM4_RUNNER_URI, + RunnerKind::Other(other) => other.as_str(), + } + } + + #[allow(deprecated)] + fn runner_specific_annotations( + &self, + annotations: &mut IndexMap, + module: &wasmer_config::package::ModuleReference, + package: Option, + main_args: Option>, + ) -> Result<(), ManifestError> { + let atom_annotation = match module { + wasmer_config::package::ModuleReference::CurrentPackage { module } => { + AtomAnnotation::new(module, None) + } + wasmer_config::package::ModuleReference::Dependency { dependency, module } => { + AtomAnnotation::new(module, dependency.to_string()) + } + }; + insert_annotation(annotations, AtomAnnotation::KEY, atom_annotation)?; + + match self { + RunnerKind::Wasi | RunnerKind::Wcgi => { + let mut wasi = Wasi::new(module.to_string()); + wasi.main_args = main_args; + wasi.package = package; + insert_annotation(annotations, Wasi::KEY, wasi)?; + } + RunnerKind::Emscripten => { + let emscripten = Emscripten { + atom: Some(module.to_string()), + package, + env: None, + main_args, + mount_atom_in_volume: None, + }; + insert_annotation(annotations, Emscripten::KEY, emscripten)?; + } + RunnerKind::Wasm4 | RunnerKind::Other(_) => { + // No extra annotations to add + } + } + + Ok(()) + } +} + +/// Infer the package's entrypoint. +fn entrypoint(manifest: &WasmerManifest) -> Option { + // check if manifest.package is none + if let Some(package) = &manifest.package { + if let Some(entrypoint) = &package.entrypoint { + return Some(entrypoint.clone()); + } + } + + if let [only_command] = manifest.commands.as_slice() { + // For convenience (and to stay compatible with old docs), if there is + // only one command we'll use that as the entrypoint + return Some(only_command.get_name().to_string()); + } + + None +} + +fn transform_bindings( + manifest: &WasmerManifest, + base_dir: &Path, +) -> Result, ManifestError> { + transform_bindings_shared( + manifest, + |wit, module| transform_wit_bindings(wit, module, base_dir), + |wit, module| transform_wai_bindings(wit, module, base_dir), + ) +} + +fn transform_in_memory_bindings(manifest: &WasmerManifest) -> Result, ManifestError> { + transform_bindings_shared( + manifest, + transform_in_memory_wit_bindings, + transform_in_memory_wai_bindings, + ) +} + +fn transform_bindings_shared( + manifest: &WasmerManifest, + wit_binding: impl Fn( + &wasmer_config::package::WitBindings, + &wasmer_config::package::Module, + ) -> Result, + wai_binding: impl Fn( + &wasmer_config::package::WaiBindings, + &wasmer_config::package::Module, + ) -> Result, +) -> Result, ManifestError> { + let mut bindings = Vec::new(); + + for module in &manifest.modules { + let b = match &module.bindings { + Some(wasmer_config::package::Bindings::Wit(wit)) => wit_binding(wit, module)?, + Some(wasmer_config::package::Bindings::Wai(wai)) => wai_binding(wai, module)?, + None => continue, + }; + bindings.push(b); + } + + Ok(bindings) +} + +fn transform_wai_bindings( + wai: &wasmer_config::package::WaiBindings, + module: &wasmer_config::package::Module, + base_dir: &Path, +) -> Result { + transform_wai_bindings_shared(wai, module, |path| metadata_volume_uri(path, base_dir)) +} + +fn transform_in_memory_wai_bindings( + wai: &wasmer_config::package::WaiBindings, + module: &wasmer_config::package::Module, +) -> Result { + transform_wai_bindings_shared(wai, module, |path| { + Ok(format!("{METADATA_VOLUME}:/{}", sanitize_path(path))) + }) +} + +fn transform_wai_bindings_shared( + wai: &wasmer_config::package::WaiBindings, + module: &wasmer_config::package::Module, + metadata_volume_path: impl Fn(&PathBuf) -> Result, +) -> Result { + let wasmer_config::package::WaiBindings { + wai_version, + exports, + imports, + } = wai; + + let bindings = WaiBindings { + exports: exports.as_ref().map(&metadata_volume_path).transpose()?, + module: module.name.clone(), + imports: imports + .iter() + .map(metadata_volume_path) + .collect::, ManifestError>>()?, + }; + let mut annotations = IndexMap::new(); + insert_annotation(&mut annotations, "wai", bindings)?; + + Ok(Binding { + name: "library-bindings".to_string(), + kind: format!("wai@{wai_version}"), + annotations: Value::Map( + annotations + .into_iter() + .map(|(k, v)| (Value::Text(k), v)) + .collect(), + ), + }) +} + +fn metadata_volume_uri(path: &Path, base_dir: &Path) -> Result { + make_relative_path(path, base_dir) + .map(sanitize_path) + .map(|p| format!("{METADATA_VOLUME}:/{p}")) +} + +fn transform_wit_bindings( + wit: &wasmer_config::package::WitBindings, + module: &wasmer_config::package::Module, + base_dir: &Path, +) -> Result { + transform_wit_bindings_shared(wit, module, |path| metadata_volume_uri(path, base_dir)) +} + +fn transform_in_memory_wit_bindings( + wit: &wasmer_config::package::WitBindings, + module: &wasmer_config::package::Module, +) -> Result { + transform_wit_bindings_shared(wit, module, |path| { + Ok(format!("{METADATA_VOLUME}:/{}", sanitize_path(path))) + }) +} + +fn transform_wit_bindings_shared( + wit: &wasmer_config::package::WitBindings, + module: &wasmer_config::package::Module, + metadata_volume_path: impl Fn(&PathBuf) -> Result, +) -> Result { + let wasmer_config::package::WitBindings { + wit_bindgen, + wit_exports, + } = wit; + + let bindings = WitBindings { + exports: metadata_volume_path(wit_exports)?, + module: module.name.clone(), + }; + let mut annotations = IndexMap::new(); + insert_annotation(&mut annotations, "wit", bindings)?; + + Ok(Binding { + name: "library-bindings".to_string(), + kind: format!("wit@{wit_bindgen}"), + annotations: Value::Map( + annotations + .into_iter() + .map(|(k, v)| (Value::Text(k), v)) + .collect(), + ), + }) +} + +/// Resolve an item relative to the base directory, returning an error if the +/// file lies outside of it. +fn make_relative_path(path: &Path, base_dir: &Path) -> Result { + let absolute_path = base_dir.join(path); + + match absolute_path.strip_prefix(base_dir) { + Ok(p) => Ok(p.into()), + Err(_) => Err(ManifestError::OutsideBaseDirectory { + path: absolute_path, + base_dir: base_dir.to_path_buf(), + }), + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + use webc::metadata::annotations::Wasi; + + use super::*; + + #[test] + fn custom_annotations_are_copied_across_verbatim() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "test" + version = "0.0.0" + description = "asdf" + + [[module]] + name = "module" + source = "file.wasm" + abi = "wasi" + + [[command]] + name = "command" + module = "module" + runner = "asdf" + annotations = { first = 42, second = ["a", "b"] } + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("file.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + let command = &transformed.commands["command"]; + assert_eq!(command.annotation::("first").unwrap(), Some(42)); + assert_eq!(command.annotation::("non-existent").unwrap(), None); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_empty_manifest() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "My awesome package" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + + let (transformed, atoms) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + assert!(atoms.is_empty()); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_manifest_with_single_atom() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "My awesome package" + + [[module]] + name = "first" + source = "./path/to/file.wasm" + abi = "wasi" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + let dir = temp.path().join("path").join("to"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("file.wasm"), b"\0asm...").unwrap(); + + let (transformed, atoms) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + assert_eq!(atoms.len(), 1); + assert_eq!(atoms["first"].as_slice(), b"\0asm..."); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_manifest_with_atom_and_command() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "My awesome package" + + [[module]] + name = "cpython" + source = "python.wasm" + abi = "wasi" + + [[command]] + name = "python" + module = "cpython" + runner = "wasi" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("python.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + assert_eq!(transformed.commands.len(), 1); + let python = &transformed.commands["python"]; + assert_eq!(&python.runner, webc::metadata::annotations::WASI_RUNNER_URI); + assert_eq!(python.wasi().unwrap().unwrap(), Wasi::new("cpython")); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_manifest_with_multiple_commands() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "My awesome package" + + [[module]] + name = "cpython" + source = "python.wasm" + abi = "wasi" + + [[command]] + name = "first" + module = "cpython" + runner = "wasi" + + [[command]] + name = "second" + module = "cpython" + runner = "wasi" + + [[command]] + name = "third" + module = "cpython" + runner = "wasi" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("python.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + assert_eq!(transformed.commands.len(), 3); + assert!(transformed.commands.contains_key("first")); + assert!(transformed.commands.contains_key("second")); + assert!(transformed.commands.contains_key("third")); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn merge_custom_attributes_with_builtin_ones() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "My awesome package" + + [[module]] + name = "cpython" + source = "python.wasm" + abi = "wasi" + + [[command]] + name = "python" + module = "cpython" + runner = "wasi" + annotations = { wasi = { env = ["KEY=val"]} } + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("python.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + assert_eq!(transformed.commands.len(), 1); + let cmd = &transformed.commands["python"]; + assert_eq!( + &cmd.wasi().unwrap().unwrap(), + Wasi::new("cpython").with_env("KEY", "val") + ); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_bash_manifest() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "sharrattj/bash" + version = "1.0.17" + description = "Bash is a modern POSIX-compliant implementation of /bin/sh." + license = "GNU" + wasmer-extra-flags = "--enable-threads --enable-bulk-memory" + + [dependencies] + "sharrattj/coreutils" = "1.0.16" + + [[module]] + name = "bash" + source = "bash.wasm" + abi = "wasi" + + [[command]] + name = "bash" + module = "bash" + runner = "wasi@unstable_" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("bash.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_wasmer_pack_manifest() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "wasmer/wasmer-pack" + version = "0.7.0" + description = "The WebAssembly interface to wasmer-pack." + license = "MIT" + readme = "README.md" + repository = "https://github.com/wasmerio/wasmer-pack" + homepage = "https://wasmer.io/" + + [[module]] + name = "wasmer-pack-wasm" + source = "wasmer_pack_wasm.wasm" + + [module.bindings] + wai-version = "0.2.0" + exports = "wasmer-pack.exports.wai" + imports = [] + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("wasmer_pack_wasm.wasm"), b"\0asm...").unwrap(); + std::fs::write(temp.path().join("wasmer-pack.exports.wai"), b"").unwrap(); + std::fs::write(temp.path().join("README.md"), b"").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_python_manifest() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "python" + version = "0.1.0" + description = "Python is an interpreted, high-level, general-purpose programming language" + license = "ISC" + repository = "https://github.com/wapm-packages/python" + + [[module]] + name = "python" + source = "bin/python.wasm" + abi = "wasi" + + [module.interfaces] + wasi = "0.0.0-unstable" + + [[command]] + name = "python" + module = "python" + + [fs] + lib = "lib" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + let bin = temp.path().join("bin"); + std::fs::create_dir_all(&bin).unwrap(); + std::fs::write(bin.join("python.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn transform_manifest_with_fs_table() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "This is a package" + + [fs] + lib = "lib" + "/public" = "out" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("python.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + let fs = transformed.filesystem().unwrap().unwrap(); + assert_eq!( + fs, + [ + FileSystemMapping { + from: None, + volume_name: "/lib".to_string(), + host_path: None, + mount_path: "/lib".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/out".to_string(), + host_path: None, + mount_path: "/public".to_string(), + } + ] + ); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } + + #[test] + fn missing_command_dependency() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [[command]] + name = "python" + module = "test/python:python" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + let bin = temp.path().join("bin"); + std::fs::create_dir_all(&bin).unwrap(); + let res = wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict); + + assert!(matches!( + res, + Err(ManifestError::UndeclaredCommandDependency { .. }) + )); + } + + #[test] + fn issue_124_command_runner_is_swallowed() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "wasmer-tests/wcgi-always-panic" + version = "0.1.0" + description = "wasmer-tests/wcgi-always-panic website" + + [[module]] + name = "wcgi-always-panic" + source = "./wcgi-always-panic.wasm" + abi = "wasi" + + [[command]] + name = "wcgi" + module = "wcgi-always-panic" + runner = "https://webc.org/runner/wcgi" + "#; + let manifest: WasmerManifest = toml::from_str(wasmer_toml).unwrap(); + std::fs::write(temp.path().join("wcgi-always-panic.wasm"), b"\0asm...").unwrap(); + + let (transformed, _) = + wasmer_manifest_to_webc(&manifest, temp.path(), Strictness::Strict).unwrap(); + + let cmd = &transformed.commands["wcgi"]; + assert_eq!(cmd.runner, webc::metadata::annotations::WCGI_RUNNER_URI); + assert_eq!(cmd.wasi().unwrap().unwrap(), Wasi::new("wcgi-always-panic")); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(&transformed); } + } + } +} diff --git a/lib/package/src/package/mod.rs b/lib/package/src/package/mod.rs new file mode 100644 index 00000000000..6a9955330a0 --- /dev/null +++ b/lib/package/src/package/mod.rs @@ -0,0 +1,296 @@ +//! Load a Wasmer package from disk. +pub(crate) mod manifest; +#[allow(clippy::module_inception)] +pub(crate) mod package; +pub(crate) mod strictness; +pub(crate) mod volume; + +pub use self::{ + manifest::ManifestError, + package::{Package, WasmerPackageError}, + strictness::Strictness, + volume::{fs::*, in_memory::*, WasmerPackageVolume}, +}; + +#[cfg(test)] +mod tests { + use sha2::Digest; + use shared_buffer::OwnedBuffer; + use tempfile::TempDir; + + use webc::{ + metadata::annotations::FileSystemMapping, + migration::{are_semantically_equivalent, v2_to_v3, v3_to_v2}, + }; + + use crate::{package::Package, utils::from_bytes}; + + #[test] + fn migration_roundtrip() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + [fs] + "/first" = "first" + second = "nested/dir" + "second/child" = "third" + empty = "empty" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + // Now we want to set up the following filesystem tree: + // + // - first/ ("/first") + // - file.txt + // - nested/ + // - dir/ ("second") + // - README.md + // - another-dir/ + // - empty.txt + // - third/ ("second/child") + // - file.txt + // - empty/ ("empty") + // + // The "/first" entry + let first = temp.path().join("first"); + std::fs::create_dir_all(&first).unwrap(); + std::fs::write(first.join("file.txt"), "File").unwrap(); + // The "second" entry + let second = temp.path().join("nested").join("dir"); + std::fs::create_dir_all(&second).unwrap(); + std::fs::write(second.join("README.md"), "please").unwrap(); + let another_dir = temp.path().join("nested").join("dir").join("another-dir"); + std::fs::create_dir_all(&another_dir).unwrap(); + std::fs::write(another_dir.join("empty.txt"), "").unwrap(); + // The "second/child" entry + let third = temp.path().join("third"); + std::fs::create_dir_all(&third).unwrap(); + std::fs::write(third.join("file.txt"), "Hello, World!").unwrap(); + // The "empty" entry + let empty_dir = temp.path().join("empty"); + std::fs::create_dir_all(empty_dir).unwrap(); + + let package = Package::from_manifest(manifest).unwrap(); + + let webc = package.serialize().unwrap(); + + let webc_v2 = v3_to_v2(webc.clone()).unwrap(); + + are_semantically_equivalent(webc_v2.clone(), webc.into()).unwrap(); + + let container = from_bytes(webc_v2.clone().into_bytes()).unwrap(); + let manifest = container.manifest(); + let fs_table = manifest.filesystem().unwrap().unwrap(); + assert_eq!( + fs_table, + [ + FileSystemMapping { + from: None, + volume_name: "atom".to_string(), + host_path: Some("/first".to_string()), + mount_path: "/first".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "atom".to_string(), + host_path: Some("/nested/dir".to_string()), + mount_path: "/second".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "atom".to_string(), + host_path: Some("/third".to_string()), + mount_path: "/second/child".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "atom".to_string(), + host_path: Some("/empty".to_string()), + mount_path: "/empty".to_string(), + }, + ] + ); + + let atom_volume = container.get_volume("atom").unwrap(); + assert_eq!( + atom_volume.read_file("/first/file.txt").unwrap(), + (OwnedBuffer::from(b"File".as_slice()), None) + ); + assert_eq!( + atom_volume.read_file("/nested/dir/README.md").unwrap(), + (OwnedBuffer::from(b"please".as_slice()), None), + ); + assert_eq!( + atom_volume + .read_file("/nested/dir/another-dir/empty.txt") + .unwrap(), + (OwnedBuffer::from(b"".as_slice()), None) + ); + assert_eq!( + atom_volume.read_file("/third/file.txt").unwrap(), + (OwnedBuffer::from(b"Hello, World!".as_slice()), None) + ); + assert_eq!( + atom_volume.read_dir("/empty").unwrap().len(), + 0, + "Directories should be included, even if empty" + ); + + // Go back to v3 + let webc_v3 = v2_to_v3(webc_v2.clone()).unwrap(); + + are_semantically_equivalent(webc_v2, webc_v3.clone()).unwrap(); + + let container = from_bytes(webc_v3.into_bytes()).unwrap(); + let manifest = container.manifest(); + let fs_table = manifest.filesystem().unwrap().unwrap(); + assert_eq!( + fs_table, + [ + FileSystemMapping { + from: None, + volume_name: "/first".to_string(), + host_path: None, + mount_path: "/first".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/nested/dir".to_string(), + host_path: None, + mount_path: "/second".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/third".to_string(), + host_path: None, + mount_path: "/second/child".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/empty".to_string(), + host_path: None, + mount_path: "/empty".to_string(), + }, + ] + ); + + let first_file_hash: [u8; 32] = sha2::Sha256::digest(b"File").into(); + let readme_hash: [u8; 32] = sha2::Sha256::digest(b"please").into(); + let empty_hash: [u8; 32] = sha2::Sha256::digest(b"").into(); + let third_file_hash: [u8; 32] = sha2::Sha256::digest(b"Hello, World!").into(); + + let first_volume = container.get_volume("/first").unwrap(); + assert_eq!( + first_volume.read_file("/file.txt").unwrap(), + (b"File".as_slice().into(), Some(first_file_hash)), + ); + + let nested_dir_volume = container.get_volume("/nested/dir").unwrap(); + assert_eq!( + nested_dir_volume.read_file("README.md").unwrap(), + (b"please".as_slice().into(), Some(readme_hash)), + ); + assert_eq!( + nested_dir_volume + .read_file("/another-dir/empty.txt") + .unwrap(), + (b"".as_slice().into(), Some(empty_hash)) + ); + + let third_volume = container.get_volume("/third").unwrap(); + assert_eq!( + third_volume.read_file("/file.txt").unwrap(), + (b"Hello, World!".as_slice().into(), Some(third_file_hash)) + ); + + let empty_volume = container.get_volume("/empty").unwrap(); + assert_eq!( + empty_volume.read_dir("/").unwrap().len(), + 0, + "Directories should be included, even if empty" + ); + } + + #[test] + fn fs_entry_is_not_required_for_migration() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + let package = Package::from_manifest(manifest).unwrap(); + + let webc = package.serialize().unwrap(); + + let webc_v2 = v3_to_v2(webc).unwrap(); + let container = from_bytes(webc_v2.clone().into_bytes()).unwrap(); + let manifest = container.manifest(); + assert!(manifest.filesystem().unwrap().is_none()); + + // Go back to v3 + let webc_v3 = v2_to_v3(webc_v2).unwrap(); + let container = from_bytes(webc_v3.into_bytes()).unwrap(); + let manifest = container.manifest(); + assert!(manifest.filesystem().unwrap().is_none()); + } + + #[test] + fn container_unpacks_atoms() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + [[module]] + name = "foo" + source = "foo.wasm" + abi = "wasi" + [fs] + "/bar" = "bar" + "#; + + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + + let atom_path = temp.path().join("foo.wasm"); + std::fs::write(&atom_path, b"").unwrap(); + + let bar = temp.path().join("bar"); + std::fs::create_dir(&bar).unwrap(); + + let webc = Package::from_manifest(&manifest) + .unwrap() + .serialize() + .unwrap(); + let container = from_bytes(webc).unwrap(); + + let out_dir = temp.path().join("out"); + container.unpack(&out_dir, false).unwrap(); + + let expected_entries = vec![ + "bar", // the volume + "metadata", // the metadata volume + "foo", // the atom + "manifest.json", + ]; + let entries = std::fs::read_dir(&out_dir) + .unwrap() + .map(|e| e.unwrap()) + .collect::>(); + + assert_eq!(expected_entries.len(), entries.len()); + assert!(expected_entries.iter().all(|e| { + entries + .iter() + .any(|entry| entry.file_name().as_os_str() == *e) + })) + } +} diff --git a/lib/package/src/package/package.rs b/lib/package/src/package/package.rs new file mode 100644 index 00000000000..35a2c064ea6 --- /dev/null +++ b/lib/package/src/package/package.rs @@ -0,0 +1,1941 @@ +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet}, + fmt::Debug, + fs::File, + io::{BufRead, BufReader}, + path::{Path, PathBuf}, + sync::Arc, +}; + +use anyhow::{Context, Error}; +use bytes::Bytes; +use flate2::bufread::GzDecoder; +use shared_buffer::OwnedBuffer; +use tar::Archive; +use tempfile::TempDir; +use wasmer_config::package::Manifest as WasmerManifest; + +use webc::{ + metadata::{annotations::Wapm, Manifest as WebcManifest}, + v3::{ + write::{FileEntry, Writer}, + ChecksumAlgorithm, Timestamps, + }, + AbstractVolume, AbstractWebc, Container, ContainerError, DetectError, PathSegment, Version, + Volume, +}; + +use super::{ + manifest::wasmer_manifest_to_webc, + volume::{fs::FsVolume, WasmerPackageVolume}, + ManifestError, MemoryVolume, Strictness, +}; + +/// Errors that may occur while loading a Wasmer package from disk. +#[derive(Debug, thiserror::Error)] +#[allow(clippy::result_large_err)] +#[non_exhaustive] +pub enum WasmerPackageError { + /// Unable to create a temporary directory. + #[error("Unable to create a temporary directory")] + TempDir(#[source] std::io::Error), + /// Unable to open a file. + #[error("Unable to open \"{}\"", path.display())] + FileOpen { + /// The file being opened. + path: PathBuf, + /// The underlying error. + #[source] + error: std::io::Error, + }, + /// Unable to read a file. + #[error("Unable to read \"{}\"", path.display())] + FileRead { + /// The file being opened. + path: PathBuf, + /// The underlying error. + #[source] + error: std::io::Error, + }, + + /// Generic IO error. + #[error("IO Error: {0:?}")] + IoError(#[from] std::io::Error), + + /// Unexpected path format + #[error("Malformed path format: {0:?}")] + MalformedPath(PathBuf), + + /// Unable to extract the tarball. + #[error("Unable to extract the tarball")] + Tarball(#[source] std::io::Error), + /// Unable to deserialize the `wasmer.toml` file. + #[error("Unable to deserialize \"{}\"", path.display())] + TomlDeserialize { + /// The file being deserialized. + path: PathBuf, + /// The underlying error. + #[source] + error: toml::de::Error, + }, + /// Unable to deserialize a json file. + #[error("Unable to deserialize \"{}\"", path.display())] + JsonDeserialize { + /// The file being deserialized. + path: PathBuf, + /// The underlying error. + #[source] + error: serde_json::Error, + }, + /// Unable to find the `wasmer.toml` file. + #[error("Unable to find the \"wasmer.toml\"")] + MissingManifest, + /// Unable to canonicalize a path. + #[error("Unable to get the absolute path for \"{}\"", path.display())] + Canonicalize { + /// The path being canonicalized. + path: PathBuf, + /// The underlying error. + #[source] + error: std::io::Error, + }, + /// Unable to load the `wasmer.toml` manifest. + #[error("Unable to load the \"wasmer.toml\" manifest")] + Manifest(#[from] ManifestError), + /// A manifest validation error. + #[error("The manifest is invalid")] + Validation(#[from] wasmer_config::package::ValidationError), + /// A path in the fs mapping does not exist + #[error("Path: \"{}\" does not exist", path.display())] + PathNotExists { + /// Path entry in fs mapping + path: PathBuf, + }, + /// Any error happening when populating the volumes tree map of a package + #[error("Volume creation failed: {0:?}")] + VolumeCreation(#[from] anyhow::Error), + + /// Error when serializing or deserializing + #[error("serde error: {0:?}")] + SerdeError(#[from] ciborium::value::Error), + + /// Container Error + #[error("container error: {0:?}")] + ContainerError(#[from] ContainerError), + + /// Detect Error + #[error("detect error: {0:?}")] + DetectError(#[from] DetectError), +} + +/// A Wasmer package that will be lazily loaded from disk. +#[derive(Debug)] +pub struct Package { + // base dir could be a temp dir, so we keep it around to prevent the directory + // from being deleted + #[allow(dead_code)] + base_dir: BaseDir, + manifest: WebcManifest, + atoms: BTreeMap, + strictness: Strictness, + volumes: BTreeMap>, +} + +impl Package { + /// Load a [`Package`] from a `*.tar.gz` file on disk. + /// + /// # Implementation Details + /// + /// This will unpack the tarball to a temporary directory on disk and use + /// memory-mapped files in order to reduce RAM usage. + pub fn from_tarball_file(path: impl AsRef) -> Result { + Package::from_tarball_file_with_strictness(path.as_ref(), Strictness::default()) + } + /// Load a [`Package`] from a `*.tar.gz` file on disk. + /// + /// # Implementation Details + /// + /// This will unpack the tarball to a temporary directory on disk and use + /// memory-mapped files in order to reduce RAM usage. + pub fn from_tarball_file_with_strictness( + path: impl AsRef, + strictness: Strictness, + ) -> Result { + let path = path.as_ref(); + let f = File::open(path).map_err(|error| WasmerPackageError::FileOpen { + path: path.to_path_buf(), + error, + })?; + + Package::from_tarball_with_strictness(BufReader::new(f), strictness) + } + + /// Load a package from a `*.tar.gz` archive. + pub fn from_tarball(tarball: impl BufRead) -> Result { + Package::from_tarball_with_strictness(tarball, Strictness::default()) + } + + /// Load a package from a `*.tar.gz` archive. + pub fn from_tarball_with_strictness( + tarball: impl BufRead, + strictness: Strictness, + ) -> Result { + let tarball = GzDecoder::new(tarball); + let temp = tempdir().map_err(WasmerPackageError::TempDir)?; + let archive = Archive::new(tarball); + unpack_archive(archive, temp.path()).map_err(WasmerPackageError::Tarball)?; + + let (_manifest_path, manifest) = read_manifest(temp.path())?; + + Package::load(manifest, temp, strictness) + } + + /// Load a package from a `wasmer.toml` manifest on disk. + pub fn from_manifest(wasmer_toml: impl AsRef) -> Result { + Package::from_manifest_with_strictness(wasmer_toml, Strictness::default()) + } + + /// Load a package from a `wasmer.toml` manifest on disk. + pub fn from_manifest_with_strictness( + wasmer_toml: impl AsRef, + strictness: Strictness, + ) -> Result { + let path = wasmer_toml.as_ref(); + let path = path + .canonicalize() + .map_err(|error| WasmerPackageError::Canonicalize { + path: path.to_path_buf(), + error, + })?; + + let wasmer_toml = + std::fs::read_to_string(&path).map_err(|error| WasmerPackageError::FileRead { + path: path.to_path_buf(), + error, + })?; + let wasmer_toml: WasmerManifest = + toml::from_str(&wasmer_toml).map_err(|error| WasmerPackageError::TomlDeserialize { + path: path.to_path_buf(), + error, + })?; + + let base_dir = path + .parent() + .expect("Canonicalizing should always result in a file with a parent directory") + .to_path_buf(); + + for path in wasmer_toml.fs.values() { + if !base_dir.join(path).exists() { + return Err(WasmerPackageError::PathNotExists { path: path.clone() }); + } + } + + Package::load(wasmer_toml, base_dir, strictness) + } + + /// (Re)loads a package from a manifest.json file which was created as the result of calling [`Container::unpack`](crate::Container::unpack) + pub fn from_json_manifest(manifest: PathBuf) -> Result { + Self::from_json_manifest_with_strictness(manifest, Strictness::default()) + } + + /// (Re)loads a package from a manifest.json file which was created as the result of calling [`Container::unpack`](crate::Container::unpack) + pub fn from_json_manifest_with_strictness( + manifest: PathBuf, + strictness: Strictness, + ) -> Result { + let base_dir = manifest + .parent() + .expect("Canonicalizing should always result in a file with a parent directory") + .to_path_buf(); + + let base_dir: BaseDir = base_dir.into(); + + let contents = std::fs::read(&manifest)?; + let manifest: WebcManifest = + serde_json::from_slice(&contents).map_err(|e| WasmerPackageError::JsonDeserialize { + path: manifest.clone(), + error: e, + })?; + + let mut atoms = BTreeMap::::new(); + for atom in manifest.atoms.keys() { + let path = base_dir.path().join(atom); + + let contents = std::fs::read(&path) + .map_err(|e| WasmerPackageError::FileRead { path, error: e })?; + + atoms.insert(atom.clone(), contents.into()); + } + + let mut volumes: BTreeMap> = + BTreeMap::new(); + if let Some(fs_mappings) = manifest.filesystem()? { + for entry in fs_mappings.iter() { + let mut dirs = BTreeSet::new(); + let path = entry.volume_name.strip_prefix('/').ok_or_else(|| { + WasmerPackageError::MalformedPath(PathBuf::from(&entry.volume_name)) + })?; + let path = base_dir.path().join(path); + dirs.insert(path); + + volumes.insert( + entry.volume_name.clone(), + Arc::new(FsVolume::new( + entry.volume_name.clone(), + base_dir.path().to_owned(), + BTreeSet::new(), + dirs, + )), + ); + } + } + + let mut files = BTreeSet::new(); + for entry in std::fs::read_dir(base_dir.path().join(FsVolume::METADATA))? { + let entry = entry?; + + files.insert(entry.path()); + } + + if let Some(wapm) = manifest.wapm().unwrap() { + if let Some(license_file) = wapm.license_file.as_ref() { + let path = license_file.path.strip_prefix('/').ok_or_else(|| { + WasmerPackageError::MalformedPath(PathBuf::from(&license_file.path)) + })?; + let path = base_dir.path().join(FsVolume::METADATA).join(path); + + files.insert(path); + } + + if let Some(readme_file) = wapm.readme.as_ref() { + let path = readme_file.path.strip_prefix('/').ok_or_else(|| { + WasmerPackageError::MalformedPath(PathBuf::from(&readme_file.path)) + })?; + let path = base_dir.path().join(FsVolume::METADATA).join(path); + + files.insert(path); + } + } + + volumes.insert( + FsVolume::METADATA.to_string(), + Arc::new(FsVolume::new_with_intermediate_dirs( + FsVolume::METADATA.to_string(), + base_dir.path().join(FsVolume::METADATA).to_owned(), + files, + BTreeSet::new(), + )), + ); + + Ok(Package { + base_dir, + manifest, + atoms, + strictness, + volumes, + }) + } + + /// Create a [`Package`] from an in-memory representation. + pub fn from_in_memory( + manifest: WasmerManifest, + volumes: BTreeMap, + atoms: BTreeMap, OwnedBuffer)>, + metadata: MemoryVolume, + strictness: Strictness, + ) -> Result { + let mut new_volumes = BTreeMap::new(); + + for (k, v) in volumes.into_iter() { + new_volumes.insert(k, Arc::new(v) as _); + } + + new_volumes.insert(MemoryVolume::METADATA.to_string(), Arc::new(metadata) as _); + + let volumes = new_volumes; + + let (mut manifest, atoms) = + super::manifest::in_memory_wasmer_manifest_to_webc(&manifest, &atoms)?; + + if let Some(entry) = manifest.package.get_mut(Wapm::KEY) { + let mut wapm: Wapm = entry.deserialized()?; + + wapm.name.take(); + wapm.version.take(); + wapm.description.take(); + + *entry = ciborium::value::Value::serialized(&wapm)?; + }; + + Ok(Package { + base_dir: BaseDir::Path(Path::new("/").to_path_buf()), + manifest, + atoms, + strictness, + volumes, + }) + } + + fn load( + wasmer_toml: WasmerManifest, + base_dir: impl Into, + strictness: Strictness, + ) -> Result { + let base_dir = base_dir.into(); + + if strictness.is_strict() { + wasmer_toml.validate()?; + } + + let (mut manifest, atoms) = + wasmer_manifest_to_webc(&wasmer_toml, base_dir.path(), strictness)?; + + // remove name, description, and version before creating the webc file + if let Some(entry) = manifest.package.get_mut(Wapm::KEY) { + let mut wapm: Wapm = entry.deserialized()?; + + wapm.name.take(); + wapm.version.take(); + wapm.description.take(); + + *entry = ciborium::value::Value::serialized(&wapm)?; + }; + + // Create volumes + let base_dir_path = base_dir.path().to_path_buf(); + // Create metadata volume + let metadata_volume = FsVolume::new_metadata(&wasmer_toml, base_dir_path.clone())?; + // Create assets volume + let mut volumes: BTreeMap> = { + let old = FsVolume::new_assets(&wasmer_toml, &base_dir_path)?; + let mut new = BTreeMap::new(); + + for (k, v) in old.into_iter() { + new.insert(k, Arc::new(v) as _); + } + + new + }; + volumes.insert( + metadata_volume.name().to_string(), + Arc::new(metadata_volume), + ); + + Ok(Package { + base_dir, + manifest, + atoms, + strictness, + volumes, + }) + } + + /// Returns the Sha256 has of the webc represented by this Package + pub fn webc_hash(&self) -> Option<[u8; 32]> { + None + } + + /// Get the WEBC manifest. + pub fn manifest(&self) -> &WebcManifest { + &self.manifest + } + + /// Get all atoms in this package. + pub fn atoms(&self) -> &BTreeMap { + &self.atoms + } + + /// Returns all volumes in this package + pub fn volumes( + &self, + ) -> impl Iterator> { + self.volumes.values() + } + + /// Serialize the package to a `*.webc` v2 file, ignoring errors due to + /// missing files. + pub fn serialize(&self) -> Result { + let mut w = Writer::new(ChecksumAlgorithm::Sha256) + .write_manifest(self.manifest())? + .write_atoms(self.atom_entries()?)?; + + for (name, volume) in &self.volumes { + w.write_volume(name.as_str(), volume.as_directory_tree(self.strictness)?)?; + } + + let serialized = w.finish(webc::v3::SignatureAlgorithm::None)?; + + Ok(serialized) + } + + fn atom_entries(&self) -> Result>, Error> { + self.atoms() + .iter() + .map(|(key, value)| { + let filename = PathSegment::parse(key) + .with_context(|| format!("\"{key}\" isn't a valid atom name"))?; + // FIXME: maybe? + Ok((filename, FileEntry::borrowed(value, Timestamps::default()))) + }) + .collect() + } + + pub(crate) fn get_volume( + &self, + name: &str, + ) -> Option> { + self.volumes.get(name).cloned() + } + + pub(crate) fn volume_names(&self) -> Vec> { + self.volumes + .keys() + .map(|name| Cow::Borrowed(name.as_str())) + .collect() + } +} + +impl AbstractWebc for Package { + fn version(&self) -> Version { + Version::V3 + } + + fn manifest(&self) -> &WebcManifest { + self.manifest() + } + + fn atom_names(&self) -> Vec> { + self.atoms() + .keys() + .map(|s| Cow::Borrowed(s.as_str())) + .collect() + } + + fn get_atom(&self, name: &str) -> Option { + self.atoms().get(name).cloned() + } + + fn get_webc_hash(&self) -> Option<[u8; 32]> { + self.webc_hash() + } + + fn get_atoms_hash(&self) -> Option<[u8; 32]> { + None + } + + fn volume_names(&self) -> Vec> { + self.volume_names() + } + + fn get_volume(&self, name: &str) -> Option { + self.get_volume(name).map(|v| { + let a: Arc = v.as_volume(); + + Volume::from(a) + }) + } +} + +impl From for Container { + fn from(value: Package) -> Self { + Container::new(value) + } +} + +const IS_WASI: bool = cfg!(all(target_family = "wasm", target_os = "wasi")); + +/// A polyfill for [`TempDir::new()`] that will work when compiling to +/// WASI-based targets. +/// +/// This works around [`std::env::temp_dir()`][tempdir] panicking +/// unconditionally on WASI. +/// +/// [tempdir]: https://github.com/wasix-org/rust/blob/ef19cdcdff77047f1e5ea4d09b4869d6fa456cc7/library/std/src/sys/wasi/os.rs#L228-L230 +fn tempdir() -> Result { + if !IS_WASI { + // The happy path. + return TempDir::new(); + } + + // Note: When compiling to wasm32-wasi, we can't use TempDir::new() + // because std::env::temp_dir() will unconditionally panic. + let temp_dir: PathBuf = std::env::var("TMPDIR") + .unwrap_or_else(|_| "/tmp".to_string()) + .into(); + + if temp_dir.exists() { + TempDir::new_in(temp_dir) + } else { + // The temporary directory doesn't exist. A naive create_dir_all() + // doesn't work when running with "wasmer run" because the root + // directory is immutable, so let's try to use the current exe's + // directory as our tempdir. + // See also: https://github.com/wasmerio/wasmer/blob/482b78890b789f6867a91be9f306385e6255b260/lib/wasix/src/syscalls/wasi/path_create_directory.rs#L30-L32 + if let Ok(current_exe) = std::env::current_exe() { + if let Some(parent) = current_exe.parent() { + if let Ok(temp) = TempDir::new_in(parent) { + return Ok(temp); + } + } + } + + // Oh well, this will probably fail, but at least we tried. + std::fs::create_dir_all(&temp_dir)?; + TempDir::new_in(temp_dir) + } +} + +/// A polyfill for [`Archive::unpack()`] that is WASI-compatible. +/// +/// This works around `canonicalize()` being [unsupported][github] on +/// `wasm32-wasi`. +/// +/// [github]: https://github.com/rust-lang/rust/blob/5b1dc9de77106cb08ce9a1a8deaa14f52751d7e4/library/std/src/sys/wasi/fs.rs#L654-L658 +fn unpack_archive( + mut archive: Archive, + dest: &Path, +) -> Result<(), std::io::Error> { + cfg_if::cfg_if! { + if #[cfg(all(target_family = "wasm", target_os = "wasi"))] + { + // A naive version of unpack() that should be good enough for WASI + // https://github.com/alexcrichton/tar-rs/blob/c77f47cb1b4b47fc4404a170d9d91cb42cc762ea/src/archive.rs#L216-L247 + for entry in archive.entries()? { + let mut entry = entry?; + let item_path = entry.path()?; + let full_path = resolve_archive_path(dest, &item_path); + + match entry.header().entry_type() { + tar::EntryType::Directory => { + std::fs::create_dir_all(&full_path)?; + } + tar::EntryType::Regular => { + if let Some(parent) = full_path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut f = File::create(&full_path)?; + std::io::copy(&mut entry, &mut f)?; + + let mtime = entry.header().mtime().unwrap_or_default(); + if let Err(e) = set_timestamp(full_path.as_path(), mtime) { + println!("WARN: {e:?}"); + } + } + _ => {} + } + } + Ok(()) + + } else { + archive.unpack(dest) + } + } +} + +#[cfg(all(target_family = "wasm", target_os = "wasi"))] +fn set_timestamp(path: &Path, timestamp: u64) -> Result<(), anyhow::Error> { + let fd = unsafe { + libc::open( + path.as_os_str().as_encoded_bytes().as_ptr() as _, + libc::O_RDONLY, + ) + }; + + if fd < 0 { + anyhow::bail!(format!("failed to open: {}", path.display())); + } + + let timespec = [ + // accessed + libc::timespec { + tv_sec: unsafe { libc::time(std::ptr::null_mut()) }, // now + tv_nsec: 0, + }, + // modified + libc::timespec { + tv_sec: timestamp as i64, + tv_nsec: 0, + }, + ]; + + let res = unsafe { libc::futimens(fd, timespec.as_ptr() as _) }; + + if res < 0 { + anyhow::bail!("failed to set timestamp for: {}", path.display()); + } + + Ok(()) +} + +#[cfg(all(target_family = "wasm", target_os = "wasi"))] +fn resolve_archive_path(base_dir: &Path, path: &Path) -> PathBuf { + let mut buffer = base_dir.to_path_buf(); + + for component in path.components() { + match component { + std::path::Component::Prefix(_) + | std::path::Component::RootDir + | std::path::Component::CurDir => continue, + std::path::Component::ParentDir => { + buffer.pop(); + } + std::path::Component::Normal(segment) => { + buffer.push(segment); + } + } + } + + buffer +} + +fn read_manifest(base_dir: &Path) -> Result<(PathBuf, WasmerManifest), WasmerPackageError> { + for path in ["wasmer.toml", "wapm.toml"] { + let path = base_dir.join(path); + + match std::fs::read_to_string(&path) { + Ok(s) => { + let toml_file = toml::from_str(&s).map_err({ + let path = path.clone(); + |error| WasmerPackageError::TomlDeserialize { path, error } + })?; + + return Ok((path, toml_file)); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(WasmerPackageError::FileRead { path, error }); + } + } + } + + Err(WasmerPackageError::MissingManifest) +} + +#[derive(Debug)] +enum BaseDir { + /// An existing directory. + Path(PathBuf), + /// A temporary directory that will be deleted on drop. + Temp(TempDir), +} + +impl BaseDir { + fn path(&self) -> &Path { + match self { + BaseDir::Path(p) => p.as_path(), + BaseDir::Temp(t) => t.path(), + } + } +} + +impl From for BaseDir { + fn from(v: TempDir) -> Self { + Self::Temp(v) + } +} + +impl From for BaseDir { + fn from(v: PathBuf) -> Self { + Self::Path(v) + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + fs::File, + path::{Path, PathBuf}, + str::FromStr, + time::SystemTime, + }; + + use flate2::{write::GzEncoder, Compression}; + use sha2::Digest; + use shared_buffer::OwnedBuffer; + use tempfile::TempDir; + use webc::{ + metadata::{ + annotations::{FileSystemMapping, VolumeSpecificPath}, + Binding, BindingsExtended, WaiBindings, WitBindings, + }, + PathSegment, PathSegments, + }; + + use crate::{package::*, utils::from_bytes}; + + #[test] + fn nonexistent_files() { + let temp = TempDir::new().unwrap(); + + assert!(Package::from_manifest(temp.path().join("nonexistent.toml")).is_err()); + assert!(Package::from_tarball_file(temp.path().join("nonexistent.tar.gz")).is_err()); + } + + #[test] + fn load_a_tarball() { + let coreutils = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("tests") + .join("old-tar-gz") + .join("coreutils-1.0.11.tar.gz"); + assert!(coreutils.exists()); + + let package = Package::from_tarball_file(coreutils).unwrap(); + + let wapm = package.manifest().wapm().unwrap().unwrap(); + assert!(wapm.name.is_none()); + assert!(wapm.version.is_none()); + assert!(wapm.description.is_none()); + } + + #[test] + fn tarball_with_no_manifest() { + let temp = TempDir::new().unwrap(); + let empty_tarball = temp.path().join("empty.tar.gz"); + let mut f = File::create(&empty_tarball).unwrap(); + tar::Builder::new(GzEncoder::new(&mut f, Compression::fast())) + .finish() + .unwrap(); + + assert!(Package::from_tarball_file(&empty_tarball).is_err()); + } + + #[test] + fn empty_package_on_disk() { + let temp = TempDir::new().unwrap(); + let manifest = temp.path().join("wasmer.toml"); + std::fs::write( + &manifest, + r#" + [package] + name = "some/package" + version = "0.0.0" + description = "A dummy package" + "#, + ) + .unwrap(); + + let package = Package::from_manifest(&manifest).unwrap(); + + let wapm = package.manifest().wapm().unwrap().unwrap(); + assert!(wapm.name.is_none()); + assert!(wapm.version.is_none()); + assert!(wapm.description.is_none()); + } + + #[test] + fn load_old_cowsay() { + let tarball = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join("tests") + .join("old-tar-gz") + .join("cowsay-0.3.0.tar.gz"); + + let pkg = Package::from_tarball_file(tarball).unwrap(); + + insta::assert_yaml_snapshot!(pkg.manifest()); + assert_eq!( + pkg.manifest.commands.keys().collect::>(), + ["cowsay", "cowthink"], + ); + } + + #[test] + fn serialize_package_with_non_existent_fs() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + + [fs] + "/first" = "./first" + "#; + let manifest = temp.path().join("wasmer.toml"); + + std::fs::write(&manifest, wasmer_toml).unwrap(); + + let error = Package::from_manifest(manifest).unwrap_err(); + + match error { + WasmerPackageError::PathNotExists { path } => { + assert_eq!(path, PathBuf::from_str("./first").unwrap()); + } + e => panic!("unexpected error: {e:?}"), + } + } + + #[test] + fn serialize_package_with_bundled_directories() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + + [fs] + "/first" = "first" + second = "nested/dir" + "second/child" = "third" + empty = "empty" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + // Now we want to set up the following filesystem tree: + // + // - first/ ("/first") + // - file.txt + // - nested/ + // - dir/ ("second") + // - README.md + // - another-dir/ + // - empty.txt + // - third/ ("second/child") + // - file.txt + // - empty/ ("empty") + // + // The "/first" entry + let first = temp.path().join("first"); + std::fs::create_dir_all(&first).unwrap(); + std::fs::write(first.join("file.txt"), "File").unwrap(); + // The "second" entry + let second = temp.path().join("nested").join("dir"); + std::fs::create_dir_all(&second).unwrap(); + std::fs::write(second.join("README.md"), "please").unwrap(); + let another_dir = temp.path().join("nested").join("dir").join("another-dir"); + std::fs::create_dir_all(&another_dir).unwrap(); + std::fs::write(another_dir.join("empty.txt"), "").unwrap(); + // The "second/child" entry + let third = temp.path().join("third"); + std::fs::create_dir_all(&third).unwrap(); + std::fs::write(third.join("file.txt"), "Hello, World!").unwrap(); + // The "empty" entry + let empty_dir = temp.path().join("empty"); + std::fs::create_dir_all(empty_dir).unwrap(); + + let package = Package::from_manifest(manifest).unwrap(); + + let webc = package.serialize().unwrap(); + let webc = from_bytes(webc).unwrap(); + let manifest = webc.manifest(); + let wapm_metadata = manifest.wapm().unwrap().unwrap(); + assert!(wapm_metadata.name.is_none()); + assert!(wapm_metadata.version.is_none()); + assert!(wapm_metadata.description.is_none()); + let fs_table = manifest.filesystem().unwrap().unwrap(); + assert_eq!( + fs_table, + [ + FileSystemMapping { + from: None, + volume_name: "/first".to_string(), + host_path: None, + mount_path: "/first".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/nested/dir".to_string(), + host_path: None, + mount_path: "/second".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/third".to_string(), + host_path: None, + mount_path: "/second/child".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/empty".to_string(), + host_path: None, + mount_path: "/empty".to_string(), + }, + ] + ); + + let first_file_hash: [u8; 32] = sha2::Sha256::digest(b"File").into(); + let readme_hash: [u8; 32] = sha2::Sha256::digest(b"please").into(); + let empty_hash: [u8; 32] = sha2::Sha256::digest(b"").into(); + let third_file_hash: [u8; 32] = sha2::Sha256::digest(b"Hello, World!").into(); + + let first_volume = webc.get_volume("/first").unwrap(); + assert_eq!( + first_volume.read_file("/file.txt").unwrap(), + (b"File".as_slice().into(), Some(first_file_hash)), + ); + + let nested_dir_volume = webc.get_volume("/nested/dir").unwrap(); + assert_eq!( + nested_dir_volume.read_file("README.md").unwrap(), + (b"please".as_slice().into(), Some(readme_hash)), + ); + assert_eq!( + nested_dir_volume + .read_file("/another-dir/empty.txt") + .unwrap(), + (b"".as_slice().into(), Some(empty_hash)) + ); + + let third_volume = webc.get_volume("/third").unwrap(); + assert_eq!( + third_volume.read_file("/file.txt").unwrap(), + (b"Hello, World!".as_slice().into(), Some(third_file_hash)) + ); + + let empty_volume = webc.get_volume("/empty").unwrap(); + assert_eq!( + empty_volume.read_dir("/").unwrap().len(), + 0, + "Directories should be included, even if empty" + ); + } + + #[test] + fn serialize_package_with_metadata_files() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + readme = "README.md" + license-file = "LICENSE" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("LICENSE"), "license").unwrap(); + + let serialized = Package::from_manifest(manifest) + .unwrap() + .serialize() + .unwrap(); + + let webc = from_bytes(serialized).unwrap(); + let metadata_volume = webc.get_volume("metadata").unwrap(); + + let readme_hash: [u8; 32] = sha2::Sha256::digest(b"readme").into(); + let license_hash: [u8; 32] = sha2::Sha256::digest(b"license").into(); + + assert_eq!( + metadata_volume.read_file("/README.md").unwrap(), + (b"readme".as_slice().into(), Some(readme_hash)) + ); + assert_eq!( + metadata_volume.read_file("/LICENSE").unwrap(), + (b"license".as_slice().into(), Some(license_hash)) + ); + } + + #[test] + fn load_package_with_wit_bindings() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "" + + [[module]] + name = "my-lib" + source = "./my-lib.wasm" + abi = "none" + bindings = { wit-bindgen = "0.1.0", wit-exports = "./file.wit" } + "#; + std::fs::write(temp.path().join("wasmer.toml"), wasmer_toml).unwrap(); + std::fs::write(temp.path().join("file.wit"), "file").unwrap(); + std::fs::write(temp.path().join("my-lib.wasm"), b"\0asm...").unwrap(); + + let package = Package::from_manifest(temp.path().join("wasmer.toml")) + .unwrap() + .serialize() + .unwrap(); + let webc = from_bytes(package).unwrap(); + + assert_eq!( + webc.manifest().bindings, + vec![Binding { + name: "library-bindings".to_string(), + kind: "wit@0.1.0".to_string(), + annotations: ciborium::value::Value::serialized(&BindingsExtended::Wit( + WitBindings { + exports: "metadata://file.wit".to_string(), + module: "my-lib".to_string(), + } + )) + .unwrap(), + }] + ); + let metadata_volume = webc.get_volume("metadata").unwrap(); + let file_hash: [u8; 32] = sha2::Sha256::digest(b"file").into(); + assert_eq!( + metadata_volume.read_file("/file.wit").unwrap(), + (b"file".as_slice().into(), Some(file_hash)) + ); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(webc.manifest()); } + } + } + + #[test] + fn load_package_with_wai_bindings() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "" + + [[module]] + name = "my-lib" + source = "./my-lib.wasm" + abi = "none" + bindings = { wai-version = "0.2.0", exports = "./file.wai", imports = ["a.wai", "b.wai"] } + "#; + std::fs::write(temp.path().join("wasmer.toml"), wasmer_toml).unwrap(); + std::fs::write(temp.path().join("file.wai"), "file").unwrap(); + std::fs::write(temp.path().join("a.wai"), "a").unwrap(); + std::fs::write(temp.path().join("b.wai"), "b").unwrap(); + std::fs::write(temp.path().join("my-lib.wasm"), b"\0asm...").unwrap(); + + let package = Package::from_manifest(temp.path().join("wasmer.toml")) + .unwrap() + .serialize() + .unwrap(); + let webc = from_bytes(package).unwrap(); + + assert_eq!( + webc.manifest().bindings, + vec![Binding { + name: "library-bindings".to_string(), + kind: "wai@0.2.0".to_string(), + annotations: ciborium::value::Value::serialized(&BindingsExtended::Wai( + WaiBindings { + exports: Some("metadata://file.wai".to_string()), + module: "my-lib".to_string(), + imports: vec![ + "metadata://a.wai".to_string(), + "metadata://b.wai".to_string(), + ] + } + )) + .unwrap(), + }] + ); + let metadata_volume = webc.get_volume("metadata").unwrap(); + + let file_hash: [u8; 32] = sha2::Sha256::digest(b"file").into(); + let a_hash: [u8; 32] = sha2::Sha256::digest(b"a").into(); + let b_hash: [u8; 32] = sha2::Sha256::digest(b"b").into(); + + assert_eq!( + metadata_volume.read_file("/file.wai").unwrap(), + (b"file".as_slice().into(), Some(file_hash)) + ); + assert_eq!( + metadata_volume.read_file("/a.wai").unwrap(), + (b"a".as_slice().into(), Some(a_hash)) + ); + assert_eq!( + metadata_volume.read_file("/b.wai").unwrap(), + (b"b".as_slice().into(), Some(b_hash)) + ); + insta::with_settings! { + { description => wasmer_toml }, + { insta::assert_yaml_snapshot!(webc.manifest()); } + } + } + + /// See for more. + #[test] + fn absolute_paths_in_wasmer_toml_issue_105() { + let temp = TempDir::new().unwrap(); + let base_dir = temp.path().canonicalize().unwrap(); + let sep = std::path::MAIN_SEPARATOR; + let wasmer_toml = format!( + r#" + [package] + name = 'some/package' + version = '0.0.0' + description = 'Test package' + readme = '{BASE_DIR}{sep}README.md' + license-file = '{BASE_DIR}{sep}LICENSE' + + [[module]] + name = 'first' + source = '{BASE_DIR}{sep}target{sep}debug{sep}package.wasm' + bindings = {{ wai-version = '0.2.0', exports = '{BASE_DIR}{sep}bindings{sep}file.wai', imports = ['{BASE_DIR}{sep}bindings{sep}a.wai'] }} + "#, + BASE_DIR = base_dir.display(), + ); + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, &wasmer_toml).unwrap(); + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("LICENSE"), "license").unwrap(); + let bindings = temp.path().join("bindings"); + std::fs::create_dir_all(&bindings).unwrap(); + std::fs::write(bindings.join("file.wai"), "file.wai").unwrap(); + std::fs::write(bindings.join("a.wai"), "a.wai").unwrap(); + let target = temp.path().join("target").join("debug"); + std::fs::create_dir_all(&target).unwrap(); + std::fs::write(target.join("package.wasm"), b"\0asm...").unwrap(); + + let serialized = Package::from_manifest(manifest) + .unwrap() + .serialize() + .unwrap(); + + let webc = from_bytes(serialized).unwrap(); + let manifest = webc.manifest(); + let wapm = manifest.wapm().unwrap().unwrap(); + + // we should be able to look up the files using the manifest + let lookup = |item: VolumeSpecificPath| { + let volume = webc.get_volume(&item.volume).unwrap(); + let (contents, _) = volume.read_file(&item.path).unwrap(); + String::from_utf8(contents.into()).unwrap() + }; + assert_eq!(lookup(wapm.license_file.unwrap()), "license"); + assert_eq!(lookup(wapm.readme.unwrap()), "readme"); + + // The paths for bindings are stored slightly differently, but it's the + // same general idea + let lookup = |item: &str| { + let (volume, path) = item.split_once(":/").unwrap(); + let volume = webc.get_volume(volume).unwrap(); + let (content, _) = volume.read_file(path).unwrap(); + String::from_utf8(content.into()).unwrap() + }; + let bindings = manifest.bindings[0].get_wai_bindings().unwrap(); + assert_eq!(lookup(&bindings.imports[0]), "a.wai"); + assert_eq!(lookup(bindings.exports.unwrap().as_str()), "file.wai"); + + // Snapshot tests for good measure + let mut settings = insta::Settings::clone_current(); + let base_dir = base_dir.display().to_string(); + settings.set_description(wasmer_toml.replace(&base_dir, "[BASE_DIR]")); + let filter = regex::escape(&base_dir); + settings.add_filter(&filter, "[BASE_DIR]"); + settings.bind(|| { + insta::assert_yaml_snapshot!(webc.manifest()); + }); + } + + #[test] + fn serializing_will_skip_missing_metadata_by_default() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = 'some/package' + version = '0.0.0' + description = 'Test package' + readme = '/this/does/not/exist/README.md' + license-file = 'LICENSE.wtf' + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + let pkg = Package::from_manifest(manifest).unwrap(); + + let serialized = pkg.serialize().unwrap(); + + let webc = from_bytes(serialized).unwrap(); + let manifest = webc.manifest(); + let wapm = manifest.wapm().unwrap().unwrap(); + // We re-wrote the WAPM annotations to just not include the license file + assert!(wapm.license_file.is_none()); + assert!(wapm.readme.is_none()); + + // Note: serializing in strict mode should still fail + let pkg = Package { + strictness: Strictness::Strict, + ..pkg + }; + assert!(pkg.serialize().is_err()); + } + + #[test] + fn serialize_package_without_local_base_fs_paths() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + readme = 'README.md' + license-file = 'LICENSE' + + [fs] + "/path_in_wasix" = "local-dir/dir1" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("LICENSE"), "license").unwrap(); + + // Now we want to set up the following filesystem tree: + // + // - local-dir/ + // - dir1/ + // - a + // - b + let dir1 = temp.path().join("local-dir").join("dir1"); + std::fs::create_dir_all(&dir1).unwrap(); + + let a = dir1.join("a"); + let b = dir1.join("b"); + + std::fs::write(a, "a").unwrap(); + std::fs::write(b, "b").unwrap(); + + let package = Package::from_manifest(manifest).unwrap(); + + let webc = package.serialize().unwrap(); + let webc = from_bytes(webc).unwrap(); + let manifest = webc.manifest(); + let wapm_metadata = manifest.wapm().unwrap().unwrap(); + + assert!(wapm_metadata.name.is_none()); + assert!(wapm_metadata.version.is_none()); + assert!(wapm_metadata.description.is_none()); + + let fs_table = manifest.filesystem().unwrap().unwrap(); + assert_eq!( + fs_table, + [FileSystemMapping { + from: None, + volume_name: "/local-dir/dir1".to_string(), + host_path: None, + mount_path: "/path_in_wasix".to_string(), + },] + ); + + let readme_hash: [u8; 32] = sha2::Sha256::digest(b"readme").into(); + let license_hash: [u8; 32] = sha2::Sha256::digest(b"license").into(); + + let a_hash: [u8; 32] = sha2::Sha256::digest(b"a").into(); + let b_hash: [u8; 32] = sha2::Sha256::digest(b"b").into(); + + let dir1_volume = webc.get_volume("/local-dir/dir1").unwrap(); + let meta_volume = webc.get_volume("metadata").unwrap(); + + assert_eq!( + meta_volume.read_file("LICENSE").unwrap(), + (b"license".as_slice().into(), Some(license_hash)), + ); + assert_eq!( + meta_volume.read_file("README.md").unwrap(), + (b"readme".as_slice().into(), Some(readme_hash)), + ); + assert_eq!( + dir1_volume.read_file("a").unwrap(), + (b"a".as_slice().into(), Some(a_hash)) + ); + assert_eq!( + dir1_volume.read_file("b").unwrap(), + (b"b".as_slice().into(), Some(b_hash)) + ); + } + + #[test] + fn serialize_package_with_nested_fs_entries_without_local_base_fs_paths() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + readme = 'README.md' + license-file = 'LICENSE' + + [fs] + "/path_in_wasix" = "local-dir/dir1" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("LICENSE"), "license").unwrap(); + + // Now we want to set up the following filesystem tree: + // + // - local-dir/ + // - dir1/ + // - dir2/ + // - a + // - b + let local_dir = temp.path().join("local-dir"); + std::fs::create_dir_all(&local_dir).unwrap(); + + let dir1 = local_dir.join("dir1"); + std::fs::create_dir_all(&dir1).unwrap(); + + let dir2 = dir1.join("dir2"); + std::fs::create_dir_all(&dir2).unwrap(); + + let a = dir2.join("a"); + let b = dir1.join("b"); + + std::fs::write(a, "a").unwrap(); + std::fs::write(b, "b").unwrap(); + + let package = Package::from_manifest(manifest).unwrap(); + + let webc = package.serialize().unwrap(); + let webc = from_bytes(webc).unwrap(); + let manifest = webc.manifest(); + let wapm_metadata = manifest.wapm().unwrap().unwrap(); + + assert!(wapm_metadata.name.is_none()); + assert!(wapm_metadata.version.is_none()); + assert!(wapm_metadata.description.is_none()); + + let fs_table = manifest.filesystem().unwrap().unwrap(); + assert_eq!( + fs_table, + [FileSystemMapping { + from: None, + volume_name: "/local-dir/dir1".to_string(), + host_path: None, + mount_path: "/path_in_wasix".to_string(), + },] + ); + + let readme_hash: [u8; 32] = sha2::Sha256::digest(b"readme").into(); + let license_hash: [u8; 32] = sha2::Sha256::digest(b"license").into(); + + let a_hash: [u8; 32] = sha2::Sha256::digest(b"a").into(); + let dir2_hash: [u8; 32] = sha2::Sha256::digest(a_hash).into(); + let b_hash: [u8; 32] = sha2::Sha256::digest(b"b").into(); + + let dir1_volume = webc.get_volume("/local-dir/dir1").unwrap(); + let meta_volume = webc.get_volume("metadata").unwrap(); + + assert_eq!( + meta_volume.read_file("LICENSE").unwrap(), + (b"license".as_slice().into(), Some(license_hash)), + ); + assert_eq!( + meta_volume.read_file("README.md").unwrap(), + (b"readme".as_slice().into(), Some(readme_hash)), + ); + assert_eq!( + dir1_volume + .read_dir("/") + .unwrap() + .into_iter() + .map(|(p, h, _)| (p, h)) + .collect::>(), + vec![ + (PathSegment::parse("b").unwrap(), Some(b_hash)), + (PathSegment::parse("dir2").unwrap(), Some(dir2_hash)) + ] + ); + assert_eq!( + dir1_volume + .read_dir("/dir2") + .unwrap() + .into_iter() + .map(|(p, h, _)| (p, h)) + .collect::>(), + vec![(PathSegment::parse("a").unwrap(), Some(a_hash))] + ); + assert_eq!( + dir1_volume.read_file("/dir2/a").unwrap(), + (b"a".as_slice().into(), Some(a_hash)) + ); + assert_eq!( + dir1_volume.read_file("/b").unwrap(), + (b"b".as_slice().into(), Some(b_hash)) + ); + } + + #[test] + fn serialize_package_mapped_to_same_dir_without_local_base_fs_paths() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "Test package" + readme = 'README.md' + license-file = 'LICENSE' + + [fs] + "/dir1" = "local-dir1/dir" + "/dir2" = "local-dir2/dir" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("LICENSE"), "license").unwrap(); + + // Now we want to set up the following filesystem tree: + // + // - local-dir1/ + // - dir + // - local-dir2/ + // - dir + let dir1 = temp.path().join("local-dir1").join("dir"); + std::fs::create_dir_all(&dir1).unwrap(); + let dir2 = temp.path().join("local-dir2").join("dir"); + std::fs::create_dir_all(&dir2).unwrap(); + + let package = Package::from_manifest(manifest).unwrap(); + + let webc = package.serialize().unwrap(); + let webc = from_bytes(webc).unwrap(); + let manifest = webc.manifest(); + let wapm_metadata = manifest.wapm().unwrap().unwrap(); + + assert!(wapm_metadata.name.is_none()); + assert!(wapm_metadata.version.is_none()); + assert!(wapm_metadata.description.is_none()); + + let fs_table = manifest.filesystem().unwrap().unwrap(); + assert_eq!( + fs_table, + [ + FileSystemMapping { + from: None, + volume_name: "/local-dir1/dir".to_string(), + host_path: None, + mount_path: "/dir1".to_string(), + }, + FileSystemMapping { + from: None, + volume_name: "/local-dir2/dir".to_string(), + host_path: None, + mount_path: "/dir2".to_string(), + }, + ] + ); + + let readme_hash: [u8; 32] = sha2::Sha256::digest(b"readme").into(); + let license_hash: [u8; 32] = sha2::Sha256::digest(b"license").into(); + + let dir1_volume = webc.get_volume("/local-dir1/dir").unwrap(); + let dir2_volume = webc.get_volume("/local-dir2/dir").unwrap(); + let meta_volume = webc.get_volume("metadata").unwrap(); + + assert_eq!( + meta_volume.read_file("LICENSE").unwrap(), + (b"license".as_slice().into(), Some(license_hash)), + ); + assert_eq!( + meta_volume.read_file("README.md").unwrap(), + (b"readme".as_slice().into(), Some(readme_hash)), + ); + assert!(dir1_volume.read_dir("/").unwrap().is_empty(),); + assert!(dir2_volume.read_dir("/").unwrap().is_empty(),); + } + + #[test] + fn metadata_only_contains_relevant_files() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "" + license-file = "./path/to/LICENSE" + readme = "README.md" + + [[module]] + name = "asdf" + source = "asdf.wasm" + abi = "none" + bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] } + "#; + + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + + let license_dir = temp.path().join("path").join("to"); + std::fs::create_dir_all(&license_dir).unwrap(); + std::fs::write(license_dir.join("LICENSE"), "license").unwrap(); + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("asdf.wasm"), b"\0asm...").unwrap(); + std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap(); + std::fs::write(temp.path().join("browser.wai"), "imports").unwrap(); + std::fs::write(temp.path().join("unwanted_file.txt"), "unwanted_file").unwrap(); + + let package = Package::from_manifest(manifest).unwrap(); + + let contents: Vec<_> = package + .get_volume("metadata") + .unwrap() + .read_dir(&PathSegments::ROOT) + .unwrap() + .into_iter() + .map(|(path, _, _)| path) + .collect(); + + assert_eq!( + contents, + vec![ + PathSegment::parse("README.md").unwrap(), + PathSegment::parse("asdf.wai").unwrap(), + PathSegment::parse("browser.wai").unwrap(), + PathSegment::parse("path").unwrap(), + ] + ); + } + + #[test] + fn create_from_in_memory() -> anyhow::Result<()> { + let wasmer_toml = r#" + [dependencies] + "wasmer/python" = "3.12.9+build.9" + + + [[command]] + module = "wasmer/python:python" + name = "hello" + runner = "wasi" + + [command.annotations.wasi] + main-args = [ "-c", "import os; print([f for f in os.walk('/public')]); " ] + + [fs] + "/public" = "public" + "#; + + let manifest = toml::from_str(wasmer_toml)?; + + let file_modified = SystemTime::now(); + let file_data = String::from("Hello, world!").as_bytes().to_vec(); + + let file = MemoryFile { + modified: file_modified, + data: file_data, + }; + + let mut nodes = BTreeMap::new(); + nodes.insert(String::from("hello.txt"), MemoryNode::File(file)); + + let dir_modified = SystemTime::now(); + let dir = MemoryDir { + modified: dir_modified, + nodes, + }; + + let volume = MemoryVolume { node: dir }; + let mut volumes = BTreeMap::new(); + + volumes.insert("public".to_string(), volume); + + let atoms = BTreeMap::new(); + let package = super::Package::from_in_memory( + manifest, + volumes, + atoms, + MemoryVolume { + node: MemoryDir { + modified: SystemTime::now(), + nodes: BTreeMap::new(), + }, + }, + Strictness::Strict, + )?; + + _ = package.serialize()?; + + Ok(()) + } + + #[test] + fn compare_fs_mem_manifest() -> anyhow::Result<()> { + let wasmer_toml = r#" + [package] + name = "test" + version = "0.0.0" + description = "asdf" + "#; + + let temp = TempDir::new()?; + let manifest_path = temp.path().join("wasmer.toml"); + std::fs::write(&manifest_path, wasmer_toml).unwrap(); + + let fs_package = super::Package::from_manifest(manifest_path)?; + + let manifest = toml::from_str(wasmer_toml)?; + let memory_package = super::Package::from_in_memory( + manifest, + Default::default(), + Default::default(), + MemoryVolume { + node: MemoryDir { + modified: SystemTime::UNIX_EPOCH, + nodes: BTreeMap::new(), + }, + }, + Strictness::Lossy, + )?; + + assert_eq!(memory_package.serialize()?, fs_package.serialize()?); + + Ok(()) + } + + #[test] + fn compare_fs_mem_manifest_and_atoms() -> anyhow::Result<()> { + let wasmer_toml = r#" + [package] + name = "test" + version = "0.0.0" + description = "asdf" + + [[module]] + name = "foo" + source = "foo.wasm" + abi = "wasi" + "#; + + let temp = TempDir::new()?; + let manifest_path = temp.path().join("wasmer.toml"); + std::fs::write(&manifest_path, wasmer_toml).unwrap(); + + let atom_path = temp.path().join("foo.wasm"); + std::fs::write(&atom_path, b"").unwrap(); + + let fs_package = super::Package::from_manifest(manifest_path)?; + + let manifest = toml::from_str(wasmer_toml)?; + let mut atoms = BTreeMap::new(); + atoms.insert("foo".to_owned(), (None, OwnedBuffer::new())); + let memory_package = super::Package::from_in_memory( + manifest, + Default::default(), + atoms, + MemoryVolume { + node: MemoryDir { + modified: SystemTime::UNIX_EPOCH, + nodes: BTreeMap::new(), + }, + }, + Strictness::Lossy, + )?; + + assert_eq!(memory_package.serialize()?, fs_package.serialize()?); + + Ok(()) + } + + #[test] + fn compare_fs_mem_volume() -> anyhow::Result<()> { + let wasmer_toml = r#" + [package] + name = "test" + version = "0.0.0" + description = "asdf" + + [[module]] + name = "foo" + source = "foo.wasm" + abi = "wasi" + + [fs] + "/bar" = "bar" + "#; + + let temp = TempDir::new()?; + let manifest_path = temp.path().join("wasmer.toml"); + std::fs::write(&manifest_path, wasmer_toml).unwrap(); + + let atom_path = temp.path().join("foo.wasm"); + std::fs::write(&atom_path, b"").unwrap(); + + let bar = temp.path().join("bar"); + std::fs::create_dir(&bar).unwrap(); + + let baz = bar.join("baz"); + std::fs::write(&baz, b"abc")?; + + let baz_metadata = std::fs::metadata(&baz)?; + + let fs_package = super::Package::from_manifest(manifest_path)?; + + let manifest = toml::from_str(wasmer_toml)?; + + let mut atoms = BTreeMap::new(); + atoms.insert("foo".to_owned(), (None, OwnedBuffer::new())); + + let mut volumes = BTreeMap::new(); + volumes.insert( + "/bar".to_owned(), + MemoryVolume { + node: MemoryDir { + modified: SystemTime::UNIX_EPOCH, + nodes: { + let mut children = BTreeMap::new(); + + children.insert( + "baz".to_owned(), + MemoryNode::File(MemoryFile { + modified: baz_metadata.modified()?, + data: b"abc".to_vec(), + }), + ); + + children + }, + }, + }, + ); + let memory_package = super::Package::from_in_memory( + manifest, + volumes, + atoms, + MemoryVolume { + node: MemoryDir { + modified: SystemTime::UNIX_EPOCH, + nodes: Default::default(), + }, + }, + Strictness::Lossy, + )?; + + assert_eq!(memory_package.serialize()?, fs_package.serialize()?); + + Ok(()) + } + + #[test] + fn compare_fs_mem_bindings() -> anyhow::Result<()> { + let temp = TempDir::new().unwrap(); + + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "" + license-file = "LICENSE" + readme = "README.md" + + [[module]] + name = "asdf" + source = "asdf.wasm" + abi = "none" + bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] } + + [fs] + "/dir1" = "local-dir1/dir" + "/dir2" = "local-dir2/dir" + "#; + + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + + std::fs::write(temp.path().join("LICENSE"), "license").unwrap(); + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("asdf.wasm"), b"\0asm...").unwrap(); + std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap(); + std::fs::write(temp.path().join("browser.wai"), "imports").unwrap(); + + // Now we want to set up the following filesystem tree: + // + // - local-dir1/ + // - dir + // - local-dir2/ + // - dir + let dir1 = temp.path().join("local-dir1").join("dir"); + std::fs::create_dir_all(&dir1).unwrap(); + let dir2 = temp.path().join("local-dir2").join("dir"); + std::fs::create_dir_all(&dir2).unwrap(); + + let fs_package = super::Package::from_manifest(manifest)?; + + let manifest = toml::from_str(wasmer_toml)?; + + let mut atoms = BTreeMap::new(); + atoms.insert( + "asdf".to_owned(), + (None, OwnedBuffer::from_static(b"\0asm...")), + ); + + let mut volumes = BTreeMap::new(); + volumes.insert( + "/local-dir1/dir".to_owned(), + MemoryVolume { + node: MemoryDir { + modified: SystemTime::UNIX_EPOCH, + nodes: Default::default(), + }, + }, + ); + volumes.insert( + "/local-dir2/dir".to_owned(), + MemoryVolume { + node: MemoryDir { + modified: SystemTime::UNIX_EPOCH, + nodes: Default::default(), + }, + }, + ); + + let memory_package = super::Package::from_in_memory( + manifest, + volumes, + atoms, + MemoryVolume { + node: MemoryDir { + modified: SystemTime::UNIX_EPOCH, + nodes: { + let mut children = BTreeMap::new(); + + children.insert( + "README.md".to_owned(), + MemoryNode::File(MemoryFile { + modified: temp.path().join("README.md").metadata()?.modified()?, + data: b"readme".to_vec(), + }), + ); + + children.insert( + "LICENSE".to_owned(), + MemoryNode::File(MemoryFile { + modified: temp.path().join("LICENSE").metadata()?.modified()?, + data: b"license".to_vec(), + }), + ); + + children.insert( + "asdf.wai".to_owned(), + MemoryNode::File(MemoryFile { + modified: temp.path().join("asdf.wai").metadata()?.modified()?, + data: b"exports".to_vec(), + }), + ); + + children.insert( + "browser.wai".to_owned(), + MemoryNode::File(MemoryFile { + modified: temp.path().join("browser.wai").metadata()?.modified()?, + data: b"imports".to_vec(), + }), + ); + + children + }, + }, + }, + Strictness::Lossy, + )?; + + let memory_package = memory_package.serialize()?; + let fs_package = fs_package.serialize()?; + + assert_eq!(memory_package, fs_package); + + Ok(()) + } +} diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__custom_annotations_are_copied_across_verbatim.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__custom_annotations_are_copied_across_verbatim.snap new file mode 100644 index 00000000000..2fe34c410f0 --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__custom_annotations_are_copied_across_verbatim.snap @@ -0,0 +1,26 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"test\"\n version = \"0.0.0\"\n description = \"asdf\"\n\n [[module]]\n name = \"module\"\n source = \"file.wasm\"\n abi = \"wasi\"\n\n [[command]]\n name = \"command\"\n module = \"module\"\n runner = \"asdf\"\n annotations = { first = 42, second = [\"a\", \"b\"] }\n " +expression: "&transformed" +--- +package: + wapm: + name: test + version: 0.0.0 + description: asdf +atoms: + module: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +commands: + command: + runner: "https://webc.org/runner/asdf" + annotations: + atom: + name: module + first: 42 + second: + - a + - b +entrypoint: command + diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__issue_124_command_runner_is_swallowed.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__issue_124_command_runner_is_swallowed.snap new file mode 100644 index 00000000000..b5257fed243 --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__issue_124_command_runner_is_swallowed.snap @@ -0,0 +1,24 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"wasmer-tests/wcgi-always-panic\"\n version = \"0.1.0\"\n description = \"wasmer-tests/wcgi-always-panic website\"\n\n [[module]]\n name = \"wcgi-always-panic\"\n source = \"./wcgi-always-panic.wasm\"\n abi = \"wasi\"\n\n [[command]]\n name = \"wcgi\"\n module = \"wcgi-always-panic\"\n runner = \"https://webc.org/runner/wcgi\"\n " +expression: "&transformed" +--- +package: + wapm: + name: wasmer-tests/wcgi-always-panic + version: 0.1.0 + description: wasmer-tests/wcgi-always-panic website +atoms: + wcgi-always-panic: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +commands: + wcgi: + runner: "https://webc.org/runner/wcgi" + annotations: + atom: + name: wcgi-always-panic + wasi: + atom: wcgi-always-panic +entrypoint: wcgi + diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__merge_custom_attributes_with_builtin_ones.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__merge_custom_attributes_with_builtin_ones.snap new file mode 100644 index 00000000000..78e6f7d309a --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__merge_custom_attributes_with_builtin_ones.snap @@ -0,0 +1,25 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"My awesome package\"\n\n [[module]]\n name = \"cpython\"\n source = \"python.wasm\"\n abi = \"wasi\"\n\n [[command]]\n name = \"python\"\n module = \"cpython\"\n runner = \"wasi\"\n annotations = { wasi = { env = [\"KEY=val\"]} }\n " +expression: "&transformed" +--- +package: + wapm: + name: some/package + version: 0.0.0 + description: My awesome package +atoms: + cpython: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +commands: + python: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: cpython + wasi: + atom: cpython + env: + - KEY=val +entrypoint: python diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_bash_manifest.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_bash_manifest.snap new file mode 100644 index 00000000000..0d24c0c175d --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_bash_manifest.snap @@ -0,0 +1,26 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"sharrattj/bash\"\n version = \"1.0.17\"\n description = \"Bash is a modern POSIX-compliant implementation of /bin/sh.\"\n license = \"GNU\"\n wasmer-extra-flags = \"--enable-threads --enable-bulk-memory\"\n\n [dependencies]\n \"sharrattj/coreutils\" = \"1.0.16\"\n\n [[module]]\n name = \"bash\"\n source = \"bash.wasm\"\n abi = \"wasi\"\n\n [[command]]\n name = \"bash\"\n module = \"bash\"\n runner = \"wasi@unstable_\"\n " +expression: "&transformed" +--- +use: + sharrattj/coreutils: sharrattj/coreutils@^1.0.16 +package: + wapm: + name: sharrattj/bash + version: 1.0.17 + description: Bash is a modern POSIX-compliant implementation of /bin/sh. + license: GNU +atoms: + bash: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +commands: + bash: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: bash + wasi: + atom: bash +entrypoint: bash diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_empty_manifest.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_empty_manifest.snap new file mode 100644 index 00000000000..0870a6ca270 --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_empty_manifest.snap @@ -0,0 +1,11 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"My awesome package\"\n " +expression: "&transformed" +--- +package: + wapm: + name: some/package + version: 0.0.0 + description: My awesome package + diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_atom_and_command.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_atom_and_command.snap new file mode 100644 index 00000000000..7088d20848e --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_atom_and_command.snap @@ -0,0 +1,24 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"My awesome package\"\n\n [[module]]\n name = \"cpython\"\n source = \"python.wasm\"\n abi = \"wasi\"\n\n [[command]]\n name = \"python\"\n module = \"cpython\"\n runner = \"wasi\"\n " +expression: "&transformed" +--- +package: + wapm: + name: some/package + version: 0.0.0 + description: My awesome package +atoms: + cpython: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +commands: + python: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: cpython + wasi: + atom: cpython +entrypoint: python + diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_fs_table.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_fs_table.snap new file mode 100644 index 00000000000..76aacb266eb --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_fs_table.snap @@ -0,0 +1,17 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"This is a package\"\n\n [fs]\n lib = \"lib\"\n \"/public\" = \"out\"\n " +expression: "&transformed" +--- +package: + wapm: + name: some/package + version: 0.0.0 + description: This is a package + fs: + - from: ~ + volume_name: /lib + mount_path: /lib + - from: ~ + volume_name: /out + mount_path: /public diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_multiple_commands.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_multiple_commands.snap new file mode 100644 index 00000000000..ca2d88afc8d --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_multiple_commands.snap @@ -0,0 +1,37 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"My awesome package\"\n\n [[module]]\n name = \"cpython\"\n source = \"python.wasm\"\n abi = \"wasi\"\n\n [[command]]\n name = \"first\"\n module = \"cpython\"\n runner = \"wasi\"\n\n [[command]]\n name = \"second\"\n module = \"cpython\"\n runner = \"wasi\"\n\n [[command]]\n name = \"third\"\n module = \"cpython\"\n runner = \"wasi\"\n " +expression: "&transformed" +--- +package: + wapm: + name: some/package + version: 0.0.0 + description: My awesome package +atoms: + cpython: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +commands: + first: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: cpython + wasi: + atom: cpython + second: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: cpython + wasi: + atom: cpython + third: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: cpython + wasi: + atom: cpython + diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_single_atom.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_single_atom.snap new file mode 100644 index 00000000000..c320b3a80db --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_manifest_with_single_atom.snap @@ -0,0 +1,15 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"My awesome package\"\n\n [[module]]\n name = \"first\"\n source = \"./path/to/file.wasm\"\n abi = \"wasi\"\n " +expression: "&transformed" +--- +package: + wapm: + name: some/package + version: 0.0.0 + description: My awesome package +atoms: + first: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" + diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_python_manifest.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_python_manifest.snap new file mode 100644 index 00000000000..a9cd3d91616 --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_python_manifest.snap @@ -0,0 +1,29 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"python\"\n version = \"0.1.0\"\n description = \"Python is an interpreted, high-level, general-purpose programming language\"\n license = \"ISC\"\n repository = \"https://github.com/wapm-packages/python\"\n\n [[module]]\n name = \"python\"\n source = \"bin/python.wasm\"\n abi = \"wasi\"\n\n [module.interfaces]\n wasi = \"0.0.0-unstable\"\n\n [[command]]\n name = \"python\"\n module = \"python\"\n\n [fs]\n lib = \"lib\"\n " +expression: "&transformed" +--- +package: + wapm: + name: python + version: 0.1.0 + description: "Python is an interpreted, high-level, general-purpose programming language" + license: ISC + repository: "https://github.com/wapm-packages/python" + fs: + - from: ~ + volume_name: /lib + mount_path: /lib +atoms: + python: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +commands: + python: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: python + wasi: + atom: python +entrypoint: python diff --git a/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_wasmer_pack_manifest.snap b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_wasmer_pack_manifest.snap new file mode 100644 index 00000000000..fa86e628265 --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__manifest__tests__transform_wasmer_pack_manifest.snap @@ -0,0 +1,28 @@ +--- +source: crates/webc/src/wasmer_package/manifest.rs +description: "\n [package]\n name = \"wasmer/wasmer-pack\"\n version = \"0.7.0\"\n description = \"The WebAssembly interface to wasmer-pack.\"\n license = \"MIT\"\n readme = \"README.md\"\n repository = \"https://github.com/wasmerio/wasmer-pack\"\n homepage = \"https://wasmer.io/\"\n\n [[module]]\n name = \"wasmer-pack-wasm\"\n source = \"wasmer_pack_wasm.wasm\"\n\n [module.bindings]\n wai-version = \"0.2.0\"\n exports = \"wasmer-pack.exports.wai\"\n imports = []\n " +expression: "&transformed" +--- +package: + wapm: + name: wasmer/wasmer-pack + version: 0.7.0 + description: The WebAssembly interface to wasmer-pack. + license: MIT + readme: + volume: metadata + path: /README.md + repository: "https://github.com/wasmerio/wasmer-pack" + homepage: "https://wasmer.io/" +atoms: + wasmer-pack-wasm: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +bindings: + - name: library-bindings + kind: wai@0.2.0 + annotations: + wai: + exports: "metadata://wasmer-pack.exports.wai" + module: wasmer-pack-wasm + imports: [] diff --git a/lib/package/src/package/snapshots/wasmer_package__package__package__tests__absolute_paths_in_wasmer_toml_issue_105.snap b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__absolute_paths_in_wasmer_toml_issue_105.snap new file mode 100644 index 00000000000..15884c47a92 --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__absolute_paths_in_wasmer_toml_issue_105.snap @@ -0,0 +1,26 @@ +--- +source: crates/webc/src/wasmer_package/package.rs +description: "\n [package]\n name = 'some/package'\n version = '0.0.0'\n description = 'Test package'\n readme = '[BASE_DIR]/README.md'\n license-file = '[BASE_DIR]/LICENSE'\n\n [[module]]\n name = 'first'\n source = '[BASE_DIR]/target/debug/package.wasm'\n bindings = { wai-version = '0.2.0', exports = '[BASE_DIR]/bindings/file.wai', imports = ['[BASE_DIR]/bindings/a.wai'] }\n " +expression: webc.manifest() +--- +package: + wapm: + license-file: + volume: metadata + path: /LICENSE + readme: + volume: metadata + path: /README.md +atoms: + first: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +bindings: + - name: library-bindings + kind: wai@0.2.0 + annotations: + wai: + exports: "metadata://bindings/file.wai" + module: first + imports: + - "metadata://bindings/a.wai" diff --git a/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_old_cowsay.snap b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_old_cowsay.snap new file mode 100644 index 00000000000..30f9928f0bb --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_old_cowsay.snap @@ -0,0 +1,29 @@ +--- +source: crates/webc/src/wasmer_package/package.rs +expression: pkg.manifest() +--- +package: + wapm: + readme: + volume: metadata + path: /README.md + repository: "https://github.com/wapm-packages/cowsay" +atoms: + cowsay: + kind: "https://webc.org/kind/wasm" + signature: "sha256:DPmhiSNXCg5261eTUi3BIvAc/aJttGj+nD+bGhQkVQo=" +commands: + cowsay: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: cowsay + wasi: + atom: cowsay + cowthink: + runner: "https://webc.org/runner/wasi" + annotations: + atom: + name: cowsay + wasi: + atom: cowsay diff --git a/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_package_with_wai_bindings.snap b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_package_with_wai_bindings.snap new file mode 100644 index 00000000000..243ecd921ff --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_package_with_wai_bindings.snap @@ -0,0 +1,21 @@ +--- +source: crates/webc/src/wasmer_package/package.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"\"\n\n [[module]]\n name = \"my-lib\"\n source = \"./my-lib.wasm\"\n abi = \"none\"\n bindings = { wai-version = \"0.2.0\", exports = \"./file.wai\", imports = [\"a.wai\", \"b.wai\"] }\n " +expression: webc.manifest() +--- +package: + wapm: {} +atoms: + my-lib: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +bindings: + - name: library-bindings + kind: wai@0.2.0 + annotations: + wai: + exports: "metadata://file.wai" + module: my-lib + imports: + - "metadata://a.wai" + - "metadata://b.wai" diff --git a/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_package_with_wit_bindings.snap b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_package_with_wit_bindings.snap new file mode 100644 index 00000000000..020af80dfed --- /dev/null +++ b/lib/package/src/package/snapshots/wasmer_package__package__package__tests__load_package_with_wit_bindings.snap @@ -0,0 +1,18 @@ +--- +source: crates/webc/src/wasmer_package/package.rs +description: "\n [package]\n name = \"some/package\"\n version = \"0.0.0\"\n description = \"\"\n\n [[module]]\n name = \"my-lib\"\n source = \"./my-lib.wasm\"\n abi = \"none\"\n bindings = { wit-bindgen = \"0.1.0\", wit-exports = \"./file.wit\" }\n " +expression: webc.manifest() +--- +package: + wapm: {} +atoms: + my-lib: + kind: "https://webc.org/kind/wasm" + signature: "sha256:Wjn+71LlO4/+39cFFVbsEF7YaYLxIqBdJyjZZ3jk65Y=" +bindings: + - name: library-bindings + kind: wit@0.1.0 + annotations: + wit: + exports: "metadata://file.wit" + module: my-lib diff --git a/lib/package/src/package/strictness.rs b/lib/package/src/package/strictness.rs new file mode 100644 index 00000000000..059078e32f3 --- /dev/null +++ b/lib/package/src/package/strictness.rs @@ -0,0 +1,57 @@ +use std::{fmt::Debug, path::Path}; + +use anyhow::Error; + +use super::ManifestError; + +/// The strictness to use when working with a +/// [`crate::wasmer_package::Package`]. +/// +/// This can be useful when loading a package that may be edited interactively +/// or if you just want to use a package and don't care if the manifest is +/// invalid. +#[derive(Debug, Default, Copy, Clone, PartialEq)] +pub enum Strictness { + /// Prefer to lose data rather than error out. + #[default] + Lossy, + /// All package issues should be errors. + Strict, +} + +impl Strictness { + pub(crate) fn is_strict(self) -> bool { + matches!(self, Strictness::Strict) + } + + pub(crate) fn on_error(&self, _path: &Path, error: Error) -> Result<(), Error> { + match self { + Strictness::Lossy => Ok(()), + Strictness::Strict => Err(error), + } + } + + pub(crate) fn outside_base_directory( + &self, + path: &Path, + base_dir: &Path, + ) -> Result<(), ManifestError> { + match self { + Strictness::Lossy => todo!(), + Strictness::Strict => Err(ManifestError::OutsideBaseDirectory { + path: path.to_path_buf(), + base_dir: base_dir.to_path_buf(), + }), + } + } + + pub(crate) fn missing_file(&self, path: &Path, base_dir: &Path) -> Result<(), ManifestError> { + match self { + Strictness::Lossy => Ok(()), + Strictness::Strict => Err(ManifestError::MissingFile { + path: path.to_path_buf(), + base_dir: base_dir.to_path_buf(), + }), + } + } +} diff --git a/lib/package/src/package/volume/fs.rs b/lib/package/src/package/volume/fs.rs new file mode 100644 index 00000000000..28db0de3033 --- /dev/null +++ b/lib/package/src/package/volume/fs.rs @@ -0,0 +1,518 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fs::File, + io::Read, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Error}; +use shared_buffer::OwnedBuffer; + +use webc::{ + sanitize_path, + v3::{ + self, + write::{DirEntry, Directory, FileEntry}, + }, + AbstractVolume, Metadata, PathSegment, PathSegments, Timestamps, ToPathSegments, +}; + +use crate::package::Strictness; + +use super::WasmerPackageVolume; + +/// A lazily loaded volume in a Wasmer package. +/// +/// Note that it is the package resolver's role to interpret a package's +/// [`crate::metadata::annotations::FileSystemMappings`]. A [`Volume`] contains +/// directories as they were when the package was published. +#[derive(Debug, Clone, PartialEq)] +pub struct FsVolume { + /// Name of the volume + name: String, + /// A pre-computed set of intermediate directories that are needed to allow + /// access to the whitelisted files and directories. + intermediate_directories: BTreeSet, + /// Specific files that this volume has access to. + metadata_files: BTreeSet, + /// Directories that allow the user to access anything inside them. + mapped_directories: BTreeSet, + /// The base directory all [`PathSegments`] will be resolved relative to. + base_dir: PathBuf, +} + +impl FsVolume { + /// The name of the volume used to store metadata files. + pub(crate) const METADATA: &'static str = "metadata"; + + /// Create a new metadata volume. + pub(crate) fn new_metadata( + manifest: &wasmer_config::package::Manifest, + base_dir: impl Into, + ) -> Result { + let base_dir = base_dir.into(); + let mut files = BTreeSet::new(); + + // check if manifest.package is None + if let Some(package) = &manifest.package { + if let Some(license_file) = &package.license_file { + files.insert(base_dir.join(license_file)); + } + + if let Some(readme) = &package.readme { + files.insert(base_dir.join(readme)); + } + } + + for module in &manifest.modules { + if let Some(bindings) = &module.bindings { + let bindings_files = bindings.referenced_files(&base_dir)?; + files.extend(bindings_files); + } + } + + Ok(FsVolume::new_with_intermediate_dirs( + FsVolume::METADATA.to_string(), + base_dir, + files, + BTreeSet::new(), + )) + } + + pub(crate) fn new_assets( + manifest: &wasmer_config::package::Manifest, + base_dir: &Path, + ) -> Result, Error> { + // Create asset volumes + let dirs: BTreeSet<_> = manifest + .fs + .values() + .map(|path| base_dir.join(path)) + .collect(); + + for path in &dirs { + // Perform a basic sanity check to make sure the directories exist. + let _ = std::fs::metadata(path).with_context(|| { + format!("Unable to get the metadata for \"{}\"", path.display()) + })?; + } + + let mut volumes = BTreeMap::new(); + for entry in manifest.fs.values() { + let name = entry + .to_str() + .ok_or_else(|| anyhow::anyhow!("Failed to convert path to str"))?; + + let name = sanitize_path(name); + + let mut dirs = BTreeSet::new(); + let dir = base_dir.join(entry); + dirs.insert(dir); + + volumes.insert( + name.clone(), + FsVolume::new( + name.to_string(), + base_dir.to_path_buf(), + BTreeSet::new(), + dirs, + ), + ); + } + + Ok(volumes) + } + + pub(crate) fn new_with_intermediate_dirs( + name: String, + base_dir: PathBuf, + whitelisted_files: BTreeSet, + whitelisted_directories: BTreeSet, + ) -> Self { + let mut intermediate_directories: BTreeSet = whitelisted_files + .iter() + .filter_map(|p| p.parent()) + .chain(whitelisted_directories.iter().map(|p| p.as_path())) + .flat_map(|dir| dir.ancestors()) + .filter(|dir| dir.starts_with(&base_dir)) + .map(|dir| dir.to_path_buf()) + .collect(); + + // The base directory is always accessible (even if its contents isn't) + intermediate_directories.insert(base_dir.clone()); + + FsVolume { + name, + intermediate_directories, + metadata_files: whitelisted_files, + mapped_directories: whitelisted_directories, + base_dir, + } + } + + pub(crate) fn new( + name: String, + base_dir: PathBuf, + whitelisted_files: BTreeSet, + whitelisted_directories: BTreeSet, + ) -> Self { + FsVolume { + name, + intermediate_directories: BTreeSet::new(), + metadata_files: whitelisted_files, + mapped_directories: whitelisted_directories, + base_dir, + } + } + + fn is_accessible(&self, path: &Path) -> bool { + self.intermediate_directories.contains(path) + || self.metadata_files.contains(path) + || self + .mapped_directories + .iter() + .any(|dir| path.starts_with(dir)) + } + + fn resolve(&self, path: &PathSegments) -> Option { + let resolved = if let Some(dir) = &self.mapped_directories.first() { + resolve(dir, path) + } else { + resolve(&self.base_dir, path) + }; + + let accessible = self.is_accessible(&resolved); + accessible.then_some(resolved) + } + + /// Returns the name of the volume + pub fn name(&self) -> &str { + self.name.as_str() + } + + /// Read a file from the volume. + pub fn read_file(&self, path: &PathSegments) -> Option { + let path = self.resolve(path)?; + let mut f = File::open(path).ok()?; + + // First we try to mmap it + if let Ok(mmapped) = OwnedBuffer::from_file(&f) { + return Some(mmapped); + } + + // otherwise, fall back to reading the file's contents into memory + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer).ok()?; + Some(OwnedBuffer::from_bytes(buffer)) + } + + /// Read the contents of a directory. + #[allow(clippy::type_complexity)] + pub fn read_dir( + &self, + path: &PathSegments, + ) -> Option, Metadata)>> { + let resolved = self.resolve(path)?; + let mut entries = Vec::new(); + + for entry in resolved.read_dir().ok()? { + let entry = entry.ok()?.path(); + + if !self.is_accessible(&entry) { + continue; + } + + let segment: PathSegment = entry.file_name()?.to_str()?.parse().ok()?; + + let path = path.join(segment.clone()); + let metadata = self.metadata(&path)?; + entries.push((segment, None, metadata)); + } + + entries.sort_by_key(|k| k.0.clone()); + + Some(entries) + } + + /// Get the metadata for a particular item. + pub fn metadata(&self, path: &PathSegments) -> Option { + let path = self.resolve(path)?; + let meta = path.metadata().ok()?; + + let timestamps = Timestamps::from_metadata(&meta).unwrap(); + + if meta.is_dir() { + Some(Metadata::Dir { + timestamps: Some(timestamps), + }) + } else if meta.is_file() { + Some(Metadata::File { + length: meta.len().try_into().ok()?, + timestamps: Some(timestamps), + }) + } else { + None + } + } + + pub(crate) fn as_directory_tree(&self, strictness: Strictness) -> Result, Error> { + if self.name() == "metadata" { + let mut root = Directory::default(); + + for file_path in self.metadata_files.iter() { + if !file_path.exists() || !file_path.is_file() { + if strictness.is_strict() { + anyhow::bail!("{} does not exist", file_path.display()); + } + + // ignore missing metadata + continue; + } + let path = file_path.strip_prefix(&self.base_dir)?; + let path = PathBuf::from("/").join(path); + let segments = path.to_path_segments()?; + let segments: Vec<_> = segments.iter().collect(); + + let file_entry = DirEntry::File(FileEntry::from_path(file_path)?); + + let mut curr_dir = &mut root; + for (index, segment) in segments.iter().enumerate() { + if segments.len() == 1 { + curr_dir.children.insert((*segment).clone(), file_entry); + break; + } else { + if index == segments.len() - 1 { + curr_dir.children.insert((*segment).clone(), file_entry); + break; + } + + let curr_entry = curr_dir + .children + .entry((*segment).clone()) + .or_insert(DirEntry::Dir(Directory::default())); + let DirEntry::Dir(dir) = curr_entry else { + unreachable!() + }; + + curr_dir = dir; + } + } + } + + Ok(root) + } else { + let paths: Vec<_> = self.mapped_directories.iter().cloned().collect(); + directory_tree(paths, &self.base_dir, strictness) + } + } +} + +impl AbstractVolume for FsVolume { + fn read_file(&self, path: &PathSegments) -> Option<(OwnedBuffer, Option<[u8; 32]>)> { + self.read_file(path).map(|c| (c, None)) + } + + fn read_dir( + &self, + path: &PathSegments, + ) -> Option, Metadata)>> { + self.read_dir(path) + } + + fn metadata(&self, path: &PathSegments) -> Option { + self.metadata(path) + } +} + +impl WasmerPackageVolume for FsVolume { + fn as_directory_tree(&self, strictness: Strictness) -> Result, Error> { + self.as_directory_tree(strictness) + } +} + +/// Resolve a [`PathSegments`] to its equivalent path on disk. +fn resolve(base_dir: &Path, path: &PathSegments) -> PathBuf { + let mut resolved = base_dir.to_path_buf(); + for segment in path.iter() { + resolved.push(segment.as_str()); + } + + resolved +} + +/// Given a list of absolute paths, create a directory tree relative to some +/// base directory. +fn directory_tree( + paths: impl IntoIterator, + base_dir: &Path, + strictness: Strictness, +) -> Result, Error> { + let paths: Vec<_> = paths.into_iter().collect(); + let mut root = Directory::default(); + + for path in paths { + if path.is_file() { + let dir_entry = v3::write::DirEntry::File(v3::write::FileEntry::from_path(&path)?); + let path = path.strip_prefix(base_dir)?; + let path_segment = PathSegment::try_from(path.as_os_str())?; + + if root.children.insert(path_segment, dir_entry).is_some() { + println!("Warning: {path:?} already exists. Overriding the old entry"); + } + } else { + match create_directory_tree(&path) { + Ok(dir) => { + for (path, child) in dir.children { + root.children.insert(path.clone(), child); + } + } + Err(e) => { + let error = e.context(format!( + "Unable to add \"{}\" to the directory tree", + path.display() + )); + strictness.on_error(&path, error)?; + } + } + } + } + + Ok(root) +} + +fn create_directory_tree(absolute: &Path) -> Result, Error> { + let mut children = BTreeMap::new(); + + for entry in absolute.read_dir()? { + let entry = entry?; + + let kind = entry.file_type()?; + let path = entry.path(); + + let entry = if kind.is_dir() { + v3::write::DirEntry::Dir(create_directory_tree(&path)?) + } else { + v3::write::DirEntry::File(v3::write::FileEntry::from_path(&path)?) + }; + + let path = path.strip_prefix(absolute)?; + let path_segment = webc::PathSegment::try_from(path.as_os_str())?; + + children.insert(path_segment, entry); + } + + let meta = absolute.metadata()?; + let timestamps = v3::Timestamps::from_metadata(&meta)?; + + let dir = v3::write::Directory::new(children, timestamps); + + Ok(dir) +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + use wasmer_config::package::Manifest; + + use super::*; + + #[test] + fn metadata_volume() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "" + license-file = "./path/to/LICENSE" + readme = "README.md" + + [[module]] + name = "asdf" + source = "asdf.wasm" + abi = "none" + bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] } + "#; + let wasmer_toml_path = temp.path().join("wasmer.toml"); + std::fs::write(&wasmer_toml_path, wasmer_toml.as_bytes()).unwrap(); + let license_dir = temp.path().join("path").join("to"); + std::fs::create_dir_all(&license_dir).unwrap(); + std::fs::write(license_dir.join("LICENSE"), "license").unwrap(); + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap(); + std::fs::write(temp.path().join("browser.wai"), "imports").unwrap(); + let manifest: Manifest = toml::from_str(wasmer_toml).unwrap(); + + let volume = FsVolume::new_metadata(&manifest, temp.path().to_path_buf()).unwrap(); + + let entries = volume.read_dir(&PathSegments::ROOT).unwrap(); + let expected = vec![ + PathSegment::parse("README.md").unwrap(), + PathSegment::parse("asdf.wai").unwrap(), + PathSegment::parse("browser.wai").unwrap(), + PathSegment::parse("path").unwrap(), + ]; + + for i in 0..expected.len() { + assert_eq!(entries[i].0, expected[i]); + assert!(entries[i].2.timestamps().is_some()); + } + + let license: PathSegments = "/path/to/LICENSE".parse().unwrap(); + assert_eq!( + String::from_utf8(volume.read_file(&license).unwrap().into()).unwrap(), + "license" + ); + } + + #[test] + fn asset_volume() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "" + license_file = "./path/to/LICENSE" + readme = "README.md" + + [[module]] + name = "asdf" + source = "asdf.wasm" + abi = "none" + bindings = { wai-version = "0.2.0", exports = "asdf.wai", imports = ["browser.wai"] } + + [fs] + "/etc" = "etc" + "#; + let license_dir = temp.path().join("path").join("to"); + std::fs::create_dir_all(&license_dir).unwrap(); + std::fs::write(license_dir.join("LICENSE"), "license").unwrap(); + std::fs::write(temp.path().join("README.md"), "readme").unwrap(); + std::fs::write(temp.path().join("asdf.wai"), "exports").unwrap(); + std::fs::write(temp.path().join("browser.wai"), "imports").unwrap(); + let share = temp.path().join("etc").join("share"); + std::fs::create_dir_all(&share).unwrap(); + std::fs::write(share.join("package.1"), "man page").unwrap(); + + let manifest: Manifest = toml::from_str(wasmer_toml).unwrap(); + + let volume = FsVolume::new_assets(&manifest, &temp.path()).unwrap(); + + let volume = &volume["/etc"]; + + let entries = volume.read_dir(&PathSegments::ROOT).unwrap(); + let expected = vec![PathSegment::parse("share").unwrap()]; + + for i in 0..expected.len() { + assert_eq!(entries[i].0, expected[i]); + assert!(entries[i].2.timestamps().is_some()); + } + + let man_page: PathSegments = "/share/package.1".parse().unwrap(); + assert_eq!( + String::from_utf8(volume.read_file(&man_page).unwrap().into()).unwrap(), + "man page" + ); + } +} diff --git a/lib/package/src/package/volume/in_memory.rs b/lib/package/src/package/volume/in_memory.rs new file mode 100644 index 00000000000..57547366618 --- /dev/null +++ b/lib/package/src/package/volume/in_memory.rs @@ -0,0 +1,465 @@ +use std::{ + collections::BTreeMap, + str::FromStr, + time::{SystemTime, UNIX_EPOCH}, +}; + +use webc::{ + v3::{self, write::FileEntry}, + AbstractVolume, Metadata, PathSegment, PathSegments, +}; + +use crate::package::Strictness; + +use super::WasmerPackageVolume; + +/// An in-memory representation of a volume. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct MemoryVolume { + /// The internal node + pub node: MemoryDir, +} + +impl MemoryVolume { + /// The name of the volume used to store metadata files. + pub(crate) const METADATA: &'static str = "metadata"; +} + +/// An in-memory representation of a filesystem node. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum MemoryNode { + /// A file + File(MemoryFile), + + /// A directory + Dir(MemoryDir), +} + +impl MemoryNode { + /// Try to return a [`MemoryDir`] out of [`self`]. + pub fn as_dir(&self) -> Option<&MemoryDir> { + match self { + MemoryNode::Dir(d) => Some(d), + _ => None, + } + } + + /// Try to return a [`MemoryFile`] out of [`self`]. + pub fn as_file(&self) -> Option<&MemoryFile> { + match self { + MemoryNode::File(f) => Some(f), + _ => None, + } + } + + fn as_dir_entry(&self) -> anyhow::Result> { + match self { + MemoryNode::File(f) => f.as_dir_entry(), + MemoryNode::Dir(d) => d.as_dir_entry(), + } + } + + fn metadata(&self) -> Metadata { + match self { + MemoryNode::File(f) => f.metadata(), + MemoryNode::Dir(d) => d.metadata(), + } + } +} + +/// An in-memory file. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct MemoryFile { + /// When the file was last modified. + pub modified: SystemTime, + /// Raw data + pub data: Vec, +} +impl MemoryFile { + fn as_dir_entry(&self) -> anyhow::Result> { + Ok(v3::write::DirEntry::File(FileEntry::owned( + self.data.clone(), + v3::Timestamps { + modified: self.modified, + }, + ))) + } + + fn metadata(&self) -> Metadata { + let modified = self.modified.duration_since(UNIX_EPOCH).unwrap().as_nanos() as u64; + Metadata::File { + length: self.data.len(), + timestamps: Some(webc::Timestamps::from_modified(modified)), + } + } +} + +/// An in-memory directory. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct MemoryDir { + /// When the directory or its contents were last modified. + pub modified: SystemTime, + /// List of nodes in the directory + pub nodes: BTreeMap, +} + +impl MemoryDir { + fn metadata(&self) -> Metadata { + let modified = self.modified.duration_since(UNIX_EPOCH).unwrap().as_nanos() as u64; + Metadata::Dir { + timestamps: Some(webc::Timestamps::from_modified(modified)), + } + } + + // Can't return a reference to MemoryNode as it can return itself. + fn find_node(&self, path: &PathSegments) -> Option { + let mut segments = path.iter().collect::>(); + if segments.is_empty() { + return Some(MemoryNode::Dir(self.clone())); + } + + let mut dir = self; + + while !segments.is_empty() { + let next = (*segments.first().unwrap()).clone(); + segments.remove(0); + + if let Some(next_node) = dir.nodes.get(&next.to_string()) { + if segments.is_empty() { + return Some(next_node.clone()); + } else { + match next_node { + MemoryNode::File(_) => break, + MemoryNode::Dir(d) => dir = d, + } + } + } + } + + None + } + + fn read_file(&self, path: &PathSegments) -> Option { + self.find_node(path).and_then(|n| { + if let MemoryNode::File(f) = n { + Some(shared_buffer::OwnedBuffer::from_bytes(f.data.clone())) + } else { + None + } + }) + } + + #[allow(clippy::type_complexity)] + fn read_dir( + &self, + path: &PathSegments, + ) -> Option, Metadata)>> { + self.find_node(path).and_then(|n| { + if let MemoryNode::Dir(d) = n { + let mut ret = vec![]; + + for (name, node) in &d.nodes { + let meta = node.metadata(); + ret.push((PathSegment::from_str(name).ok()?, None, meta)) + } + + Some(ret) + } else { + None + } + }) + } + + fn find_meta(&self, path: &PathSegments) -> Option { + self.find_node(path).map(|n| n.metadata()) + } + + fn as_directory_tree( + &self, + _strictness: Strictness, + ) -> Result, anyhow::Error> { + let mut children = BTreeMap::new(); + + for (key, value) in self.nodes.iter() { + children.insert(PathSegment::from_str(key)?, value.as_dir_entry()?); + } + + let dir = v3::write::Directory::new( + children, + v3::Timestamps { + modified: self.modified, + }, + ); + + Ok(dir) + } + + fn as_dir_entry(&self) -> anyhow::Result> { + Ok(v3::write::DirEntry::Dir( + self.as_directory_tree(Strictness::default())?, + )) + } +} + +impl AbstractVolume for MemoryVolume { + fn read_file( + &self, + path: &PathSegments, + ) -> Option<(shared_buffer::OwnedBuffer, Option<[u8; 32]>)> { + self.node.read_file(path).map(|c| (c, None)) + } + + fn read_dir( + &self, + path: &PathSegments, + ) -> Option, Metadata)>> { + self.node.read_dir(path) + } + + fn metadata(&self, path: &PathSegments) -> Option { + self.node.find_meta(path) + } +} + +impl WasmerPackageVolume for MemoryVolume { + fn as_directory_tree( + &self, + strictness: Strictness, + ) -> Result, anyhow::Error> { + let res = self.node.as_directory_tree(strictness); + res + } +} + +#[cfg(test)] +mod tests { + use sha2::{Digest, Sha256}; + use v3::{ + write::Writer, Checksum, ChecksumAlgorithm, Index, IndexEntry, Signature, + SignatureAlgorithm, Span, Tag, Timestamps, + }; + use webc::metadata::Manifest; + + use super::*; + + fn sha256(data: impl AsRef<[u8]>) -> [u8; 32] { + let mut state = Sha256::default(); + state.update(data.as_ref()); + state.finalize().into() + } + + #[test] + fn volume_metadata() -> anyhow::Result<()> { + let file_modified = SystemTime::now(); + let file_data = String::from("Hello, world!").as_bytes().to_vec(); + let file_data_len = file_data.len(); + + let file = MemoryFile { + modified: file_modified, + data: file_data, + }; + + let mut nodes = BTreeMap::new(); + nodes.insert(String::from("hello.txt"), MemoryNode::File(file)); + + let dir_modified = SystemTime::now(); + let dir = MemoryDir { + modified: dir_modified, + nodes, + }; + + let volume = MemoryVolume { node: dir }; + + let file_metadata = volume.metadata(&PathSegments::from_str("hello.txt")?); + assert!(file_metadata.is_some()); + + let file_metadata = file_metadata.unwrap(); + assert!(file_metadata.is_file()); + + let (length, timestamps) = match file_metadata { + Metadata::File { length, timestamps } => (length, timestamps), + _ => unreachable!(), + }; + + assert_eq!( + timestamps.unwrap().modified(), + file_modified.duration_since(UNIX_EPOCH)?.as_nanos() as u64 + ); + + assert_eq!(length, file_data_len); + + let dir_metadata = volume.metadata(&PathSegments::from_str("/")?); + assert!(dir_metadata.is_some()); + + let dir_metadata = dir_metadata.unwrap(); + assert!(dir_metadata.is_dir()); + + let timestamps = match dir_metadata { + Metadata::Dir { timestamps } => timestamps, + _ => unreachable!(), + }; + + assert_eq!( + timestamps.unwrap().modified(), + dir_modified.duration_since(UNIX_EPOCH)?.as_nanos() as u64 + ); + + Ok(()) + } + + #[test] + fn create_webc_file_from_memory() -> Result<(), Box> { + let manifest = Manifest::default(); + + let mut writer = Writer::new(ChecksumAlgorithm::Sha256) + .write_manifest(&manifest)? + .write_atoms(BTreeMap::new())?; + + let file_contents = "Hello, World!"; + let file = MemoryFile { + modified: SystemTime::UNIX_EPOCH, + data: file_contents.as_bytes().to_vec(), + }; + let mut nodes = BTreeMap::new(); + nodes.insert(String::from("a"), MemoryNode::File(file)); + + let dir_modified = std::time::SystemTime::UNIX_EPOCH; + let dir = MemoryDir { + modified: dir_modified, + nodes, + }; + + let volume = MemoryVolume { node: dir }; + + writer.write_volume( + "first", + dbg!(WasmerPackageVolume::as_directory_tree( + &volume, + Strictness::Strict, + )?), + )?; + + let webc = writer.finish(SignatureAlgorithm::None)?; + + let mut data = vec![]; + ciborium::into_writer(&manifest, &mut data).unwrap(); + let manifest_hash: [u8; 32] = sha2::Sha256::digest(data).into(); + let manifest_section = bytes! { + Tag::Manifest, + manifest_hash, + 1_u64.to_le_bytes(), + [0xa0], + }; + + let empty_hash: [u8; 32] = sha2::Sha256::new().finalize().into(); + + let atoms_header_and_data = bytes! { + // header section + 65_u64.to_le_bytes(), + Tag::Directory, + 56_u64.to_le_bytes(), + Timestamps::default(), + empty_hash, + // data section (empty) + 0_u64.to_le_bytes(), + }; + + let atoms_hash: [u8; 32] = sha2::Sha256::digest(&atoms_header_and_data).into(); + let atoms_section = bytes! { + Tag::Atoms, + atoms_hash, + 81_u64.to_le_bytes(), + atoms_header_and_data, + }; + + let a_hash: [u8; 32] = sha2::Sha256::digest(file_contents).into(); + let dir_hash: [u8; 32] = sha2::Sha256::digest(a_hash).into(); + let volume_header_and_data = bytes! { + // ==== Name ==== + 5_u64.to_le_bytes(), + "first", + // ==== Header Section ==== + 187_u64.to_le_bytes(), + // ---- root directory ---- + Tag::Directory, + 105_u64.to_le_bytes(), + Timestamps::default(), + dir_hash, + // first entry + 114_u64.to_le_bytes(), + a_hash, + 1_u64.to_le_bytes(), + "a", + + // ---- first item ---- + Tag::File, + 0_u64.to_le_bytes(), + 13_u64.to_le_bytes(), + sha256("Hello, World!"), + Timestamps::default(), + + // ==== Data Section ==== + 13_u64.to_le_bytes(), + file_contents, + }; + let volume_hash: [u8; 32] = sha2::Sha256::digest(&volume_header_and_data).into(); + let first_volume_section = bytes! { + Tag::Volume, + volume_hash, + 229_u64.to_le_bytes(), + volume_header_and_data, + }; + + let index = Index::new( + IndexEntry::new( + Span::new(437, 42), + Checksum::sha256(sha256(&manifest_section[41..])), + ), + IndexEntry::new( + Span::new(479, 122), + Checksum::sha256(sha256(&atoms_section[41..])), + ), + [( + "first".to_string(), + IndexEntry::new( + Span::new(601, 270), + Checksum::sha256(sha256(&first_volume_section[41..])), + ), + )] + .into_iter() + .collect(), + Signature::none(), + ); + + let mut serialized_index = vec![]; + ciborium::into_writer(&index, &mut serialized_index).unwrap(); + let index_section = bytes! { + Tag::Index, + 420_u64.to_le_bytes(), + serialized_index, + // padding bytes to compensate for an unknown index length + // NOTE: THIS VALUE IS COMPLETELY RANDOM AND YOU SHOULD GUESS WHAT VALUE + // WILL WORK. + [0_u8; 75], + }; + + assert_bytes_eq!( + &webc, + bytes! { + webc::MAGIC, + webc::Version::V3, + index_section, + manifest_section, + atoms_section, + first_volume_section, + } + ); + + // make sure the index is accurate + assert_bytes_eq!(&webc[index.manifest.span], manifest_section); + assert_bytes_eq!(&webc[index.atoms.span], atoms_section); + assert_bytes_eq!(&webc[index.volumes["first"].span], first_volume_section); + + Ok(()) + } +} diff --git a/lib/package/src/package/volume/mod.rs b/lib/package/src/package/volume/mod.rs new file mode 100644 index 00000000000..cab61986082 --- /dev/null +++ b/lib/package/src/package/volume/mod.rs @@ -0,0 +1,39 @@ +pub(crate) mod fs; +pub(crate) mod in_memory; + +use std::{fmt::Debug, sync::Arc}; + +use anyhow::Error; + +use webc::{v3::write::Directory, AbstractVolume}; + +use super::Strictness; + +pub trait IntoSuper { + fn into_super(self: Arc) -> Arc; +} + +impl + IntoSuper for T +{ + fn into_super(self: Arc) -> Arc { + self + } +} + +/// An abstraction over concrete volumes implementation to be used in a Wasmer Package. +pub trait WasmerPackageVolume: + AbstractVolume + + Send + + Sync + + 'static + + Debug + + IntoSuper +{ + fn as_volume(self: Arc) -> Arc { + self.into_super() + } + + /// Serialize the volume as a [`webc::v3::write::Directory`]. + fn as_directory_tree(&self, strictness: Strictness) -> Result, Error>; +} diff --git a/lib/package/src/utils.rs b/lib/package/src/utils.rs new file mode 100644 index 00000000000..a7163691311 --- /dev/null +++ b/lib/package/src/utils.rs @@ -0,0 +1,111 @@ +use bytes::{Buf, Bytes}; +use std::{ + fs::File, + io::{BufRead, BufReader, Read, Seek}, + path::Path, +}; +use webc::{Container, ContainerError, Version}; + +use crate::package::{Package, WasmerPackageError}; + +/// Check if something looks like a `*.tar.gz` file. +fn is_tarball(mut file: impl Read + Seek) -> bool { + /// Magic bytes for a `*.tar.gz` file according to + /// [Wikipedia](https://en.wikipedia.org/wiki/List_of_file_signatures). + const TAR_GZ_MAGIC_BYTES: [u8; 2] = [0x1F, 0x8B]; + + let mut buffer = [0_u8; 2]; + let result = match file.read_exact(&mut buffer) { + Ok(_) => buffer == TAR_GZ_MAGIC_BYTES, + Err(_) => false, + }; + + let _ = file.rewind(); + + result +} + +pub fn from_disk(path: impl AsRef) -> Result { + let path = path.as_ref(); + + if path.is_dir() { + return parse_dir(path); + } + + let mut f = File::open(path).map_err(|error| ContainerError::Open { + error, + path: path.to_path_buf(), + })?; + + if is_tarball(&mut f) { + return parse_tarball(BufReader::new(f)); + } + + match webc::detect(&mut f) { + Ok(Version::V1) => parse_v1_mmap(f).map_err(Into::into), + Ok(Version::V2) => parse_v2_mmap(f).map_err(Into::into), + Ok(Version::V3) => parse_v3_mmap(f).map_err(Into::into), + Ok(other) => { + // fall back to the allocating generic version + let mut buffer = Vec::new(); + f.rewind() + .and_then(|_| f.read_to_end(&mut buffer)) + .map_err(|error| ContainerError::Read { + path: path.to_path_buf(), + error, + })?; + + Container::from_bytes_and_version(buffer.into(), other).map_err(Into::into) + } + Err(e) => Err(ContainerError::Detect(e).into()), + } +} + +pub fn from_bytes(bytes: impl Into) -> Result { + let bytes: Bytes = bytes.into(); + + if is_tarball(std::io::Cursor::new(&bytes)) { + return parse_tarball(bytes.reader()); + } + + let version = webc::detect(bytes.as_ref())?; + Container::from_bytes_and_version(bytes, version).map_err(Into::into) +} + +#[allow(clippy::result_large_err)] +fn parse_tarball(reader: impl BufRead) -> Result { + let pkg = Package::from_tarball(reader)?; + Ok(Container::new(pkg)) +} + +#[allow(clippy::result_large_err)] +fn parse_dir(path: &Path) -> Result { + let wasmer_toml = path.join("wasmer.toml"); + let pkg = Package::from_manifest(wasmer_toml)?; + Ok(Container::new(pkg)) +} + +#[allow(clippy::result_large_err)] +fn parse_v1_mmap(f: File) -> Result { + // We need to explicitly use WebcMmap to get a memory-mapped + // parser + let options = webc::v1::ParseOptions::default(); + let webc = webc::v1::WebCMmap::from_file(f, &options)?; + Ok(Container::new(webc)) +} + +#[allow(clippy::result_large_err)] +fn parse_v2_mmap(f: File) -> Result { + // Note: OwnedReader::from_file() will automatically try to + // use a memory-mapped file when possible. + let webc = webc::v2::read::OwnedReader::from_file(f)?; + Ok(Container::new(webc)) +} + +#[allow(clippy::result_large_err)] +fn parse_v3_mmap(f: File) -> Result { + // Note: OwnedReader::from_file() will automatically try to + // use a memory-mapped file when possible. + let webc = webc::v3::read::OwnedReader::from_file(f)?; + Ok(Container::new(webc)) +} diff --git a/lib/virtual-fs/Cargo.toml b/lib/virtual-fs/Cargo.toml index 9c49c4ccdd6..beb5a129d6f 100644 --- a/lib/virtual-fs/Cargo.toml +++ b/lib/virtual-fs/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] +wasmer-package.workspace = true dashmap.workspace = true dunce = "1.0.4" anyhow = { version = "1.0.66", optional = true } diff --git a/lib/virtual-fs/src/webc_volume_fs.rs b/lib/virtual-fs/src/webc_volume_fs.rs index 745a0570e39..08ac62ab685 100644 --- a/lib/virtual-fs/src/webc_volume_fs.rs +++ b/lib/virtual-fs/src/webc_volume_fs.rs @@ -10,8 +10,8 @@ use std::{ use futures::future::BoxFuture; use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite}; use webc::{ - compat::{Container, SharedBytes, Volume}, - PathSegmentError, PathSegments, ToPathSegments, + compat::SharedBytes, Container, Metadata as WebcMetadata, PathSegmentError, PathSegments, + ToPathSegments, Volume, }; use crate::{ @@ -308,9 +308,9 @@ fn get_modified(timestamps: Option) -> u64 { timestamps.map(|t| t.modified()).unwrap_or(1) } -fn compat_meta(meta: webc::compat::Metadata) -> Metadata { +fn compat_meta(meta: WebcMetadata) -> Metadata { match meta { - webc::compat::Metadata::Dir { timestamps } => Metadata { + WebcMetadata::Dir { timestamps } => Metadata { ft: FileType { dir: true, ..Default::default() @@ -318,7 +318,7 @@ fn compat_meta(meta: webc::compat::Metadata) -> Metadata { modified: get_modified(timestamps), ..Default::default() }, - webc::compat::Metadata::File { + WebcMetadata::File { length, timestamps, .. } => Metadata { ft: FileType { @@ -355,6 +355,7 @@ mod tests { use crate::DirEntry; use std::convert::TryFrom; use tokio::io::AsyncReadExt; + use wasmer_package::utils::from_bytes; const PYTHON_WEBC: &[u8] = include_bytes!("../../c-api/examples/assets/python-0.1.0.wasmer"); @@ -418,7 +419,7 @@ mod tests { #[test] fn mount_all_volumes_in_python() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let fs = WebcVolumeFileSystem::mount_all(&container); @@ -429,7 +430,7 @@ mod tests { #[test] fn read_dir() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let volumes = container.volumes(); let volume = volumes["atom"].clone(); @@ -501,7 +502,7 @@ mod tests { #[test] fn metadata() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let volumes = container.volumes(); let volume = volumes["atom"].clone(); @@ -553,7 +554,7 @@ mod tests { #[tokio::test] async fn file_opener() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let volumes = container.volumes(); let volume = volumes["atom"].clone(); @@ -596,7 +597,7 @@ mod tests { #[test] fn remove_dir_is_not_allowed() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let volumes = container.volumes(); let volume = volumes["atom"].clone(); @@ -618,7 +619,7 @@ mod tests { #[test] fn remove_file_is_not_allowed() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let volumes = container.volumes(); let volume = volumes["atom"].clone(); @@ -640,7 +641,7 @@ mod tests { #[test] fn create_dir_is_not_allowed() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let volumes = container.volumes(); let volume = volumes["atom"].clone(); @@ -662,7 +663,7 @@ mod tests { #[tokio::test] async fn rename_is_not_allowed() { - let container = Container::from_bytes(PYTHON_WEBC).unwrap(); + let container = from_bytes(PYTHON_WEBC).unwrap(); let volumes = container.volumes(); let volume = volumes["atom"].clone(); diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index f318344c2a4..1773054d3c4 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -13,6 +13,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] +wasmer-package.workspace = true wasmer-wasix-types = { path = "../wasi-types", version = "0.29.0", features = [ "enable-serde" ] } wasmer-types = { path = "../types", version = "=5.0.0-rc.1", default-features = false } wasmer = { path = "../api", version = "=5.0.0-rc.1", default-features = false, features = ["wat", "js-serializable-module"] } diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 051b9268c6a..9d94a64f09b 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -6,7 +6,9 @@ use once_cell::sync::OnceCell; use sha2::Digest; use virtual_fs::FileSystem; use wasmer_config::package::{PackageHash, PackageId, PackageSource}; -use webc::{compat::SharedBytes, Container}; +use wasmer_package::package::Package; +use webc::compat::SharedBytes; +use webc::Container; use crate::{ runners::MappedDirectory, @@ -96,7 +98,7 @@ impl BinaryPackage { let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash)); let manifest_path = dir.join("wasmer.toml"); - let webc = webc::wasmer_package::Package::from_manifest(&manifest_path)?; + let webc = Package::from_manifest(&manifest_path)?; let container = Container::from(webc); let manifest = container.manifest(); @@ -250,6 +252,7 @@ mod tests { use sha2::Digest; use tempfile::TempDir; use virtual_fs::AsyncReadExt; + use wasmer_package::utils::from_disk; use crate::{ runtime::{package_loader::BuiltinPackageLoader, task_manager::VirtualTaskManager}, @@ -297,12 +300,12 @@ mod tests { .with_shared_http_client(runtime.http_client().unwrap().clone()), ); - let pkg = webc::wasmer_package::Package::from_manifest(&manifest).unwrap(); + let pkg = Package::from_manifest(&manifest).unwrap(); let data = pkg.serialize().unwrap(); let webc_path = temp.path().join("package.webc"); std::fs::write(&webc_path, data).unwrap(); - let pkg = BinaryPackage::from_webc(&Container::from_disk(&webc_path).unwrap(), &runtime) + let pkg = BinaryPackage::from_webc(&from_disk(&webc_path).unwrap(), &runtime) .await .unwrap(); @@ -347,9 +350,7 @@ mod tests { let atom_path = temp.path().join("foo.wasm"); std::fs::write(&atom_path, b"").unwrap(); - let webc: Container = webc::wasmer_package::Package::from_manifest(&manifest) - .unwrap() - .into(); + let webc: Container = Package::from_manifest(&manifest).unwrap().into(); let tasks = task_manager(); let mut runtime = PluggableRuntime::new(tasks); diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index fa90ffd8d7e..88ab7fb0d14 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -10,8 +10,8 @@ use std::{ use anyhow::Context; use virtual_fs::{AsyncReadExt, FileSystem}; use wasmer::FunctionEnvMut; +use wasmer_package::utils::from_bytes; use wasmer_wasix_types::wasi::Errno; -use webc::Container; mod binary_package; mod exec; @@ -212,7 +212,7 @@ async fn load_executable_from_filesystem( let bytes: bytes::Bytes = data.into(); - if let Ok(container) = Container::from_bytes(bytes.clone()) { + if let Ok(container) = from_bytes(bytes.clone()) { let pkg = BinaryPackage::from_webc(&container, rt) .await .context("Unable to load the package")?; diff --git a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs index 20e4d26a769..ad861477911 100644 --- a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs +++ b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs @@ -8,8 +8,8 @@ use crate::{ }; use virtual_fs::{AsyncReadExt, FileSystem}; use wasmer::{FunctionEnvMut, Store}; +use wasmer_package::utils::from_bytes; use wasmer_wasix_types::wasi::Errno; -use webc::Container; use crate::{ bin_factory::{spawn_exec, BinaryPackage}, @@ -97,7 +97,7 @@ impl CmdWasmer { let bytes: bytes::Bytes = data.into(); - if let Ok(container) = Container::from_bytes(bytes.clone()) { + if let Ok(container) = from_bytes(bytes.clone()) { let pkg = BinaryPackage::from_webc(&container, &*self.runtime) .await .unwrap(); diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index f33005d024e..c743029736a 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -513,6 +513,8 @@ mod tests { async fn test_volume_mount_with_webcs() { use std::sync::Arc; + use wasmer_package::utils::from_bytes; + let root_fs = virtual_fs::RootFileSystemBuilder::new().build(); let tokrt = tokio::runtime::Handle::current(); @@ -536,7 +538,7 @@ mod tests { let webc_path = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("../../tests/integration/cli/tests/webc/wasmer-tests--volume-static-webserver@0.1.0.webc"); let webc_data = std::fs::read(webc_path).unwrap(); - let container = webc::Container::from_bytes(webc_data).unwrap(); + let container = from_bytes(webc_data).unwrap(); let binpkg = crate::bin_factory::BinaryPackage::from_webc(&container, &rt) .await diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 85ff26b548c..7bdf41b3a6f 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -385,7 +385,7 @@ mod tests { use tempfile::TempDir; use virtual_fs::{DirEntry, FileType, Metadata, WebcVolumeFileSystem}; - use webc::Container; + use wasmer_package::utils::from_bytes; use super::*; @@ -462,7 +462,7 @@ mod tests { guest: "/home".to_string(), host: sub_dir, })]; - let container = Container::from_bytes(PYTHON).unwrap(); + let container = from_bytes(PYTHON).unwrap(); let webc_fs = WebcVolumeFileSystem::mount_all(&container); let root_fs = RootFileSystemBuilder::default().build(); diff --git a/lib/wasix/src/runtime/package_loader/builtin_loader.rs b/lib/wasix/src/runtime/package_loader/builtin_loader.rs index 5c7e7125910..81363068058 100644 --- a/lib/wasix/src/runtime/package_loader/builtin_loader.rs +++ b/lib/wasix/src/runtime/package_loader/builtin_loader.rs @@ -11,10 +11,12 @@ use bytes::Bytes; use http::{HeaderMap, Method}; use tempfile::NamedTempFile; use url::Url; -use webc::{ - compat::{Container, ContainerError}, - DetectError, +use wasmer_package::{ + package::WasmerPackageError, + utils::{from_bytes, from_disk}, }; +use webc::DetectError; +use webc::{Container, ContainerError}; use crate::{ bin_factory::BinaryPackage, @@ -369,7 +371,7 @@ impl PackageLoader for BuiltinPackageLoader { // The sad path - looks like we don't have a filesystem cache so we'll // need to keep the whole thing in memory. - let container = crate::spawn_blocking(move || Container::from_bytes(bytes)).await??; + let container = crate::spawn_blocking(move || from_bytes(bytes)).await??; // We still want to cache it in memory, of course self.in_memory.save(&container, summary.dist.webc_sha256); Ok(container) @@ -487,18 +489,16 @@ impl FileSystemCache { let container = crate::spawn_blocking({ let path = path.clone(); - move || Container::from_disk(path) + move || from_disk(path) }) .await?; match container { Ok(c) => Ok(Some(c)), - Err(ContainerError::Open { error, .. }) - | Err(ContainerError::Read { error, .. }) - | Err(ContainerError::Detect(DetectError::Io(error))) - if error.kind() == ErrorKind::NotFound => - { - Ok(None) - } + Err(WasmerPackageError::ContainerError(ContainerError::Open { error, .. })) + | Err(WasmerPackageError::ContainerError(ContainerError::Read { error, .. })) + | Err(WasmerPackageError::ContainerError(ContainerError::Detect(DetectError::Io( + error, + )))) if error.kind() == ErrorKind::NotFound => Ok(None), Err(e) => { let msg = format!("Unable to read \"{}\"", path.display()); Err(Error::new(e).context(msg)) diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 2f46600050c..70be668c6cb 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -11,10 +11,8 @@ use once_cell::sync::OnceCell; use petgraph::visit::EdgeRef; use virtual_fs::{FileSystem, OverlayFileSystem, UnionFileSystem, WebcVolumeFileSystem}; use wasmer_config::package::PackageId; -use webc::{ - compat::{Container, Volume}, - metadata::annotations::Atom as AtomAnnotation, -}; +use webc::metadata::annotations::Atom as AtomAnnotation; +use webc::{Container, Volume}; use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, diff --git a/lib/wasix/src/runtime/package_loader/types.rs b/lib/wasix/src/runtime/package_loader/types.rs index 3eeb1cbf701..9488f62b0a0 100644 --- a/lib/wasix/src/runtime/package_loader/types.rs +++ b/lib/wasix/src/runtime/package_loader/types.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, ops::Deref}; use anyhow::Error; -use webc::compat::Container; +use webc::Container; use crate::{ bin_factory::BinaryPackage, diff --git a/lib/wasix/src/runtime/package_loader/unsupported.rs b/lib/wasix/src/runtime/package_loader/unsupported.rs index c0c245ff216..a16477ddabe 100644 --- a/lib/wasix/src/runtime/package_loader/unsupported.rs +++ b/lib/wasix/src/runtime/package_loader/unsupported.rs @@ -1,5 +1,5 @@ use anyhow::Error; -use webc::compat::Container; +use webc::Container; use crate::{ bin_factory::BinaryPackage, diff --git a/lib/wasix/src/runtime/resolver/filesystem_source.rs b/lib/wasix/src/runtime/resolver/filesystem_source.rs index 1927f8af02a..0b977e37e6c 100644 --- a/lib/wasix/src/runtime/resolver/filesystem_source.rs +++ b/lib/wasix/src/runtime/resolver/filesystem_source.rs @@ -1,6 +1,6 @@ use anyhow::Context; use wasmer_config::package::{PackageHash, PackageId, PackageSource}; -use webc::compat::Container; +use wasmer_package::utils::from_disk; use crate::runtime::resolver::{ DistributionInfo, PackageInfo, PackageSummary, QueryError, Source, WebcHash, @@ -14,7 +14,7 @@ impl FileSystemSource { async fn load_path(path: &std::path::Path) -> Result, anyhow::Error> { let webc_sha256 = crate::block_in_place(|| WebcHash::for_file(path)) .with_context(|| format!("Unable to hash \"{}\"", path.display()))?; - let container = crate::block_in_place(|| Container::from_disk(path)) + let container = crate::block_in_place(|| from_disk(path)) .with_context(|| format!("Unable to parse \"{}\"", path.display()))?; let url = crate::runtime::resolver::utils::url_from_file_path(path) diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/wasix/src/runtime/resolver/inputs.rs index 8b5114b8126..2e54f7252ce 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/wasix/src/runtime/resolver/inputs.rs @@ -10,10 +10,8 @@ use semver::VersionReq; use sha2::{Digest, Sha256}; use url::Url; use wasmer_config::package::{NamedPackageId, PackageHash, PackageId, PackageSource}; -use webc::{ - metadata::{annotations::Wapm as WapmAnnotations, Manifest, UrlOrManifest}, - Container, -}; +use wasmer_package::utils::from_disk; +use webc::metadata::{annotations::Wapm as WapmAnnotations, Manifest, UrlOrManifest}; /// A dependency constraint. #[derive(Debug, Clone, PartialEq, Eq)] @@ -53,7 +51,7 @@ impl PackageSummary { pub fn from_webc_file(path: impl AsRef) -> Result { let path = path.as_ref().canonicalize()?; - let container = Container::from_disk(&path)?; + let container = from_disk(&path)?; let webc_sha256 = WebcHash::for_file(&path)?; let url = crate::runtime::resolver::utils::url_from_file_path(&path).ok_or_else(|| { anyhow::anyhow!("Unable to turn \"{}\" into a file:// URL", path.display()) diff --git a/lib/wasix/src/runtime/resolver/web_source.rs b/lib/wasix/src/runtime/resolver/web_source.rs index 75371bef1e4..110893fe1c7 100644 --- a/lib/wasix/src/runtime/resolver/web_source.rs +++ b/lib/wasix/src/runtime/resolver/web_source.rs @@ -12,7 +12,7 @@ use sha2::{Digest, Sha256}; use tempfile::NamedTempFile; use url::Url; use wasmer_config::package::{PackageHash, PackageId, PackageSource}; -use webc::compat::Container; +use wasmer_package::utils::from_disk; use crate::{ http::{HttpClient, HttpRequest}, @@ -236,7 +236,7 @@ impl WebSource { // Note: We want to use Container::from_disk() rather than the bytes // our HTTP client gave us because then we can use memory-mapped files - let container = crate::block_in_place(|| Container::from_disk(&local_path)) + let container = crate::block_in_place(|| from_disk(&local_path)) .with_context(|| format!("Unable to load \"{}\"", local_path.display()))?; let id = PackageInfo::package_id_from_manifest(container.manifest())? diff --git a/lib/wasix/tests/runners.rs b/lib/wasix/tests/runners.rs index 467091ad494..ae46aa2c61e 100644 --- a/lib/wasix/tests/runners.rs +++ b/lib/wasix/tests/runners.rs @@ -21,10 +21,10 @@ use wasmer_wasix::{ }, PluggableRuntime, Runtime, }; -use webc::Container; mod wasi { use virtual_fs::{AsyncReadExt, AsyncSeekExt}; + use wasmer_package::utils::from_bytes; use wasmer_wasix::{bin_factory::BinaryPackage, runners::wasi::WasiRunner, WasiError}; use super::*; @@ -32,7 +32,7 @@ mod wasi { #[tokio::test] async fn can_run_wat2wasm() { let webc = download_cached("https://wasmer.io/wasmer/wabt@1.0.37").await; - let container = Container::from_bytes(webc).unwrap(); + let container = from_bytes(webc).unwrap(); let command = &container.manifest().commands["wat2wasm"]; assert!(WasiRunner::can_run_command(command).unwrap()); @@ -41,7 +41,7 @@ mod wasi { #[tokio::test(flavor = "multi_thread")] async fn wat2wasm() { let webc = download_cached("https://wasmer.io/wasmer/wabt@1.0.37").await; - let container = Container::from_bytes(webc).unwrap(); + let container = from_bytes(webc).unwrap(); let (rt, tasks) = runtime(); let pkg = BinaryPackage::from_webc(&container, &rt).await.unwrap(); let mut stdout = virtual_fs::ArcFile::new(Box::::default()); @@ -72,7 +72,7 @@ mod wasi { async fn python() { let webc = download_cached("https://wasmer.io/python/python@0.1.0").await; let (rt, tasks) = runtime(); - let container = Container::from_bytes(webc).unwrap(); + let container = from_bytes(webc).unwrap(); let pkg = BinaryPackage::from_webc(&container, &rt).await.unwrap(); let handle = std::thread::spawn(move || { @@ -113,7 +113,7 @@ mod wcgi { #[tokio::test] async fn can_run_staticserver() { let webc = download_cached("https://wasmer.io/Michael-F-Bryan/staticserver@1.0.3").await; - let container = Container::from_bytes(webc).unwrap(); + let container = from_bytes(webc).unwrap(); let entrypoint = container.manifest().entrypoint.as_ref().unwrap(); assert!(WcgiRunner::can_run_command(&container.manifest().commands[entrypoint]).unwrap()); @@ -123,7 +123,7 @@ mod wcgi { async fn staticserver() { let webc = download_cached("https://wasmer.io/Michael-F-Bryan/staticserver@1.0.3").await; let (rt, tasks) = runtime(); - let container = Container::from_bytes(webc).unwrap(); + let container = from_bytes(webc).unwrap(); let mut runner = WcgiRunner::new(NoOpWcgiCallbacks); let port = rand::thread_rng().gen_range(10000_u16..65535_u16); let (cb, started) = callbacks(Handle::current()); diff --git a/tests/old-tar-gz/coreutils-1.0.11.tar.gz b/tests/old-tar-gz/coreutils-1.0.11.tar.gz new file mode 100644 index 00000000000..a30ac16849e Binary files /dev/null and b/tests/old-tar-gz/coreutils-1.0.11.tar.gz differ diff --git a/tests/old-tar-gz/cowsay-0.3.0.tar.gz b/tests/old-tar-gz/cowsay-0.3.0.tar.gz new file mode 100644 index 00000000000..13f0f75e96f Binary files /dev/null and b/tests/old-tar-gz/cowsay-0.3.0.tar.gz differ