From f9959d34f1422fd8ad1d86f5873f6940b776b7f4 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 1 Aug 2024 22:55:37 +0200 Subject: [PATCH 01/53] deps: Upgrade some dependencies + lift to workspace root Also ignores a WASIX test from the workspace. --- Cargo.lock | 108 ++++++++++++++----------------- Cargo.toml | 9 ++- lib/backend-api/Cargo.toml | 4 +- lib/c-api/Cargo.toml | 2 +- lib/cli/Cargo.toml | 4 +- lib/compiler/Cargo.toml | 2 +- lib/registry/Cargo.toml | 2 +- lib/virtual-fs/Cargo.toml | 4 +- lib/virtual-io/Cargo.toml | 6 +- lib/virtual-net/Cargo.toml | 8 +-- lib/wasix/Cargo.toml | 4 +- tests/integration/cli/Cargo.toml | 2 +- tests/lib/wast/Cargo.toml | 2 +- tests/wasmer-argus/Cargo.toml | 2 +- 14 files changed, 78 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 33610f2c0dd..12f864f2941 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,7 +216,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.38.1", + "tokio 1.39.2", ] [[package]] @@ -242,7 +242,7 @@ dependencies = [ "pin-project-lite", "rustls-native-certs", "rustls-pki-types", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-rustls 0.25.0", "tungstenite", ] @@ -2310,7 +2310,7 @@ dependencies = [ "itoa", "pin-project-lite", "smallvec 1.13.2", - "tokio 1.38.1", + "tokio 1.39.2", "want", ] @@ -2326,7 +2326,7 @@ dependencies = [ "hyper-util", "rustls 0.23.11", "rustls-pki-types", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-rustls 0.26.0", "tower-service", "webpki-roots", @@ -2343,7 +2343,7 @@ dependencies = [ "hyper", "hyper-util", "native-tls", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-native-tls", "tower-service", ] @@ -2358,7 +2358,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-tungstenite", "tungstenite", ] @@ -2376,8 +2376,8 @@ dependencies = [ "http-body", "hyper", "pin-project-lite", - "socket2 0.5.7", - "tokio 1.38.1", + "socket2", + "tokio 1.39.2", "tower", "tower-service", "tracing", @@ -3077,14 +3077,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi", "libc", "log 0.4.22", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3819,7 +3820,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.11", "thiserror", - "tokio 1.38.1", + "tokio 1.39.2", "tracing", ] @@ -3848,7 +3849,7 @@ checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" dependencies = [ "libc", "once_cell", - "socket2 0.5.7", + "socket2", "windows-sys 0.52.0", ] @@ -4091,7 +4092,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-native-tls", "tokio-rustls 0.26.0", "tokio-socks", @@ -4840,16 +4841,6 @@ dependencies = [ "managed", ] -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi 0.3.9", -] - [[package]] name = "socket2" version = "0.5.7" @@ -5226,20 +5217,19 @@ dependencies = [ [[package]] name = "tokio" -version = "1.38.1" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes 1.6.1", "libc", - "mio 0.8.11", - "num_cpus", + "mio 1.0.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.7", + "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5316,9 +5306,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -5332,7 +5322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.38.1", + "tokio 1.39.2", ] [[package]] @@ -5362,7 +5352,7 @@ checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls 0.22.4", "rustls-pki-types", - "tokio 1.38.1", + "tokio 1.39.2", ] [[package]] @@ -5373,7 +5363,7 @@ checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ "rustls 0.23.11", "rustls-pki-types", - "tokio 1.38.1", + "tokio 1.39.2", ] [[package]] @@ -5403,7 +5393,7 @@ dependencies = [ "either", "futures-util", "thiserror", - "tokio 1.38.1", + "tokio 1.39.2", ] [[package]] @@ -5414,7 +5404,7 @@ checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-util", ] @@ -5482,7 +5472,7 @@ dependencies = [ "rustls 0.22.4", "rustls-native-certs", "rustls-pki-types", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-rustls 0.25.0", "tungstenite", "webpki-roots", @@ -5531,7 +5521,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.38.1", + "tokio 1.39.2", ] [[package]] @@ -5613,7 +5603,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.38.1", + "tokio 1.39.2", "tower-layer", "tower-service", "tracing", @@ -6030,7 +6020,7 @@ dependencies = [ "slab", "tempfile", "thiserror", - "tokio 1.38.1", + "tokio 1.39.2", "tracing", "tracing-test", "typetag", @@ -6045,9 +6035,9 @@ dependencies = [ "bytes 1.6.1", "derivative", "futures 0.3.30", - "mio 0.8.11", + "mio 1.0.1", "serde", - "socket2 0.4.10", + "socket2", "thiserror", "tracing", ] @@ -6068,15 +6058,15 @@ dependencies = [ "hyper-tungstenite", "hyper-util", "libc", - "mio 0.8.11", + "mio 1.0.1", "pin-project-lite", "rkyv", "serde", "serial_test", "smoltcp", - "socket2 0.4.10", + "socket2", "thiserror", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-serde", "tokio-tungstenite", "tokio-util", @@ -6482,7 +6472,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "time 0.3.36", - "tokio 1.38.1", + "tokio 1.39.2", "tracing", "url", "uuid", @@ -6506,7 +6496,7 @@ dependencies = [ "serde", "serde_json", "shared-buffer", - "tokio 1.38.1", + "tokio 1.39.2", "tracing", "tracing-subscriber", "url", @@ -6544,7 +6534,7 @@ dependencies = [ "libc", "paste", "thiserror", - "tokio 1.38.1", + "tokio 1.39.2", "tracing", "tracing-subscriber", "typetag", @@ -6641,7 +6631,7 @@ dependencies = [ "libc", "log 0.4.22", "mac_address", - "mio 0.8.11", + "mio 1.0.1", "normpath", "object 0.32.2", "once_cell", @@ -6666,7 +6656,7 @@ dependencies = [ "time 0.1.45", "time 0.3.36", "tldextract", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-tungstenite", "toml 0.5.11", "tracing", @@ -6711,7 +6701,7 @@ dependencies = [ "lazy_static", "leb128", "libc", - "memmap2 0.5.10", + "memmap2 0.6.2", "more-asserts", "region", "rkyv", @@ -6933,7 +6923,7 @@ dependencies = [ "tar", "target-lexicon 0.12.15", "tempfile", - "tokio 1.38.1", + "tokio 1.39.2", "wasmer-registry", ] @@ -7023,7 +7013,7 @@ dependencies = [ "thiserror", "time 0.3.36", "tldextract", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-tungstenite", "toml 0.5.11", "tracing", @@ -7151,7 +7141,7 @@ dependencies = [ "terminal_size", "termios", "thiserror", - "tokio 1.38.1", + "tokio 1.39.2", "tokio-stream", "toml 0.8.15", "tower", @@ -7229,7 +7219,7 @@ dependencies = [ "serde", "tempfile", "thiserror", - "tokio 1.38.1", + "tokio 1.39.2", "virtual-fs", "wasmer", "wasmer-types", @@ -7258,7 +7248,7 @@ dependencies = [ "tempfile", "test-generator", "test-log", - "tokio 1.38.1", + "tokio 1.39.2", "tracing", "tracing-subscriber", "ureq", @@ -7381,7 +7371,7 @@ dependencies = [ "http 0.2.12", "schemars", "serde", - "tokio 1.38.1", + "tokio 1.39.2", "wasmparser 0.95.0", "wcgi 0.1.2", ] @@ -7394,7 +7384,7 @@ checksum = "1cfc134be64da186b33675e0154827ba8eb4eb30579c1142cebec88607cd2cf6" dependencies = [ "http 1.1.0", "serde", - "tokio 1.38.1", + "tokio 1.39.2", "wasmparser 0.95.0", "wcgi 0.2.0", ] diff --git a/Cargo.toml b/Cargo.toml index 324c5008164..548c7d20f9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ wasmer-middlewares = { version = "=4.3.5", path = "lib/middlewares", optional = # Third party dependencies cfg-if = "1.0" -tokio = { version = "1", features = [ +tokio = { version = "1.39", features = [ "rt", "rt-multi-thread", "macros", @@ -75,6 +75,9 @@ members = [ "tests/wasi-wast", "tests/wasmer-argus", ] +exclude = [ + "./lib/cli/tests/packages/axum" +] resolver = "2" [workspace.package] @@ -109,6 +112,10 @@ toml = {version = "0.5.9", features = ["preserve_order"]} indexmap = "2" serde_yaml = "0.9.34" libc = { version = "^0.2", default-features = false } +mio = "1" +# MIO 1.0 starts at tokio version 1.39, hence the minimum requirement. +tokio = { version = "1.39.0", default-features = false} +socket2 = "0.5.7" [build-dependencies] test-generator = { path = "tests/lib/test-generator" } diff --git a/lib/backend-api/Cargo.toml b/lib/backend-api/Cargo.toml index 8d4120c1a7b..2143e06dc92 100644 --- a/lib/backend-api/Cargo.toml +++ b/lib/backend-api/Cargo.toml @@ -23,7 +23,7 @@ webc.workspace = true anyhow = "1" serde = { version = "1", features = ["derive"] } time = { version = "0.3", features = ["formatting", "parsing"] } -tokio = { version = "1.23.0" } +tokio = { workspace = true } serde_json = "1" url = "2" futures = "0.3" @@ -41,5 +41,5 @@ features = ["js"] [dev-dependencies] base64 = "0.13.1" -tokio = { version = "1.3", features = ["macros", "rt"] } +tokio = { workspace = true, features = ["macros", "rt"] } uuid = { version = "1", features = ["v4"] } diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index b22a6c79801..b264889b022 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -42,7 +42,7 @@ libc.workspace = true thiserror = "1" typetag = { version = "0.1", optional = true } paste = "1.0" -tokio = { version = "1", features = [ "rt", "rt-multi-thread", "io-util", "sync", "macros"], default-features = false } +tokio = { workspace = true, features = [ "rt", "rt-multi-thread", "io-util", "sync", "macros"], default-features = false } tracing = { version = "0.1" } tracing-subscriber = { version = "0.3", features = [ "env-filter", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 18d3beecc9e..b382c9d3556 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -202,7 +202,7 @@ tracing-subscriber = { version = "0.3", features = [ "json", ] } async-trait = "0.1.68" -tokio = { version = "1.28.1", features = ["macros", "rt-multi-thread"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } once_cell = "1.17.1" indicatif = "0.17.5" opener = "0.6.1" @@ -222,7 +222,7 @@ comfy-table = "7.0.1" # Used by tuntap and connect futures-util = "0.3" -mio = { version = "0.8", optional = true } +mio = { workspace = true, optional = true } tokio-tungstenite = { version = "0.21.0", features = [ "rustls-tls-webpki-roots", "stream", diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index ce9d426d611..1cf88543467 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -25,7 +25,7 @@ smallvec = "1.6" xxhash-rust = { version = "0.8.10", features = ["xxh64"] } backtrace = "0.3" -memmap2 = "0.5" +memmap2 = "0.6" more-asserts = "0.2" lazy_static = "1.4" diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index 92fef924477..5b52e84e119 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -44,7 +44,7 @@ tempfile = "3.6.0" thiserror = "1.0.37" time = { version = "0.3.17", default-features = false, features = ["parsing", "std", "formatting"], optional = true } tldextract = "0.6.0" -tokio = {version = "1", features = ["rt-multi-thread"]} +tokio = { workspace = true, features = ["rt-multi-thread"] } tokio-tungstenite = {version = "0.21", features = ["rustls-tls-native-roots"]} toml.workspace = true tracing = "0.1.40" diff --git a/lib/virtual-fs/Cargo.toml b/lib/virtual-fs/Cargo.toml index f6db418f880..1a4d333cdf1 100644 --- a/lib/virtual-fs/Cargo.toml +++ b/lib/virtual-fs/Cargo.toml @@ -26,7 +26,7 @@ replace_with = "0.1.7" shared-buffer = { workspace = true } slab = { version = "0.4" } thiserror = "1" -tokio = { version = "1", features = ["io-util", "sync", "macros"], default-features = false } +tokio = { workspace = true, features = ["io-util", "sync", "macros"], default-features = false } tracing = { version = "0.1" } typetag = { version = "0.1", optional = true } webc = { workspace = true, optional = true, features = ["v1"] } @@ -42,7 +42,7 @@ getrandom = { version = "0.2", features = [ "js" ] } pretty_assertions = "1.3.0" tempfile = "3.6.0" tracing-test = "0.2.4" -tokio = { version = "1", features = ["io-util", "rt"], default-features = false } +tokio = { workspace = true, features = ["io-util", "rt"], default-features = false } [features] default = ["host-fs", "webc-fs", "static-fs"] diff --git a/lib/virtual-io/Cargo.toml b/lib/virtual-io/Cargo.toml index 4d25ba07536..bb9a02f31e8 100644 --- a/lib/virtual-io/Cargo.toml +++ b/lib/virtual-io/Cargo.toml @@ -14,11 +14,11 @@ thiserror = "1" bytes = "1.1" async-trait = { version = "^0.1" } tracing = "0.1" -mio = { version = "0.8", features = [ "os-poll" ], optional = true } -socket2 = { version = "0.4", optional = true } +mio = { workspace = true, features = [ "os-poll" ], optional = true } +socket2 = { workspace = true, optional = true } derivative = { version = "^2" } futures = { version = "0.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } [features] -sys = [ "mio", "socket2" ] \ No newline at end of file +sys = [ "mio", "socket2" ] diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index 2d7c3ce76b3..6f409b97faa 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -14,10 +14,10 @@ thiserror = "1" bytes = "1.1" async-trait = { version = "^0.1" } tracing = "0.1" -tokio = { version = "1", default-features = false, features = ["io-util"] } +tokio = { workspace = true, default-features = false, features = ["io-util"] } libc = { workspace = true, optional = true } -mio = { version = "0.8", optional = true } -socket2 = { version = "0.4", optional = true } +mio = { workspace = true, optional = true } +socket2 = { workspace = true, optional = true } derivative = { version = "^2" } virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } base64 = "0.21" @@ -41,7 +41,7 @@ default-features = false features = ["proto-ipv4", "std", "alloc"] [dev-dependencies] -tokio = { version = "1", default-features = false, features = [ "macros", "rt-multi-thread" ] } +tokio = { workspace = true, default-features = false, features = [ "macros", "rt-multi-thread" ] } tracing-test = { version = "0.2" } serial_test = "2.0.0" diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index ccd0447a264..b1a22636f8f 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -44,7 +44,7 @@ sha2 = { version = "0.10" } waker-fn = { version = "1.1" } cooked-waker = "^5" rand = "0.8" -tokio = { version = "1", features = [ +tokio = { workspace = true, features = [ "sync", "macros", "time", @@ -131,7 +131,7 @@ terminal_size = { version = "0.3.0" } [dev-dependencies] wasmer = { path = "../api", version = "=4.3.5", default-features = false, features = ["wat", "js-serializable-module"] } -tokio = { version = "1", features = [ "sync", "macros", "rt" ], default-features = false } +tokio = { workspace = true, features = [ "sync", "macros", "rt" ], default-features = false } pretty_assertions = "1.3.0" tracing-test = "0.2.4" wasm-bindgen-test = "0.3.0" diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index fee7a1f194a..e05bc519ad9 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -17,7 +17,7 @@ hex = "0.4.3" pretty_assertions = "1.3.0" object = "0.30.0" reqwest = { workspace = true, default-features = false, features = ["json", "blocking", "rustls-tls"] } -tokio = { version = "1", features = [ "rt", "rt-multi-thread", "macros" ] } +tokio = { workspace = true, features = [ "rt", "rt-multi-thread", "macros" ] } assert_cmd = "2.0.8" predicates = "2.1.5" once_cell = "1.17.1" diff --git a/tests/lib/wast/Cargo.toml b/tests/lib/wast/Cargo.toml index a4c9a32b9d8..2d73ca3b3f3 100644 --- a/tests/lib/wast/Cargo.toml +++ b/tests/lib/wast/Cargo.toml @@ -22,7 +22,7 @@ serde = "1" tempfile = "3.6.0" thiserror = "1.0" futures = "0.3" -tokio = { version = "1", features = [ +tokio = { workspace = true, features = [ "io-util", "rt", ], default-features = false } diff --git a/tests/wasmer-argus/Cargo.toml b/tests/wasmer-argus/Cargo.toml index 09a116fa553..5640c4601d2 100644 --- a/tests/wasmer-argus/Cargo.toml +++ b/tests/wasmer-argus/Cargo.toml @@ -19,7 +19,7 @@ cynic = "3.4.3" url = "2.5.0" futures = "0.3.30" tracing = "0.1.40" -tokio = { version = "1.36.0", features = ["rt-multi-thread", "sync", "time", "fs"] } +tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time", "fs"] } clap = {version = "4.4.11", features = ["derive", "string"]} tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } serde = { version = "1.0.197", features = ["derive"] } From 2335a82a2eb6cbdd5dfa6b87a5952d26a7f7d2be Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 8 Aug 2024 13:12:17 +0200 Subject: [PATCH 02/53] Partial fix for mio net refactor --- lib/virtual-net/src/host.rs | 67 +++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/lib/virtual-net/src/host.rs b/lib/virtual-net/src/host.rs index 0d14a1e2675..de9ca4dbca1 100644 --- a/lib/virtual-net/src/host.rs +++ b/lib/virtual-net/src/host.rs @@ -16,6 +16,9 @@ use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr}; use std::os::fd::AsRawFd; #[cfg(not(target_os = "windows"))] use std::os::fd::RawFd; +#[cfg(windows)] +use std::os::windows::io::AsRawSocket; + use std::sync::Arc; use std::task::Poll; use std::time::Duration; @@ -87,7 +90,21 @@ impl VirtualNetworking for LocalNetworking { _reuse_addr: bool, ) -> Result> { let socket = mio::net::UdpSocket::bind(addr).map_err(io_err_into_net_error)?; - socket2::SockRef::from(&socket).set_nonblocking(true).ok(); + + { + #[cfg(not(windows))] + let sockref = socket2::SockRef::from(&socket); + #[cfg(windows)] + let sockref = unsafe { + socket2::SockRef::from(&std::os::windows::io::BorrowedSocket::borrow_raw( + socket.stream.as_raw_socket(), + )) + .set_nonblocking(true) + .ok(); + }; + + sockref.set_nonblocking(true).ok(); + } #[allow(unused_mut)] let mut ret = LocalUdpSocket { @@ -118,6 +135,7 @@ impl VirtualNetworking for LocalNetworking { mut peer: SocketAddr, ) -> Result> { let stream = mio::net::TcpStream::connect(peer).map_err(io_err_into_net_error)?; + socket2::SockRef::from(&stream).set_nonblocking(true).ok(); if let Ok(p) = stream.peer_addr() { peer = p; @@ -333,6 +351,20 @@ impl LocalTcpStream { ret } + + fn sock_ref<'a>(&'a self) -> socket2::SockRef<'a> { + #[cfg(not(windows))] + let r = socket2::SockRef::from(&self.stream); + + #[cfg(windows)] + let r = unsafe { + socket2::SockRef::from(&std::os::windows::io::BorrowedSocket::borrow_raw( + self.stream.as_raw_socket(), + )) + }; + + r + } } impl VirtualTcpSocket for LocalTcpStream { @@ -363,16 +395,14 @@ impl VirtualTcpSocket for LocalTcpStream { } fn set_keepalive(&mut self, keepalive: bool) -> Result<()> { - socket2::SockRef::from(&self.stream) + self.sock_ref() .set_keepalive(true) .map_err(io_err_into_net_error)?; Ok(()) } fn keepalive(&self) -> Result { - let ret = socket2::SockRef::from(&self.stream) - .keepalive() - .map_err(io_err_into_net_error)?; + let ret = self.sock_ref().keepalive().map_err(io_err_into_net_error)?; Ok(ret) } @@ -444,16 +474,15 @@ impl VirtualTcpSocket for LocalTcpStream { impl VirtualConnectedSocket for LocalTcpStream { fn set_linger(&mut self, linger: Option) -> Result<()> { - socket2::SockRef::from(&self.stream) + self.sock_ref() .set_linger(linger) .map_err(io_err_into_net_error)?; Ok(()) } fn linger(&self) -> Result> { - socket2::SockRef::from(&self.stream) - .linger() - .map_err(io_err_into_net_error) + let sockref = self.sock_ref(); + sockref.linger().map_err(io_err_into_net_error) } fn try_send(&mut self, data: &[u8]) -> Result { @@ -654,6 +683,22 @@ pub struct LocalUdpSocket { backlog: VecDeque<(BytesMut, SocketAddr)>, } +impl LocalUdpSocket { + fn sock_ref<'a>(&'a self) -> socket2::SockRef<'a> { + #[cfg(not(windows))] + let r = socket2::SockRef::from(&self.socket); + + #[cfg(windows)] + let r = unsafe { + socket2::SockRef::from(&std::os::windows::io::BorrowedSocket::borrow_raw( + self.stream.as_raw_socket(), + )) + }; + + r + } +} + impl VirtualUdpSocket for LocalUdpSocket { fn set_broadcast(&mut self, broadcast: bool) -> Result<()> { self.socket @@ -702,13 +747,13 @@ impl VirtualUdpSocket for LocalUdpSocket { } fn join_multicast_v4(&mut self, multiaddr: Ipv4Addr, iface: Ipv4Addr) -> Result<()> { - socket2::SockRef::from(&self.socket) + self.sock_ref() .join_multicast_v4(&multiaddr, &iface) .map_err(io_err_into_net_error) } fn leave_multicast_v4(&mut self, multiaddr: Ipv4Addr, iface: Ipv4Addr) -> Result<()> { - socket2::SockRef::from(&self.socket) + self.sock_ref() .leave_multicast_v4(&multiaddr, &iface) .map_err(io_err_into_net_error) } From 1bb71f5c891264937fde26b3a718b0a8371bfebb Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 11:01:22 +0200 Subject: [PATCH 03/53] build: Lift pretty_assertions to workspace dependencies --- Cargo.toml | 1 + lib/cli/Cargo.toml | 2 +- lib/config/Cargo.toml | 2 +- lib/registry/Cargo.toml | 2 +- lib/virtual-fs/Cargo.toml | 2 +- lib/wasix/Cargo.toml | 2 +- tests/integration/cli/Cargo.toml | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bff17a4d1dd..17a5ea9a5dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,7 @@ toml = {version = "0.5.9", features = ["preserve_order"]} indexmap = "2" serde_yaml = "0.9.34" libc = { version = "^0.2", default-features = false } +pretty_assertions = "1.4.0" [build-dependencies] test-generator = { path = "tests/lib/test-generator" } diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index fbac043c74e..a71aa595e32 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -277,7 +277,7 @@ unix_mode = "0.1.3" [dev-dependencies] assert_cmd = "2.0.11" predicates = "3.0.3" -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true [target.'cfg(target_os = "windows")'.dependencies] colored = "2.0.0" diff --git a/lib/config/Cargo.toml b/lib/config/Cargo.toml index 6475abb9679..1f2d6b80044 100644 --- a/lib/config/Cargo.toml +++ b/lib/config/Cargo.toml @@ -29,6 +29,6 @@ hex = "0.4.3" ciborium = "0.2.2" [dev-dependencies] -pretty_assertions = "1.4.0" +pretty_assertions.workspace = true serde_json = "1.0.116" tempfile = "3.3.0" diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index bdf0b036a25..d6d90d05b9d 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -57,7 +57,7 @@ webc.workspace = true async-tungstenite = { version = "0.25.1", features = ["tokio-runtime", "tokio-rustls-native-certs"] } [dev-dependencies] -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true [package.metadata.docs.rs] features = ["build-package"] diff --git a/lib/virtual-fs/Cargo.toml b/lib/virtual-fs/Cargo.toml index 8b7b41a3cbe..b740223f236 100644 --- a/lib/virtual-fs/Cargo.toml +++ b/lib/virtual-fs/Cargo.toml @@ -40,7 +40,7 @@ getrandom = { version = "0.2" } getrandom = { version = "0.2", features = [ "js" ] } [dev-dependencies] -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true tempfile = "3.6.0" tracing-test = "0.2.4" tokio = { version = "1", features = ["io-util", "rt"], default-features = false } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 0e8b2fb289a..fe47676735b 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -132,7 +132,7 @@ terminal_size = { version = "0.3.0" } [dev-dependencies] wasmer = { path = "../api", version = "=4.3.7", default-features = false, features = ["wat", "js-serializable-module"] } tokio = { version = "1", features = [ "sync", "macros", "rt" ], default-features = false } -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true tracing-test = "0.2.4" wasm-bindgen-test = "0.3.0" diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index a29b9d0a376..4b051c920ee 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -14,7 +14,7 @@ serde = { version = "1.0.147", features = ["derive"] } insta = { version = "1.21.1", features = ["json"] } md5 = "0.7.0" hex = "0.4.3" -pretty_assertions = "1.3.0" +pretty_assertions.workspace = true object = "0.30.0" reqwest = { workspace = true, default-features = false, features = ["json", "blocking", "rustls-tls"] } tokio = { version = "1", features = [ "rt", "rt-multi-thread", "macros" ] } From 64b7559cf2d717d1f8eadcd6ff17e5d38ddde552 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 11:07:05 +0200 Subject: [PATCH 04/53] build: Add a new wasmer-package crate The crate will hold package related logic. --- Cargo.lock | 24 +++++++++++++++++++----- Cargo.toml | 1 + lib/package/Cargo.toml | 21 +++++++++++++++++++++ lib/package/src/lib.rs | 1 + 4 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 lib/package/Cargo.toml create mode 100644 lib/package/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3849b3ec7b3..f50134f7a3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4649,11 +4649,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -5000,14 +5001,15 @@ checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if 1.0.0", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6690,6 +6692,7 @@ dependencies = [ "wasmer-config 0.8.0", "wasmer-emscripten", "wasmer-object", + "wasmer-package", "wasmer-registry", "wasmer-types", "wasmer-vm", @@ -6987,6 +6990,17 @@ dependencies = [ "wasmer-types", ] +[[package]] +name = "wasmer-package" +version = "0.1.0" +dependencies = [ + "pretty_assertions", + "tempfile", + "toml 0.8.15", + "wasmer-config 0.8.0", + "webc", +] + [[package]] name = "wasmer-registry" version = "5.19.0" diff --git a/Cargo.toml b/Cargo.toml index 17a5ea9a5dd..862f8897b19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ members = [ "lib/derive", "lib/emscripten", "lib/object", + "lib/package", "lib/registry", "lib/sys-utils", "lib/types", diff --git a/lib/package/Cargo.toml b/lib/package/Cargo.toml new file mode 100644 index 00000000000..b7d68262df4 --- /dev/null +++ b/lib/package/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "wasmer-package" +version = "0.1.0" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +webc.workspace = true +wasmer-config = { version = "0.8.0", path = "../config" } +toml = "0.8.0" + +[dev-dependencies] +pretty_assertions.workspace = true +tempfile = "3.12.0" diff --git a/lib/package/src/lib.rs b/lib/package/src/lib.rs new file mode 100644 index 00000000000..b5b67213dac --- /dev/null +++ b/lib/package/src/lib.rs @@ -0,0 +1 @@ +pub mod convert; From af0995c854712020f14bc82cd4243800c429e52b Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Sun, 15 Sep 2024 19:06:03 +0330 Subject: [PATCH 05/53] add the right to fetch stats on pipes --- lib/wasix/src/syscalls/wasix/fd_pipe.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/syscalls/wasix/fd_pipe.rs b/lib/wasix/src/syscalls/wasix/fd_pipe.rs index 7faf4518707..2ca80e2fd27 100644 --- a/lib/wasix/src/syscalls/wasix/fd_pipe.rs +++ b/lib/wasix/src/syscalls/wasix/fd_pipe.rs @@ -66,7 +66,8 @@ pub fn fd_pipe_internal( | Rights::FD_DATASYNC | Rights::POLL_FD_READWRITE | Rights::SOCK_SEND - | Rights::FD_FDSTAT_SET_FLAGS; + | Rights::FD_FDSTAT_SET_FLAGS + | Rights::FD_FILESTAT_GET; let fd1 = if let Some(fd) = with_fd1 { state From b94acb1ba099c6d42af9e697d9a3bfc5a0ad0112 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 11:07:27 +0200 Subject: [PATCH 06/53] feat(package): Implement restoring packages from webcs Convert webc images to a package directory with a wasmer.toml --- lib/config/src/package/mod.rs | 14 + lib/package/src/convert/error.rs | 39 +++ lib/package/src/convert/mod.rs | 4 + lib/package/src/convert/webc_to_package.rs | 331 +++++++++++++++++++++ 4 files changed, 388 insertions(+) create mode 100644 lib/package/src/convert/error.rs create mode 100644 lib/package/src/convert/mod.rs create mode 100644 lib/package/src/convert/webc_to_package.rs diff --git a/lib/config/src/package/mod.rs b/lib/config/src/package/mod.rs index 9f3ce2e743f..75aa3c054cd 100644 --- a/lib/config/src/package/mod.rs +++ b/lib/config/src/package/mod.rs @@ -185,6 +185,10 @@ pub struct Package { } impl Package { + pub fn new_empty() -> Self { + PackageBuilder::default().build().unwrap() + } + /// Create a [`PackageBuilder`] populated with all mandatory fields. pub fn builder( name: impl Into, @@ -732,6 +736,16 @@ pub struct Manifest { } impl Manifest { + pub fn new_empty() -> Self { + Self { + package: None, + dependencies: HashMap::new(), + fs: IndexMap::new(), + modules: Vec::new(), + commands: Vec::new(), + } + } + /// Create a [`ManifestBuilder`] populated with all mandatory fields. pub fn builder(package: Package) -> ManifestBuilder { ManifestBuilder::new(package) diff --git a/lib/package/src/convert/error.rs b/lib/package/src/convert/error.rs new file mode 100644 index 00000000000..28aec4fc2f7 --- /dev/null +++ b/lib/package/src/convert/error.rs @@ -0,0 +1,39 @@ +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct ManifestConversionError { + message: String, + cause: Option>, +} + +impl ManifestConversionError { + pub fn msg(msg: impl Into) -> Self { + Self { + message: msg.into(), + cause: None, + } + } + + pub fn with_cause( + msg: impl Into, + cause: impl std::error::Error + Send + Sync + 'static, + ) -> Self { + Self { + message: msg.into(), + cause: Some(Arc::new(cause)), + } + } +} + +impl std::fmt::Display for ManifestConversionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "could not convert manifest: {}", self.message)?; + if let Some(cause) = &self.cause { + write!(f, " (cause: {})", cause)?; + } + + Ok(()) + } +} + +impl std::error::Error for ManifestConversionError {} diff --git a/lib/package/src/convert/mod.rs b/lib/package/src/convert/mod.rs new file mode 100644 index 00000000000..a41517148af --- /dev/null +++ b/lib/package/src/convert/mod.rs @@ -0,0 +1,4 @@ +mod error; +mod webc_to_package; + +pub use self::{error::ManifestConversionError, webc_to_package::webc_to_package_dir}; diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs new file mode 100644 index 00000000000..eb424f0348e --- /dev/null +++ b/lib/package/src/convert/webc_to_package.rs @@ -0,0 +1,331 @@ +use std::path::Path; + +use wasmer_config::package::ModuleReference; + +use super::ManifestConversionError; + +/// Convert a webc image into a directory with a wasmer.toml file that can +/// be used for generating a new pacakge. +pub fn webc_to_package_dir( + webc: &webc::Container, + target_dir: &Path, +) -> Result<(), ManifestConversionError> { + let mut pkg_manifest = wasmer_config::package::Manifest::new_empty(); + + let manifest = webc.manifest(); + // Convert the package annotation. + + let pkg_annotation = manifest.wapm().map_err(|err| { + ManifestConversionError::with_cause("could not read package annotation", err) + })?; + if let Some(ann) = pkg_annotation { + let mut pkg = wasmer_config::package::Package::new_empty(); + + pkg.name = ann.name; + pkg.version = if let Some(raw) = ann.version { + let v = raw + .parse() + .map_err(|e| ManifestConversionError::with_cause("invalid package version", e))?; + Some(v) + } else { + None + }; + + pkg.description = ann.description; + pkg.license = ann.license; + + // TODO: map license_file and README (paths!) + + pkg.homepage = ann.homepage; + pkg.repository = ann.repository; + pkg.private = ann.private; + pkg.entrypoint = manifest.entrypoint.clone(); + + pkg_manifest.package = Some(pkg); + } + + // Map dependencies. + for (_name, target) in &manifest.use_map { + match target { + webc::metadata::UrlOrManifest::Url(_url) => { + // Not supported. + } + webc::metadata::UrlOrManifest::Manifest(_) => { + // Not supported. + } + webc::metadata::UrlOrManifest::RegistryDependentUrl(raw) => { + let (name, version) = if let Some((name, version_raw)) = raw.split_once('@') { + let version = version_raw.parse().map_err(|err| { + ManifestConversionError::with_cause( + format!("Could not parse version of dependency: '{}'", raw), + err, + ) + })?; + (name.to_string(), version) + } else { + (raw.to_string(), "*".parse().unwrap()) + }; + + pkg_manifest.dependencies.insert(name, version); + } + } + } + + // Convert filesystem mappings. + + let fs_annotation = manifest + .filesystem() + .map_err(|err| ManifestConversionError::with_cause("could n ot read fs annotation", err))?; + if let Some(ann) = fs_annotation { + for mapping in ann.0 { + if mapping.from.is_some() { + // wasmer.toml does not allow specifying dependency mounts. + continue; + } + + // Extract the volume to "/". + let volume = webc.get_volume(&mapping.volume_name).ok_or_else(|| { + ManifestConversionError::msg(format!( + "Package annotations specify a volume that does not exist: '{}'", + mapping.volume_name + )) + })?; + + let volume_path = target_dir.join(mapping.volume_name.trim_start_matches('/')); + + std::fs::create_dir_all(&volume_path).map_err(|err| { + ManifestConversionError::with_cause( + format!( + "could not create volume directory '{}'", + volume_path.display() + ), + err, + ) + })?; + + volume.unpack("/", &volume_path).map_err(|err| { + ManifestConversionError::with_cause("could not unpack volume to filesystemt", err) + })?; + + let mut source_path = mapping + .volume_name + .trim_start_matches('/') + .trim_end_matches('/') + .to_string(); + if let Some(subpath) = mapping.host_path { + if !source_path.ends_with('/') { + source_path.push('/'); + } + source_path.push_str(&subpath); + } + source_path.insert_str(0, "./"); + + pkg_manifest + .fs + .insert(mapping.mount_path, source_path.into()); + } + } + + // Convert modules. + + let module_dir_name = "modules"; + let module_dir = target_dir.join(module_dir_name); + + for (atom_name, data) in webc.atoms() { + let atom_path = module_dir.join(&atom_name); + + std::fs::create_dir_all(&module_dir).map_err(|err| { + ManifestConversionError::with_cause( + format!("Could not create directory '{}'", module_dir.display(),), + err, + ) + })?; + + std::fs::write(&atom_path, &data).map_err(|err| { + ManifestConversionError::with_cause( + format!("Could not write atom to path '{}'", atom_path.display()), + err, + ) + })?; + + let relative_path = format!("./{module_dir_name}/{atom_name}"); + + pkg_manifest.modules.push(wasmer_config::package::Module { + name: atom_name, + source: relative_path.into(), + abi: wasmer_config::package::Abi::None, + kind: None, + interfaces: None, + bindings: None, + }); + } + + // Convert commands. + for (name, spec) in &manifest.commands { + let mut annotations = toml::Table::new(); + for (key, value) in &spec.annotations { + if key == webc::metadata::annotations::Atom::KEY { + continue; + } + + let raw_toml = toml::to_string(&value).unwrap(); + let toml_value = toml::from_str::(&raw_toml).unwrap(); + annotations.insert(key.into(), toml_value); + } + + let atom_annotation = spec + .annotation::(webc::metadata::annotations::Atom::KEY) + .map_err(|err| { + ManifestConversionError::with_cause( + format!("could not read atom annotation for command '{}'", name), + err, + ) + })? + .ok_or_else(|| { + ManifestConversionError::msg(format!( + "Command '{name}' is missing the required atom annotation" + )) + })?; + + let module = if let Some(dep) = atom_annotation.dependency { + ModuleReference::Dependency { + dependency: dep, + module: atom_annotation.name, + } + } else { + ModuleReference::CurrentPackage { + module: atom_annotation.name, + } + }; + + let cmd = wasmer_config::package::Command::V2(wasmer_config::package::CommandV2 { + name: name.clone(), + module, + runner: spec.runner.clone(), + annotations: Some(wasmer_config::package::CommandAnnotations::Raw( + annotations.into(), + )), + }); + + pkg_manifest.commands.push(cmd); + } + + // Write out the manifest. + let manifest_toml = toml::to_string(&pkg_manifest).map_err(|err| { + ManifestConversionError::with_cause("could not serialize package manifest", err) + })?; + std::fs::write(target_dir.join("wasmer.toml"), manifest_toml) + .map_err(|err| ManifestConversionError::with_cause("could not write wasmer.toml", err))?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::fs::create_dir_all; + + use pretty_assertions::assert_eq; + + use super::*; + + // Build a webc from a pacakge directory, and then restore the directory + // from the webc. + #[test] + fn test_wasmer_package_webc_roundtrip() { + let tmpdir = tempfile::tempdir().unwrap(); + let dir = tmpdir.path(); + + let webc = { + let dir_input = dir.join("input"); + let dir_public = dir_input.join("public"); + + create_dir_all(&dir_public).unwrap(); + + std::fs::write(dir_public.join("index.html"), "INDEX").unwrap(); + + std::fs::write(dir_input.join("mywasm.wasm"), "()").unwrap(); + + std::fs::write( + dir_input.join("wasmer.toml"), + r#" +[package] +name = "testns/testpkg" +version = "0.0.1" +description = "descr1" +license = "MIT" + +[dependencies] +"wasmer/python" = "8.12.0" + +[fs] +public = "./public" + +[[module]] +name = "mywasm" +source = "./mywasm.wasm" + +[[command]] +name = "run" +module = "mywasm" +runner = "wasi" + +[command.annotations.wasi] +env = ["A=B"] +main-args = ["/mounted/script.py"] +"#, + ) + .unwrap(); + + let pkg = webc::wasmer_package::Package::from_manifest(dir_input.join("wasmer.toml")) + .unwrap(); + let raw = pkg.serialize().unwrap(); + webc::Container::from_bytes(raw).unwrap() + }; + + let dir_output = dir.join("output"); + webc_to_package_dir(&webc, &dir_output).unwrap(); + + assert_eq!( + std::fs::read_to_string(dir_output.join("public/index.html")).unwrap(), + "INDEX", + ); + + assert_eq!( + std::fs::read_to_string(dir_output.join("modules/mywasm")).unwrap(), + "()", + ); + + assert_eq!( + std::fs::read_to_string(dir_output.join("wasmer.toml")) + .unwrap() + .trim(), + r#" + +[package] +license = "MIT" +entrypoint = "run" + +[dependencies] +"wasmer/python" = "^8.12.0" + +[fs] +"/public" = "./public" + +[[module]] +name = "mywasm" +source = "./modules/mywasm" + +[[command]] +name = "run" +module = "mywasm" +runner = "https://webc.org/runner/wasi" + +[command.annotations.wasi] +atom = "mywasm" +env = ["A=B"] +main-args = ["/mounted/script.py"] + "# + .trim(), + ); + } +} From bf40de3e6e32c31607e441f608db06fdb826a2e7 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 11:08:13 +0200 Subject: [PATCH 07/53] feat(cli): Allow "package unpack" to restore packages Add a new --format option to package unpack. * package: Try to restore a usable package dir with a wasmer.toml * webc: just unpack the webc --- lib/cli/Cargo.toml | 2 ++ lib/cli/src/commands/package/unpack.rs | 45 +++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index a71aa595e32..538735e6736 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -117,6 +117,8 @@ wasmer-compiler-cranelift = { version = "=4.3.7", path = "../compiler-cranelift" wasmer-compiler-singlepass = { version = "=4.3.7", path = "../compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=4.3.7", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=4.3.7", path = "../emscripten" } +wasmer-package = { version = "=0.1.0", path = "../package" } + wasmer-vm = { version = "=4.3.7", path = "../vm", optional = true } wasmer-wasix = { path = "../wasix", version = "=0.27.0", features = [ "logging", diff --git a/lib/cli/src/commands/package/unpack.rs b/lib/cli/src/commands/package/unpack.rs index 0f4784886e8..23d53b90164 100644 --- a/lib/cli/src/commands/package/unpack.rs +++ b/lib/cli/src/commands/package/unpack.rs @@ -5,27 +5,53 @@ use dialoguer::console::{style, Emoji}; use indicatif::ProgressBar; /// Extract contents of a webc image to a directory. +/// +/// See --format flag for available output formats. #[derive(clap::Parser, Debug)] pub struct PackageUnpack { /// The output directory. #[clap(short = 'o', long)] - out_dir: PathBuf, + pub out_dir: PathBuf, /// Overwrite existing directories/files. #[clap(long)] - overwrite: bool, + pub overwrite: bool, /// Run the unpack command without any output #[clap(long)] pub quiet: bool, /// Path to the package. - package_path: PathBuf, + pub package_path: PathBuf, + + /// Output format. + /// + /// * package + /// Restore a package directory with a wasmer.toml + /// NOTE: this conversion is lossy, because webcs don't store the original + /// wasmer.toml and the full contents can not be restored. + /// + /// * webc + /// Directly unpack the webc contents. + /// - Volumes will be placed in subdirectories. + /// - atoms will be placed in the root directory + /// - the full webc manifest will be placed in a manifest.json file + #[clap(short, long, default_value = "package")] + pub format: Format, } static PACKAGE_EMOJI: Emoji<'_, '_> = Emoji("📦 ", ""); static EXTRACTED_TO_EMOJI: Emoji<'_, '_> = Emoji("📂 ", ""); +/// Webc unpack format. +#[derive(clap::ValueEnum, Clone, Debug)] +pub enum Format { + /// See [`PackageUnpack::format`] for details. + Package, + /// See [`PackageUnpack::format`] for details. + Webc, +} + impl PackageUnpack { pub(crate) fn execute(&self) -> Result<(), anyhow::Error> { // Setup the progress bar @@ -52,8 +78,16 @@ impl PackageUnpack { std::fs::create_dir_all(outdir) .with_context(|| format!("could not create output directory '{}'", outdir.display()))?; - pkg.unpack(outdir, self.overwrite) - .with_context(|| "could not extract package".to_string())?; + match self.format { + Format::Package => { + wasmer_package::convert::webc_to_package_dir(&pkg, outdir) + .with_context(|| "could not extract package")?; + } + Format::Webc => { + pkg.unpack(outdir, self.overwrite) + .with_context(|| "could not extract package".to_string())?; + } + } pb.println(format!( "{} {}Extracted package contents to '{}'", @@ -89,6 +123,7 @@ mod tests { overwrite: false, package_path, quiet: true, + format: Format::Webc, }; cmd.execute().unwrap(); From 5cae10d6897931e409b70a6eac942d719bc1e799 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Mon, 16 Sep 2024 10:13:19 +0200 Subject: [PATCH 08/53] chore: Rename ManifestConversionError to ConversionError + impl Error::source --- lib/package/src/convert/error.rs | 16 +++++++--- lib/package/src/convert/mod.rs | 2 +- lib/package/src/convert/webc_to_package.rs | 37 +++++++++++----------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/lib/package/src/convert/error.rs b/lib/package/src/convert/error.rs index 28aec4fc2f7..2f6e7566b35 100644 --- a/lib/package/src/convert/error.rs +++ b/lib/package/src/convert/error.rs @@ -1,12 +1,12 @@ use std::sync::Arc; #[derive(Clone, Debug)] -pub struct ManifestConversionError { +pub struct ConversionError { message: String, cause: Option>, } -impl ManifestConversionError { +impl ConversionError { pub fn msg(msg: impl Into) -> Self { Self { message: msg.into(), @@ -25,7 +25,7 @@ impl ManifestConversionError { } } -impl std::fmt::Display for ManifestConversionError { +impl std::fmt::Display for ConversionError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "could not convert manifest: {}", self.message)?; if let Some(cause) = &self.cause { @@ -36,4 +36,12 @@ impl std::fmt::Display for ManifestConversionError { } } -impl std::error::Error for ManifestConversionError {} +impl std::error::Error for ConversionError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + if let Some(e) = &self.cause { + Some(e) + } else { + None + } + } +} diff --git a/lib/package/src/convert/mod.rs b/lib/package/src/convert/mod.rs index a41517148af..ca75e1a1b6f 100644 --- a/lib/package/src/convert/mod.rs +++ b/lib/package/src/convert/mod.rs @@ -1,4 +1,4 @@ mod error; mod webc_to_package; -pub use self::{error::ManifestConversionError, webc_to_package::webc_to_package_dir}; +pub use self::{error::ConversionError, webc_to_package::webc_to_package_dir}; diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs index eb424f0348e..a3aaacd8269 100644 --- a/lib/package/src/convert/webc_to_package.rs +++ b/lib/package/src/convert/webc_to_package.rs @@ -2,22 +2,22 @@ use std::path::Path; use wasmer_config::package::ModuleReference; -use super::ManifestConversionError; +use super::ConversionError; /// Convert a webc image into a directory with a wasmer.toml file that can /// be used for generating a new pacakge. pub fn webc_to_package_dir( webc: &webc::Container, target_dir: &Path, -) -> Result<(), ManifestConversionError> { +) -> Result<(), ConversionError> { let mut pkg_manifest = wasmer_config::package::Manifest::new_empty(); let manifest = webc.manifest(); // Convert the package annotation. - let pkg_annotation = manifest.wapm().map_err(|err| { - ManifestConversionError::with_cause("could not read package annotation", err) - })?; + let pkg_annotation = manifest + .wapm() + .map_err(|err| ConversionError::with_cause("could not read package annotation", err))?; if let Some(ann) = pkg_annotation { let mut pkg = wasmer_config::package::Package::new_empty(); @@ -25,7 +25,7 @@ pub fn webc_to_package_dir( pkg.version = if let Some(raw) = ann.version { let v = raw .parse() - .map_err(|e| ManifestConversionError::with_cause("invalid package version", e))?; + .map_err(|e| ConversionError::with_cause("invalid package version", e))?; Some(v) } else { None @@ -56,7 +56,7 @@ pub fn webc_to_package_dir( webc::metadata::UrlOrManifest::RegistryDependentUrl(raw) => { let (name, version) = if let Some((name, version_raw)) = raw.split_once('@') { let version = version_raw.parse().map_err(|err| { - ManifestConversionError::with_cause( + ConversionError::with_cause( format!("Could not parse version of dependency: '{}'", raw), err, ) @@ -75,7 +75,7 @@ pub fn webc_to_package_dir( let fs_annotation = manifest .filesystem() - .map_err(|err| ManifestConversionError::with_cause("could n ot read fs annotation", err))?; + .map_err(|err| ConversionError::with_cause("could n ot read fs annotation", err))?; if let Some(ann) = fs_annotation { for mapping in ann.0 { if mapping.from.is_some() { @@ -85,7 +85,7 @@ pub fn webc_to_package_dir( // Extract the volume to "/". let volume = webc.get_volume(&mapping.volume_name).ok_or_else(|| { - ManifestConversionError::msg(format!( + ConversionError::msg(format!( "Package annotations specify a volume that does not exist: '{}'", mapping.volume_name )) @@ -94,7 +94,7 @@ pub fn webc_to_package_dir( let volume_path = target_dir.join(mapping.volume_name.trim_start_matches('/')); std::fs::create_dir_all(&volume_path).map_err(|err| { - ManifestConversionError::with_cause( + ConversionError::with_cause( format!( "could not create volume directory '{}'", volume_path.display() @@ -104,7 +104,7 @@ pub fn webc_to_package_dir( })?; volume.unpack("/", &volume_path).map_err(|err| { - ManifestConversionError::with_cause("could not unpack volume to filesystemt", err) + ConversionError::with_cause("could not unpack volume to filesystemt", err) })?; let mut source_path = mapping @@ -135,14 +135,14 @@ pub fn webc_to_package_dir( let atom_path = module_dir.join(&atom_name); std::fs::create_dir_all(&module_dir).map_err(|err| { - ManifestConversionError::with_cause( + ConversionError::with_cause( format!("Could not create directory '{}'", module_dir.display(),), err, ) })?; std::fs::write(&atom_path, &data).map_err(|err| { - ManifestConversionError::with_cause( + ConversionError::with_cause( format!("Could not write atom to path '{}'", atom_path.display()), err, ) @@ -176,13 +176,13 @@ pub fn webc_to_package_dir( let atom_annotation = spec .annotation::(webc::metadata::annotations::Atom::KEY) .map_err(|err| { - ManifestConversionError::with_cause( + ConversionError::with_cause( format!("could not read atom annotation for command '{}'", name), err, ) })? .ok_or_else(|| { - ManifestConversionError::msg(format!( + ConversionError::msg(format!( "Command '{name}' is missing the required atom annotation" )) })?; @@ -211,11 +211,10 @@ pub fn webc_to_package_dir( } // Write out the manifest. - let manifest_toml = toml::to_string(&pkg_manifest).map_err(|err| { - ManifestConversionError::with_cause("could not serialize package manifest", err) - })?; + let manifest_toml = toml::to_string(&pkg_manifest) + .map_err(|err| ConversionError::with_cause("could not serialize package manifest", err))?; std::fs::write(target_dir.join("wasmer.toml"), manifest_toml) - .map_err(|err| ManifestConversionError::with_cause("could not write wasmer.toml", err))?; + .map_err(|err| ConversionError::with_cause("could not write wasmer.toml", err))?; Ok(()) } From 90b0d0955b3ad9c85a668b76e60fca8dbec4dd2b Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:20:14 +0200 Subject: [PATCH 09/53] build: Ignore direnv related files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9e6499be8aa..f6deb5be7f0 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ lcov.info link/ link.tar.gz .vscode +# Ignore files used by the "direnv" tool. +.direnv +.envrc From 47c29be5d9ed867de749808f0da0d8f3411586d9 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:21:32 +0200 Subject: [PATCH 10/53] refactor: Remove edge-schema dependency --- Cargo.toml | 1 - lib/backend-api/Cargo.toml | 1 - lib/cli/Cargo.toml | 1 - lib/cli/src/commands/app/logs.rs | 6 +-- lib/cli/src/utils/mod.rs | 1 + lib/cli/src/utils/timestamp.rs | 68 ++++++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 lib/cli/src/utils/timestamp.rs diff --git a/Cargo.toml b/Cargo.toml index bff17a4d1dd..4ce1c769ef5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,7 +93,6 @@ wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates webc = { version = "6.0.0", default-features = false, features = ["package"] } -edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" # Third-party crates diff --git a/lib/backend-api/Cargo.toml b/lib/backend-api/Cargo.toml index 2adc6a88bc8..e2b0f25c2cb 100644 --- a/lib/backend-api/Cargo.toml +++ b/lib/backend-api/Cargo.toml @@ -15,7 +15,6 @@ rust-version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] # Wasmer dependencies. -edge-schema.workspace = true wasmer-config = { version = "0.8.0", path = "../config" } webc.workspace = true diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index fbac043c74e..9829a8ed696 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -146,7 +146,6 @@ virtual-mio = { version = "0.3.1", path = "../virtual-io" } webc = { workspace = true } wasmer-api = { version = "=0.0.34", path = "../backend-api" } -edge-schema.workspace = true edge-util = { version = "=0.1.0" } lazy_static = "1.4.0" diff --git a/lib/cli/src/commands/app/logs.rs b/lib/cli/src/commands/app/logs.rs index 1d297da73cc..55201d75876 100644 --- a/lib/cli/src/commands/app/logs.rs +++ b/lib/cli/src/commands/app/logs.rs @@ -1,8 +1,8 @@ //! Show logs for an Edge app. +use crate::utils::timestamp::parse_timestamp_or_relative_time_negative_offset; use colored::Colorize; use comfy_table::{Cell, Table}; -use edge_schema::pretty_duration::parse_timestamp_or_relative_time; use futures::StreamExt; use time::{format_description::well_known::Rfc3339, OffsetDateTime}; use wasmer_api::types::{Log, LogStream}; @@ -37,7 +37,7 @@ pub struct CmdAppLogs { /// * Unix timestamp (`1136196245`) /// * Relative time (`10m` / `-1h`, `1d1h30s`) // TODO: should default to trailing logs once trailing is implemented. - #[clap(long, value_parser = parse_timestamp_or_relative_time, conflicts_with = "request_id")] + #[clap(long, value_parser = parse_timestamp_or_relative_time_negative_offset, conflicts_with = "request_id")] from: Option, /// The date of the latest log entry. @@ -48,7 +48,7 @@ pub struct CmdAppLogs { /// * Simple date (`2022-11-11`) /// * Unix timestamp (`1136196245`) /// * Relative time (`10m` / `1h`, `1d1h30s`) - #[clap(long, value_parser = parse_timestamp_or_relative_time, conflicts_with = "request_id")] + #[clap(long, value_parser = parse_timestamp_or_relative_time_negative_offset, conflicts_with = "request_id")] until: Option, /// Maximum log lines to fetch. diff --git a/lib/cli/src/utils/mod.rs b/lib/cli/src/utils/mod.rs index 7cd7550963d..73fe8c4330f 100644 --- a/lib/cli/src/utils/mod.rs +++ b/lib/cli/src/utils/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod package_wizard; pub(crate) mod prompts; pub(crate) mod render; +pub(crate) mod timestamp; use std::{ path::{Path, PathBuf}, diff --git a/lib/cli/src/utils/timestamp.rs b/lib/cli/src/utils/timestamp.rs new file mode 100644 index 00000000000..61b61f187fb --- /dev/null +++ b/lib/cli/src/utils/timestamp.rs @@ -0,0 +1,68 @@ +use time::{ + format_description::well_known::{Rfc2822, Rfc3339}, + Date, OffsetDateTime, PrimitiveDateTime, Time, +}; + +/// Try to parse the string as a timestamp in a number of well-known formats. +/// +/// Supported formats, +/// +/// - RFC 3339 (`2006-01-02T03:04:05-07:00`) +/// - RFC 2822 (`Mon, 02 Jan 2006 03:04:05 MST`) +/// - Date (`2006-01-02`) +/// - Unix timestamp (`1136196245`) +/// - Relative time (`10m` / `-1h`, `1d`) +pub fn parse_timestamp_or_relative_time( + s: &str, + assume_negative_offset: bool, +) -> Result { + parse_timestamp_or_relative_time_based(s, OffsetDateTime::now_utc(), assume_negative_offset) +} + +/// See [`parse_timestamp_or_relative_time`]. +/// +/// NOTE: assumes a negative offset if time is specified in a format like "1h". +pub fn parse_timestamp_or_relative_time_negative_offset( + s: &str, +) -> Result { + parse_timestamp_or_relative_time(s, true) +} + +/// See [`parse_timestamp_or_relative_time`]. +pub fn parse_timestamp_or_relative_time_based( + s: &str, + base: OffsetDateTime, + assume_negative_offset: bool, +) -> Result { + if let Ok(t) = OffsetDateTime::parse(s, &Rfc3339) { + return Ok(t); + } + if let Ok(t) = OffsetDateTime::parse(s, &Rfc2822) { + return Ok(t); + } + if let Ok(t) = Date::parse(s, time::macros::format_description!("[year]-[month]-[day]")) { + return Ok(PrimitiveDateTime::new(t, Time::MIDNIGHT).assume_utc()); + } + if let Ok(t) = OffsetDateTime::parse(s, time::macros::format_description!("[unix_timestamp]")) { + return Ok(t); + } + + // Relative time. + let (is_negative, v) = match s.strip_prefix('-') { + Some(rest) => (true, rest), + // Note: `assume_negative_offset` forces offsets to be negative. + None => (assume_negative_offset, s.trim_start_matches('+')), + }; + + if let Ok(duration) = humantime::parse_duration(v) { + let time = if is_negative { + base - duration + } else { + base + duration + }; + + return Ok(time); + } + + anyhow::bail!("Unable to parse the timestamp - no known format matched") +} From a3d79376ef1405c320dbda5a9ccf0fb8b51ccf91 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:23:47 +0200 Subject: [PATCH 11/53] chore: Remove unused generate_deploy_token function --- lib/registry/src/api.rs | 52 +---------------------------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/lib/registry/src/api.rs b/lib/registry/src/api.rs index 5c107cea7d4..7684f148eff 100644 --- a/lib/registry/src/api.rs +++ b/lib/registry/src/api.rs @@ -3,57 +3,7 @@ use anyhow::Context; use crate::RegistryClient; use crate::graphql::mutations::{self}; -use crate::types::{NewNonceOutput, PublishDeployAppOutput, PublishDeployAppRawVars}; - -/// Generate a Deploy token for for the given Deploy app version id. -pub async fn generate_deploy_token( - client: &RegistryClient, - app_version_id: String, -) -> Result { - let vars = mutations::generate_deploy_token::Variables { app_version_id }; - let res = client - .execute::(vars) - .await?; - let token = res - .generate_deploy_token - .context("Query did not return a token")? - .token; - - Ok(token) -} - -/// Publish a Deploy app. -/// -/// Takes a raw, unvalidated deployment config. -// TODO: Add a variant of this query that takes a typed DeployV1 config. -pub async fn publish_deploy_app_raw( - client: &RegistryClient, - data: PublishDeployAppRawVars, -) -> Result { - let vars2 = mutations::publish_deploy_app::Variables { - name: data.name, - owner: data.namespace, - config: serde_json::to_string(&data.config)?, - }; - - let version = client - .execute::(vars2) - .await? - .publish_deploy_app - .context("Query did not return data")? - .deploy_app_version; - - let app = version.app; - // let app = version.app.context("Query did not return expected data")?; - - Ok(PublishDeployAppOutput { - app_id: app.id, - app_name: app.name, - version_id: version.id, - version_name: version.version, - owner_name: app.owner.global_name, - }) -} +use crate::types::NewNonceOutput; /// Generate a new Nonce /// From 62de6a062eff1b6ed1b2124768bbf11370b65cee Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:24:33 +0200 Subject: [PATCH 12/53] deps: lift base64 into a workspace dependency --- Cargo.toml | 1 + lib/backend-api/Cargo.toml | 6 ++++-- lib/journal/Cargo.toml | 7 ++++--- lib/virtual-net/Cargo.toml | 9 +++++---- lib/wasix/Cargo.toml | 13 +++++++------ 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4ce1c769ef5..225f7755a64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,7 @@ toml = {version = "0.5.9", features = ["preserve_order"]} indexmap = "2" serde_yaml = "0.9.34" libc = { version = "^0.2", default-features = false } +base64 = "0.22.0" [build-dependencies] test-generator = { path = "tests/lib/test-generator" } diff --git a/lib/backend-api/Cargo.toml b/lib/backend-api/Cargo.toml index e2b0f25c2cb..e67d45522a3 100644 --- a/lib/backend-api/Cargo.toml +++ b/lib/backend-api/Cargo.toml @@ -18,7 +18,10 @@ rust-version.workspace = true wasmer-config = { version = "0.8.0", path = "../config" } webc.workspace = true + # crates.io dependencies. +reqwest = { workspace = true, default-features = false, features = ["json"] } + anyhow = "1" serde = { version = "1", features = ["derive"] } time = { version = "0.3", features = ["formatting", "parsing"] } @@ -31,7 +34,6 @@ cynic = { version = "3.7.2", features = ["http-reqwest"] } pin-project-lite = "0.2.10" serde_path_to_error = "0.1.14" harsh = "0.2.2" -reqwest = { workspace = true, default-features = false, features = ["json"] } merge-streams = "0.1.2" [target.'cfg(target_family = "wasm")'.dependencies.getrandom] @@ -39,6 +41,6 @@ version = "0.2.14" features = ["js"] [dev-dependencies] -base64 = "0.13.1" +base64.workspace = true tokio = { version = "1.3", features = ["macros", "rt"] } uuid = { version = "1", features = ["v4"] } diff --git a/lib/journal/Cargo.toml b/lib/journal/Cargo.toml index 891040d40f0..19478d0ac2d 100644 --- a/lib/journal/Cargo.toml +++ b/lib/journal/Cargo.toml @@ -20,16 +20,17 @@ virtual-net = { path = "../virtual-net", version = "0.8.0", default-features = f virtual-fs = { path = "../virtual-fs", version = "0.16.0", default-features = false } shared-buffer = { workspace = true, optional = true } +base64.workspace = true +rkyv = { workspace = true } + thiserror = "1" bytes = "1.1" async-trait = { version = "^0.1" } tracing = "0.1" derivative = { version = "^2" } -base64 = "0.21" bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } anyhow = "1.0" -rkyv = { workspace = true } bytecheck = { version = "0.6.8" } lz4_flex = { version = "0.11" } num_enum = "0.5.7" @@ -37,4 +38,4 @@ serde_json = { version = "^1" } [dev-dependencies] tracing-test = "0.2.4" -tempfile = "3.6.0" \ No newline at end of file +tempfile = "3.6.0" diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index b7b01572cae..d2a41b770fb 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -10,17 +10,20 @@ repository.workspace = true rust-version.workspace = true [dependencies] +base64.workspace = true +libc = { workspace = true, optional = true } +hyper = { workspace = true, optional = true } +rkyv = { workspace = true, optional = true } + thiserror = "1" bytes = "1.1" async-trait = { version = "^0.1" } tracing = "0.1" tokio = { version = "1", default-features = false, features = ["io-util"] } -libc = { workspace = true, optional = true } mio = { version = "0.8", optional = true, features = ["net"] } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } virtual-mio = { path = "../virtual-io", version = "0.3.0", default-features = false } -base64 = "0.21" bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } pin-project-lite = "0.2.9" @@ -29,9 +32,7 @@ anyhow = "1.0" tokio-serde = { version = "0.8", features = [ "bincode" ], optional = true } tokio-util = { version = "0.7.8", features = ["codec"], optional = true } hyper-tungstenite = { version = "0.13", optional = true } -hyper = { workspace = true, optional = true } tokio-tungstenite = { version = "0.21", optional = true } -rkyv = { workspace = true, optional = true } bytecheck = { version = "0.6.8", optional = true } hyper-util = { version = "0.1.5", features = ["tokio"], optional = true } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 0e8b2fb289a..7b4644a9558 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -25,6 +25,13 @@ wasmer-config = { version = "0.8.0", path = "../config" } http.workspace = true dashmap.workspace = true +base64.workspace = true +webc.workspace = true +serde_yaml.workspace = true +rkyv.workspace = true +shared-buffer.workspace = true +hyper = { workspace = true, features = ["server"], optional = true } + xxhash-rust = { version = "0.8.8", features = ["xxh64"] } rusty_pool = { version = "0.7.0", optional = true } cfg-if = "1.0" @@ -37,7 +44,6 @@ bincode = { version = "1.3" } chrono = { version = "^0.4.31", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" -webc = { workspace = true } anyhow = { version = "1.0.66" } lazy_static = "1.4" sha2 = { version = "0.10" } @@ -57,7 +63,6 @@ async-trait = { version = "^0.1" } urlencoding = { version = "^2" } serde_derive = { version = "^1" } serde_json = { version = "^1" } -serde_yaml.workspace = true weezl = { version = "^0.1" } hex = { version = "^0.4" } linked_hash_set = { version = "0.1" } @@ -68,7 +73,6 @@ semver = "1.0.17" tempfile = "3.6.0" num_enum = "0.5.7" # Used by the WCGI runner -hyper = { workspace = true, features = ["server"], optional = true } wcgi = { version = "0.2.0", optional = true } wcgi-host = { version = "0.2.0", optional = true } tower-http = { version = "0.5.0", features = [ @@ -79,12 +83,9 @@ tower-http = { version = "0.5.0", features = [ ], optional = true } tower = { version = "0.4.13", features = ["make", "util"], optional = true } url = "2.3.1" -rkyv = { workspace = true } bytecheck = "0.6.8" blake3 = "1.0" -shared-buffer = { workspace = true } petgraph = "0.6.3" -base64 = "0.21" lz4_flex = { version = "0.11" } rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } From 0c459e480f104684c60cbedfb41278c891909422 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:25:21 +0200 Subject: [PATCH 13/53] chore: Remove unused code from backend-api --- lib/backend-api/src/query.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/lib/backend-api/src/query.rs b/lib/backend-api/src/query.rs index 479cd7568b4..fe888837882 100644 --- a/lib/backend-api/src/query.rs +++ b/lib/backend-api/src/query.rs @@ -2,7 +2,6 @@ use std::{collections::HashSet, time::Duration}; use anyhow::{bail, Context}; use cynic::{MutationBuilder, QueryBuilder}; -use edge_schema::schema::NetworkTokenV1; use futures::StreamExt; use merge_streams::MergeStreams; use time::OffsetDateTime; @@ -1596,31 +1595,9 @@ pub fn get_package_releases_stream( ) } -/// Generate a new Edge token. -pub async fn generate_deploy_token_raw( - client: &WasmerClient, - app_version_id: String, -) -> Result { - let res = client - .run_graphql(types::GenerateDeployToken::build( - types::GenerateDeployTokenVars { app_version_id }, - )) - .await?; - - res.generate_deploy_token - .map(|x| x.token) - .context("no token returned") -} - -#[derive(Debug, PartialEq)] -pub enum GenerateTokenBy { - Id(NetworkTokenV1), -} - #[derive(Debug, PartialEq)] pub enum TokenKind { SSH, - Network(GenerateTokenBy), } pub async fn generate_deploy_config_token_raw( @@ -1632,9 +1609,6 @@ pub async fn generate_deploy_config_token_raw( types::GenerateDeployConfigTokenVars { input: match token_kind { TokenKind::SSH => "{}".to_string(), - TokenKind::Network(by) => match by { - GenerateTokenBy::Id(token) => serde_json::to_string(&token)?, - }, }, }, )) From b079e15f66aa31c593a42fef9c85e89c4f3a9da6 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:26:07 +0200 Subject: [PATCH 14/53] chore: Remove commented out command and add a note --- lib/cli/src/commands/connect.rs | 2 ++ lib/cli/src/commands/mod.rs | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cli/src/commands/connect.rs b/lib/cli/src/commands/connect.rs index dee96296ca3..3549087b060 100644 --- a/lib/cli/src/commands/connect.rs +++ b/lib/cli/src/commands/connect.rs @@ -1,3 +1,5 @@ +// NOTE: not currently in use - but kept here to allow easy re-implementation. + use std::net::IpAddr; use crate::config::WasmerEnv; diff --git a/lib/cli/src/commands/mod.rs b/lib/cli/src/commands/mod.rs index 5b56d25b081..aaae8e1ec19 100644 --- a/lib/cli/src/commands/mod.rs +++ b/lib/cli/src/commands/mod.rs @@ -198,9 +198,6 @@ impl WasmerCmd { Some(Cmd::Container(cmd)) => match cmd { crate::commands::Container::Unpack(cmd) => cmd.execute(), }, - /* - Some(Cmd::Connect(connect)) => connect.execute(), - */ #[cfg(feature = "static-artifact-create")] Some(Cmd::GenCHeader(gen_heder)) => gen_heder.execute(), #[cfg(feature = "wast")] From 3b679310d795fcbac218c350e103112e8a7472b8 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:26:30 +0200 Subject: [PATCH 15/53] deps: Update Cargo.lock --- Cargo.lock | 56 +++++++++--------------------------------------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3849b3ec7b3..8a7aa7da35c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,18 +277,6 @@ dependencies = [ "rustc-demangle", ] -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - [[package]] name = "base64" version = "0.22.1" @@ -1525,37 +1513,13 @@ dependencies = [ "wcgi-host 0.1.2", ] -[[package]] -name = "edge-schema" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0966f1fd49610cc67a835124e6fb4d00a36104e1aa34383c5ef5a265ca00ea2a" -dependencies = [ - "anyhow", - "bytesize", - "once_cell", - "parking_lot 0.12.3", - "rand_chacha", - "rand_core", - "schemars", - "serde", - "serde_json", - "serde_path_to_error", - "serde_yaml 0.8.26", - "sparx", - "time 0.3.36", - "url", - "uuid", - "wcgi-host 0.1.2", -] - [[package]] name = "edge-util" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60d06780ae676369e0a9ac2c75f7d1a6d22e157ef5b10cfeda0ee6a465cf7337" dependencies = [ - "edge-schema 0.0.3", + "edge-schema", "http 0.2.12", "serde", "wasmparser 0.121.2", @@ -4076,7 +4040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "async-compression", - "base64 0.22.1", + "base64", "bytes 1.6.1", "futures-channel", "futures-core", @@ -4340,7 +4304,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.22.1", + "base64", "rustls-pki-types", ] @@ -5938,7 +5902,7 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" dependencies = [ - "base64 0.22.1", + "base64", "flate2", "log 0.4.22", "once_cell", @@ -6059,7 +6023,7 @@ version = "0.8.0" dependencies = [ "anyhow", "async-trait", - "base64 0.21.7", + "base64", "bincode", "bytecheck", "bytes 1.6.1", @@ -6470,9 +6434,8 @@ name = "wasmer-api" version = "0.0.34" dependencies = [ "anyhow", - "base64 0.13.1", + "base64", "cynic", - "edge-schema 0.1.0", "futures 0.3.30", "getrandom", "harsh", @@ -6622,7 +6585,6 @@ dependencies = [ "dialoguer", "dirs", "dotenvy", - "edge-schema 0.1.0", "edge-util", "flate2", "fuse", @@ -6948,7 +6910,7 @@ version = "0.9.0" dependencies = [ "anyhow", "async-trait", - "base64 0.21.7", + "base64", "bincode", "bytecheck", "bytes 1.6.1", @@ -7106,7 +7068,7 @@ dependencies = [ "ahash 0.8.11", "anyhow", "async-trait", - "base64 0.21.7", + "base64", "bincode", "blake3", "bytecheck", @@ -7416,7 +7378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6cf1915e27168c1ed82f7f0d03f60317648f257fecaf4256a1904428d8e4ccd" dependencies = [ "anyhow", - "base64 0.22.1", + "base64", "bytes 1.6.1", "cfg-if 1.0.0", "ciborium", From fbd14f9b3405b1106d423aa47a5f92db57c8f46f Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:27:16 +0200 Subject: [PATCH 16/53] chore: Fix warning in test --- tests/integration/cli/tests/publish.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/cli/tests/publish.rs b/tests/integration/cli/tests/publish.rs index c58a224a238..44393b0afa7 100644 --- a/tests/integration/cli/tests/publish.rs +++ b/tests/integration/cli/tests/publish.rs @@ -133,7 +133,7 @@ fn wasmer_init_publish() { .assert() .success(); - let s = std::fs::read_to_string(path.join("randomversion").join("wasmer.toml")).unwrap(); + let _s = std::fs::read_to_string(path.join("randomversion").join("wasmer.toml")).unwrap(); // publish let mut cmd = std::process::Command::new(get_wasmer_path()); From 2cd732e25507fbda00790b0e00f021ea12e1d349 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Sun, 15 Sep 2024 07:28:31 +0200 Subject: [PATCH 17/53] tests: Add WasiRunner file system mounting tests --- lib/wasix/src/runners/wasi.rs | 88 ++++++++++++++++++ ...-tests--volume-static-webserver@0.1.0.webc | Bin 0 -> 4634 bytes 2 files changed, 88 insertions(+) create mode 100644 tests/integration/cli/tests/webc/wasmer-tests--volume-static-webserver@0.1.0.webc diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 7713b80f6d2..9837b4a2102 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -448,4 +448,92 @@ mod tests { assert_send::(); assert_sync::(); } + + #[tokio::test] + async fn test_volume_mount_without_webcs() { + use std::sync::Arc; + + let root_fs = virtual_fs::RootFileSystemBuilder::new().build(); + + let tokrt = tokio::runtime::Handle::current(); + + let hostdir = virtual_fs::host_fs::FileSystem::new(tokrt.clone(), "/").unwrap(); + let hostdir_dyn: Arc = Arc::new(hostdir); + + root_fs + .mount("/host".into(), &hostdir_dyn, "/".into()) + .unwrap(); + + let envb = crate::runners::wasi::WasiRunner::new(); + + let annotations = webc::metadata::annotations::Wasi::new("test"); + + let tm = Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::new( + tokrt.clone(), + )); + let rt = crate::PluggableRuntime::new(tm); + + let envb = envb + .prepare_webc_env("test", &annotations, None, Arc::new(rt), Some(root_fs)) + .unwrap(); + + let init = envb.build_init().unwrap(); + + let fs = &init.state.fs.root_fs; + + fs.read_dir(&std::path::Path::new("/host")).unwrap(); + } + + #[tokio::test] + async fn test_volume_mount_with_webcs() { + use std::sync::Arc; + + let root_fs = virtual_fs::RootFileSystemBuilder::new().build(); + + let tokrt = tokio::runtime::Handle::current(); + + let hostdir = virtual_fs::host_fs::FileSystem::new(tokrt.clone(), "/").unwrap(); + let hostdir_dyn: Arc = Arc::new(hostdir); + + root_fs + .mount("/host".into(), &hostdir_dyn, "/".into()) + .unwrap(); + + let envb = crate::runners::wasi::WasiRunner::new(); + + let annotations = webc::metadata::annotations::Wasi::new("test"); + + let tm = Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::new( + tokrt.clone(), + )); + let mut rt = crate::PluggableRuntime::new(tm); + rt.set_package_loader(crate::runtime::package_loader::BuiltinPackageLoader::new()); + + let webc_path = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("../../tests/integration/cli/tests/webc/wasmer-tests--volume-static-webserver@0.1.0.webc"); + let webc_data = std::fs::read(webc_path).unwrap(); + let container = webc::Container::from_bytes(webc_data).unwrap(); + + let binpkg = crate::bin_factory::BinaryPackage::from_webc(&container, &rt) + .await + .unwrap(); + + let mut envb = envb + .prepare_webc_env( + "test", + &annotations, + Some(&binpkg), + Arc::new(rt), + Some(root_fs), + ) + .unwrap(); + + envb = envb.preopen_dir("/host").unwrap(); + + let init = envb.build_init().unwrap(); + + let fs = &init.state.fs.root_fs; + + fs.read_dir(&std::path::Path::new("/host")).unwrap(); + fs.read_dir(&std::path::Path::new("/settings")).unwrap(); + } } diff --git a/tests/integration/cli/tests/webc/wasmer-tests--volume-static-webserver@0.1.0.webc b/tests/integration/cli/tests/webc/wasmer-tests--volume-static-webserver@0.1.0.webc new file mode 100644 index 0000000000000000000000000000000000000000..ca28cc71e5dd45fc1ba9c7925f23eeb02fa9690d GIT binary patch literal 4634 zcmcIoZEPGz8E!s+_Gm#x6;0G89o^9;F`nDA)1;PLquNf=AneAqoune9&D`$n-q^c4 zo0-|WbCVW`1VX5gN~H>EL8Wdae$Yk;@gosR#79t(kRm{ZKM1t~eu0qqEK;ci?>oDD zK7U~&760(%zHXv@jN^+WwvhI>xMku zy5Y+Wx53?6oksJnVSy~ZQQctud-n1s3se3m>3iq&c6I5!pTGHsKm7yQ_s8D&%`IDB zPp*9E(hq+7`jy+hevUIEBfVI~rfaBT%M@1DM+2mX5I&#wec8Kx1}^TN^G7!e0c)NTHl_2`z#_37L6fm?qrX3mXYr~iEG zxs#Vg-(QYgKhf{~`l+|;-)c-94jK_R%wvWvcIwt=tW#fHpOQD8J~6F7^XaF0qRn?b zcI}Z*T>09Ov)}yWZ;r)-2VUIvs`{JIB8Ztu6}QokJ6~xmW1a>7!7P#g&;J|NOz(zF z{RKDG{H6Am7Sj<|4nWBWmjdRf=Bmk6`;OI49(=4XNSHgz0zU7xSQ34s?(5IK?D$>_oeg6WJ?J=C0#&I3 zCp&P!Br(h4SfaHu2G}52=RK2Sfmd99+@bxU^jrktbW(T{V&c3sF?<@4Vs zUwiw~0KNXFr@!~XtuMdvqw(6v_tR%CKJYX00H4@$-#IY04opTRfI^0(!I{UEF1bjh;44yTP6b}0QeGIB8VP(+ zf+hmYOHe4q^WvNPQ6EX55_PFU9+rj=l8^g@5yeBOkS2#S(;z|u-hibz_Ap{7>dur{B3=cUu3+}<` z=?SNwe3VR&O@Uch%>8iFXQAfBcWOdSU2el%E-#|a@;Q`S5H(diGiZs7g#%jTE|e|{ z@yl{%itUMZ5}Rx6l3~x0SEKeWNtsilhbE^0E>ql1l`wOid{&6gNlTMiA+A)KvM`h_ zDwP1ZSkVPO_7zbkq0l%5OP#pWoC20~;bIvB(STE61VtWS&J9X2twW?Q2i9@{h(qXF z#Fr{!W(5)10Y)TCd=`um=$$SZP7zQrHf%G7h}Mxz8IX~o@biEOIq`Vb;{mR95qs!u zh4^0gy`xgiS`;cM6>lS6Sh zmt$u#&C&#$0aQM2N`z?uUvYH`Xu`!l*UFgfJ_VGHp`~NYYobqHF^#Zc5(qgC2m*T< zEED1hTU2BxZ?hnWl^quA$S4#TrLKhZ|KDD)#QK-PaFV?gw1DVe0Xzw7yRwQuO^;RiR{F(c|0| zVgdkwM#$UHyIdH$IM#N0(irYx#jH15Nzgl!@Srz|gI?jv-az=>xvRdLu3*)X!;@ps z;|Q3^zAE^KnVAKUKoWM>aAvJX z)e=-KQB%A~EPK7F#@sFm9ew>4-MRNI81YbY%ZqQZMcK6#t88hyme}Pm4X?i1@fkD= z8CT&ScwZrh5`3>=Aq<-=1GxTGI^R}_%zv?CW*s@uEuDqm=KqLLCIt%)!YYJgdT|<> z*{R*QY}PAeBsBzvAa0LkWGd&o5!9-I0ixkV=xVhJf^faU64MMiS~YKNRTxIrdAMUY z)ND2sOR8Z=1m4~uqe|hwv_HfEwi~}M;*5JC1xxZiWhR3 Xo7egUfaNb9H|C$kuMY3}yWM{PLKf;w literal 0 HcmV?d00001 From e55159566a5621a0f1116eda3c521c9f75bd9ed0 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Mon, 16 Sep 2024 09:49:24 +0200 Subject: [PATCH 18/53] tests: Feature gate test --- lib/wasix/src/runners/wasi.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 9837b4a2102..f69f7e9ae1d 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -449,6 +449,7 @@ mod tests { assert_sync::(); } + #[cfg(all(feature = "host-fs", feature = "sys"))] #[tokio::test] async fn test_volume_mount_without_webcs() { use std::sync::Arc; @@ -484,6 +485,7 @@ mod tests { fs.read_dir(&std::path::Path::new("/host")).unwrap(); } + #[cfg(all(feature = "host-fs", feature = "sys"))] #[tokio::test] async fn test_volume_mount_with_webcs() { use std::sync::Arc; From bc81277a13aadbcdf2fdf6f080ac90b8258ee73f Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 13:36:04 +0330 Subject: [PATCH 19/53] fix issues with windows --- lib/virtual-net/src/host.rs | 69 +++++++++++++++---------------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/lib/virtual-net/src/host.rs b/lib/virtual-net/src/host.rs index de9ca4dbca1..3fd1000d7bc 100644 --- a/lib/virtual-net/src/host.rs +++ b/lib/virtual-net/src/host.rs @@ -91,21 +91,6 @@ impl VirtualNetworking for LocalNetworking { ) -> Result> { let socket = mio::net::UdpSocket::bind(addr).map_err(io_err_into_net_error)?; - { - #[cfg(not(windows))] - let sockref = socket2::SockRef::from(&socket); - #[cfg(windows)] - let sockref = unsafe { - socket2::SockRef::from(&std::os::windows::io::BorrowedSocket::borrow_raw( - socket.stream.as_raw_socket(), - )) - .set_nonblocking(true) - .ok(); - }; - - sockref.set_nonblocking(true).ok(); - } - #[allow(unused_mut)] let mut ret = LocalUdpSocket { selector: self.selector.clone(), @@ -136,7 +121,6 @@ impl VirtualNetworking for LocalNetworking { ) -> Result> { let stream = mio::net::TcpStream::connect(peer).map_err(io_err_into_net_error)?; - socket2::SockRef::from(&stream).set_nonblocking(true).ok(); if let Ok(p) = stream.peer_addr() { peer = p; } @@ -179,9 +163,6 @@ impl LocalTcpListener { fn try_accept_internal(&mut self) -> Result<(Box, SocketAddr)> { match self.stream.accept().map_err(io_err_into_net_error) { Ok((stream, addr)) => { - socket2::SockRef::from(&self.stream) - .set_nonblocking(true) - .ok(); let mut socket = LocalTcpStream::new(self.selector.clone(), stream, addr); if let Some(no_delay) = self.no_delay { socket.set_nodelay(no_delay).ok(); @@ -352,18 +333,21 @@ impl LocalTcpStream { ret } - fn sock_ref<'a>(&'a self) -> socket2::SockRef<'a> { + fn with_sock_ref(&self, f: F) -> R + where + for<'a> F: FnOnce(socket2::SockRef<'a>) -> R, + { #[cfg(not(windows))] let r = socket2::SockRef::from(&self.stream); #[cfg(windows)] - let r = unsafe { - socket2::SockRef::from(&std::os::windows::io::BorrowedSocket::borrow_raw( - self.stream.as_raw_socket(), - )) + let b = unsafe { + std::os::windows::io::BorrowedSocket::borrow_raw(self.stream.as_raw_socket()) }; + #[cfg(windows)] + let r = socket2::SockRef::from(&b); - r + f(r) } } @@ -395,14 +379,15 @@ impl VirtualTcpSocket for LocalTcpStream { } fn set_keepalive(&mut self, keepalive: bool) -> Result<()> { - self.sock_ref() - .set_keepalive(true) + self.with_sock_ref(|s| s.set_keepalive(true)) .map_err(io_err_into_net_error)?; Ok(()) } fn keepalive(&self) -> Result { - let ret = self.sock_ref().keepalive().map_err(io_err_into_net_error)?; + let ret = self + .with_sock_ref(|s| s.keepalive()) + .map_err(io_err_into_net_error)?; Ok(ret) } @@ -474,15 +459,14 @@ impl VirtualTcpSocket for LocalTcpStream { impl VirtualConnectedSocket for LocalTcpStream { fn set_linger(&mut self, linger: Option) -> Result<()> { - self.sock_ref() - .set_linger(linger) + self.with_sock_ref(|s| s.set_linger(linger)) .map_err(io_err_into_net_error)?; Ok(()) } fn linger(&self) -> Result> { - let sockref = self.sock_ref(); - sockref.linger().map_err(io_err_into_net_error) + self.with_sock_ref(|s| s.linger()) + .map_err(io_err_into_net_error) } fn try_send(&mut self, data: &[u8]) -> Result { @@ -684,18 +668,21 @@ pub struct LocalUdpSocket { } impl LocalUdpSocket { - fn sock_ref<'a>(&'a self) -> socket2::SockRef<'a> { + fn with_sock_ref(&self, f: F) -> R + where + for<'a> F: FnOnce(socket2::SockRef<'a>) -> R, + { #[cfg(not(windows))] let r = socket2::SockRef::from(&self.socket); #[cfg(windows)] - let r = unsafe { - socket2::SockRef::from(&std::os::windows::io::BorrowedSocket::borrow_raw( - self.stream.as_raw_socket(), - )) + let b = unsafe { + std::os::windows::io::BorrowedSocket::borrow_raw(self.socket.as_raw_socket()) }; + #[cfg(windows)] + let r = socket2::SockRef::from(&b); - r + f(r) } } @@ -747,14 +734,12 @@ impl VirtualUdpSocket for LocalUdpSocket { } fn join_multicast_v4(&mut self, multiaddr: Ipv4Addr, iface: Ipv4Addr) -> Result<()> { - self.sock_ref() - .join_multicast_v4(&multiaddr, &iface) + self.with_sock_ref(|s| s.join_multicast_v4(&multiaddr, &iface)) .map_err(io_err_into_net_error) } fn leave_multicast_v4(&mut self, multiaddr: Ipv4Addr, iface: Ipv4Addr) -> Result<()> { - self.sock_ref() - .leave_multicast_v4(&multiaddr, &iface) + self.with_sock_ref(|s| s.leave_multicast_v4(&multiaddr, &iface)) .map_err(io_err_into_net_error) } From 13367830424fbf199d194037f34ccf1a488004d4 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Mon, 16 Sep 2024 12:50:07 +0200 Subject: [PATCH 20/53] chore: Prevent create_dir_all in a loop --- lib/package/src/convert/webc_to_package.rs | 38 ++++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs index a3aaacd8269..22edafa39c1 100644 --- a/lib/package/src/convert/webc_to_package.rs +++ b/lib/package/src/convert/webc_to_package.rs @@ -131,33 +131,35 @@ pub fn webc_to_package_dir( let module_dir_name = "modules"; let module_dir = target_dir.join(module_dir_name); - for (atom_name, data) in webc.atoms() { - let atom_path = module_dir.join(&atom_name); - + let atoms = webc.atoms(); + if !atoms.is_empty() { std::fs::create_dir_all(&module_dir).map_err(|err| { ConversionError::with_cause( format!("Could not create directory '{}'", module_dir.display(),), err, ) })?; + for (atom_name, data) in atoms { + let atom_path = module_dir.join(&atom_name); - std::fs::write(&atom_path, &data).map_err(|err| { - ConversionError::with_cause( - format!("Could not write atom to path '{}'", atom_path.display()), - err, - ) - })?; + std::fs::write(&atom_path, &data).map_err(|err| { + ConversionError::with_cause( + format!("Could not write atom to path '{}'", atom_path.display()), + err, + ) + })?; - let relative_path = format!("./{module_dir_name}/{atom_name}"); + let relative_path = format!("./{module_dir_name}/{atom_name}"); - pkg_manifest.modules.push(wasmer_config::package::Module { - name: atom_name, - source: relative_path.into(), - abi: wasmer_config::package::Abi::None, - kind: None, - interfaces: None, - bindings: None, - }); + pkg_manifest.modules.push(wasmer_config::package::Module { + name: atom_name, + source: relative_path.into(), + abi: wasmer_config::package::Abi::None, + kind: None, + interfaces: None, + bindings: None, + }); + } } // Convert commands. From 20c29a68c7903dcf182bb2721184fdf404f6704e Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 19:06:41 +0330 Subject: [PATCH 21/53] update webc dep --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3849b3ec7b3..63753b4cd4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7411,9 +7411,9 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.0" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6cf1915e27168c1ed82f7f0d03f60317648f257fecaf4256a1904428d8e4ccd" +checksum = "c48441419be082f8d2537c84d8b1f502624d77bc08fbbd09ab17cadfe7f0ac53" dependencies = [ "anyhow", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index bff17a4d1dd..e1d8742bb82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,7 +92,7 @@ wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.0.0", default-features = false, features = ["package"] } +webc = { version = "6.0.1", default-features = false, features = ["package"] } edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" From 0df236746548ed001bae41c8a08d8eca45c703d5 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 20:30:49 +0330 Subject: [PATCH 22/53] apply the proxy setting in wasmer config --- lib/backend-api/src/client.rs | 26 +++++-------------- lib/cli/src/commands/package/download.rs | 16 ++---------- lib/cli/src/config/env.rs | 33 ++++++++++-------------- lib/cli/src/config/mod.rs | 6 ++++- tests/wasmer-argus/src/argus/mod.rs | 2 +- 5 files changed, 29 insertions(+), 54 deletions(-) diff --git a/lib/backend-api/src/client.rs b/lib/backend-api/src/client.rs index 6407ab0edd1..46ddaeaf220 100644 --- a/lib/backend-api/src/client.rs +++ b/lib/backend-api/src/client.rs @@ -75,10 +75,10 @@ impl WasmerClient { }) } - pub fn new_with_proxy( + pub fn new( graphql_endpoint: Url, user_agent: &str, - proxy: reqwest::Proxy, + proxy: Option, ) -> Result { let builder = { #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] @@ -89,7 +89,11 @@ impl WasmerClient { .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(90)); - builder.proxy(proxy) + if let Some(proxy) = proxy { + builder.proxy(proxy) + } else { + builder + } }; let client = builder.build().context("failed to create reqwest client")?; @@ -97,22 +101,6 @@ impl WasmerClient { Self::new_with_client(client, graphql_endpoint, user_agent) } - pub fn new(graphql_endpoint: Url, user_agent: &str) -> Result { - #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] - let client = reqwest::Client::builder() - .build() - .context("could not construct http client")?; - - #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(90)) - .build() - .context("could not construct http client")?; - - Self::new_with_client(client, graphql_endpoint, user_agent) - } - pub fn with_auth_token(mut self, auth_token: String) -> Self { self.auth_token = Some(auth_token); self diff --git a/lib/cli/src/commands/package/download.rs b/lib/cli/src/commands/package/download.rs index bc168fae95e..97cc9650a4a 100644 --- a/lib/cli/src/commands/package/download.rs +++ b/lib/cli/src/commands/package/download.rs @@ -5,7 +5,6 @@ use dialoguer::console::{style, Emoji}; use indicatif::{ProgressBar, ProgressStyle}; use tempfile::NamedTempFile; use wasmer_config::package::{PackageIdent, PackageSource}; -use wasmer_wasix::http::reqwest::get_proxy; use crate::config::WasmerEnv; @@ -28,10 +27,6 @@ pub struct PackageDownload { #[clap(long)] pub quiet: bool, - /// proxy to use for downloading - #[clap(long)] - pub proxy: Option, - /// The package to download. package: PackageSource, } @@ -101,13 +96,7 @@ impl PackageDownload { // caveat: client_unauthennticated will use a token if provided, it // just won't fail if none is present. So, _unauthenticated() can actually // produce an authenticated client. - let client = if let Some(proxy) = &self.proxy { - let proxy = reqwest::Proxy::all(proxy)?; - - self.env.client_unauthennticated_with_proxy(proxy)? - } else { - self.env.client_unauthennticated()? - }; + let client = self.env.client_unauthennticated()?; let version = id.version_or_default().to_string(); let version = if version == "*" { @@ -171,7 +160,7 @@ impl PackageDownload { let builder = { let mut builder = reqwest::blocking::ClientBuilder::new(); - if let Some(proxy) = get_proxy()? { + if let Some(proxy) = self.env.proxy()? { builder = builder.proxy(proxy); } builder @@ -309,7 +298,6 @@ mod tests { out_path: Some(out_path.clone()), package: "wasmer/hello@0.1.0".parse().unwrap(), quiet: true, - proxy: None, }; cmd.execute().unwrap(); diff --git a/lib/cli/src/config/env.rs b/lib/cli/src/config/env.rs index 496962c14f7..f202d7eed7c 100644 --- a/lib/cli/src/config/env.rs +++ b/lib/cli/src/config/env.rs @@ -91,6 +91,17 @@ impl WasmerEnv { }) } + /// Returns the proxy specified in wasmer config if present + pub fn proxy(&self) -> Result, Error> { + self.config()? + .proxy + .url + .as_ref() + .map(reqwest::Proxy::all) + .transpose() + .map_err(Into::into) + } + /// The directory all Wasmer artifacts are stored in. pub fn dir(&self) -> &Path { &self.wasmer_dir @@ -127,27 +138,11 @@ impl WasmerEnv { pub fn client_unauthennticated(&self) -> Result { let registry_url = self.registry_endpoint()?; - let client = wasmer_api::WasmerClient::new(registry_url, &DEFAULT_WASMER_CLI_USER_AGENT)?; - let client = if let Some(token) = self.token() { - client.with_auth_token(token) - } else { - client - }; + let proxy = self.proxy()?; - Ok(client) - } - - pub fn client_unauthennticated_with_proxy( - &self, - proxy: reqwest::Proxy, - ) -> Result { - let registry_url = self.registry_endpoint()?; - let client = wasmer_api::WasmerClient::new_with_proxy( - registry_url, - &DEFAULT_WASMER_CLI_USER_AGENT, - proxy, - )?; + let client = + wasmer_api::WasmerClient::new(registry_url, &DEFAULT_WASMER_CLI_USER_AGENT, proxy)?; let client = if let Some(token) = self.token() { client.with_auth_token(token) diff --git a/lib/cli/src/config/mod.rs b/lib/cli/src/config/mod.rs index 23c95f21f3a..8ecb38df7db 100644 --- a/lib/cli/src/config/mod.rs +++ b/lib/cli/src/config/mod.rs @@ -110,7 +110,11 @@ fn endpoint_from_domain_name(domain_name: &str) -> String { } async fn test_if_registry_present(registry: &str) -> anyhow::Result<()> { - let client = WasmerClient::new(url::Url::parse(registry)?, &DEFAULT_WASMER_CLI_USER_AGENT)?; + let client = WasmerClient::new( + url::Url::parse(registry)?, + &DEFAULT_WASMER_CLI_USER_AGENT, + None, + )?; wasmer_api::query::current_user(&client).await.map(|_| ()) } diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs index 63260730592..fde65e51e38 100644 --- a/tests/wasmer-argus/src/argus/mod.rs +++ b/tests/wasmer-argus/src/argus/mod.rs @@ -28,7 +28,7 @@ impl TryFrom for Argus { type Error = anyhow::Error; fn try_from(config: ArgusConfig) -> Result { - let client = WasmerClient::new(Url::parse(&config.registry_url)?, "wasmer-argus")?; + let client = WasmerClient::new(Url::parse(&config.registry_url)?, "wasmer-argus", None)?; let client = client.with_auth_token(config.auth_token.clone()); Ok(Argus { client, config }) From f0506e1933ac4612eaf932702b4dc36e4c0685e0 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 21:08:41 +0330 Subject: [PATCH 23/53] propagate proxy config to upload --- lib/cli/src/commands/package/common/mod.rs | 19 ++++++++++++++----- lib/cli/src/commands/package/push.rs | 10 +++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/cli/src/commands/package/common/mod.rs b/lib/cli/src/commands/package/common/mod.rs index 5641f96bfde..76f3e2c0bf6 100644 --- a/lib/cli/src/commands/package/common/mod.rs +++ b/lib/cli/src/commands/package/common/mod.rs @@ -44,6 +44,7 @@ pub(super) async fn upload( timeout: humantime::Duration, package: &Package, pb: ProgressBar, + proxy: Option, ) -> anyhow::Result { let hash_str = hash.to_string(); let hash_str = hash_str.trim_start_matches("sha256:"); @@ -68,11 +69,19 @@ pub(super) async fn upload( tracing::info!("signed url is: {session_uri}"); - let client = reqwest::Client::builder() - .default_headers(reqwest::header::HeaderMap::default()) - .timeout(timeout.into()) - .build() - .unwrap(); + let client = { + let builder = reqwest::Client::builder() + .default_headers(reqwest::header::HeaderMap::default()) + .timeout(timeout.into()); + + let builder = if let Some(proxy) = proxy { + builder.proxy(proxy) + } else { + builder + }; + + builder.build().unwrap() + }; let res = client .post(&session_uri) diff --git a/lib/cli/src/commands/package/push.rs b/lib/cli/src/commands/package/push.rs index 03de8e9aad7..7c2628949ea 100644 --- a/lib/cli/src/commands/package/push.rs +++ b/lib/cli/src/commands/package/push.rs @@ -128,7 +128,15 @@ impl PackagePush { ) -> anyhow::Result<()> { let pb = make_spinner!(self.quiet, "Uploading the package.."); - let signed_url = upload(client, package_hash, self.timeout, package, pb.clone()).await?; + let signed_url = upload( + client, + package_hash, + self.timeout, + package, + pb.clone(), + self.env.proxy()?, + ) + .await?; spinner_ok!(pb, "Package correctly uploaded"); let pb = make_spinner!(self.quiet, "Waiting for package to become available..."); From 95e46782d9b49abce4b23e57ebd34f7393cfcb71 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 21:28:42 +0330 Subject: [PATCH 24/53] prevent api breakage --- lib/backend-api/src/client.rs | 6 +++++- lib/cli/src/config/env.rs | 2 +- lib/cli/src/config/mod.rs | 6 +----- tests/wasmer-argus/src/argus/mod.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/backend-api/src/client.rs b/lib/backend-api/src/client.rs index 46ddaeaf220..d3579939f72 100644 --- a/lib/backend-api/src/client.rs +++ b/lib/backend-api/src/client.rs @@ -75,7 +75,11 @@ impl WasmerClient { }) } - pub fn new( + pub fn new(graphql_endpoint: Url, user_agent: &str) -> Result { + Self::new_with_proxy(graphql_endpoint, user_agent, None) + } + + pub fn new_with_proxy( graphql_endpoint: Url, user_agent: &str, proxy: Option, diff --git a/lib/cli/src/config/env.rs b/lib/cli/src/config/env.rs index f202d7eed7c..e5dfc052dc9 100644 --- a/lib/cli/src/config/env.rs +++ b/lib/cli/src/config/env.rs @@ -142,7 +142,7 @@ impl WasmerEnv { let proxy = self.proxy()?; let client = - wasmer_api::WasmerClient::new(registry_url, &DEFAULT_WASMER_CLI_USER_AGENT, proxy)?; + wasmer_api::WasmerClient::new_with_proxy(registry_url, &DEFAULT_WASMER_CLI_USER_AGENT, proxy)?; let client = if let Some(token) = self.token() { client.with_auth_token(token) diff --git a/lib/cli/src/config/mod.rs b/lib/cli/src/config/mod.rs index 8ecb38df7db..23c95f21f3a 100644 --- a/lib/cli/src/config/mod.rs +++ b/lib/cli/src/config/mod.rs @@ -110,11 +110,7 @@ fn endpoint_from_domain_name(domain_name: &str) -> String { } async fn test_if_registry_present(registry: &str) -> anyhow::Result<()> { - let client = WasmerClient::new( - url::Url::parse(registry)?, - &DEFAULT_WASMER_CLI_USER_AGENT, - None, - )?; + let client = WasmerClient::new(url::Url::parse(registry)?, &DEFAULT_WASMER_CLI_USER_AGENT)?; wasmer_api::query::current_user(&client).await.map(|_| ()) } diff --git a/tests/wasmer-argus/src/argus/mod.rs b/tests/wasmer-argus/src/argus/mod.rs index fde65e51e38..63260730592 100644 --- a/tests/wasmer-argus/src/argus/mod.rs +++ b/tests/wasmer-argus/src/argus/mod.rs @@ -28,7 +28,7 @@ impl TryFrom for Argus { type Error = anyhow::Error; fn try_from(config: ArgusConfig) -> Result { - let client = WasmerClient::new(Url::parse(&config.registry_url)?, "wasmer-argus", None)?; + let client = WasmerClient::new(Url::parse(&config.registry_url)?, "wasmer-argus")?; let client = client.with_auth_token(config.auth_token.clone()); Ok(Argus { client, config }) From fe1b72a3b23722dbf4766e837ee88d558bf68197 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 21:35:00 +0330 Subject: [PATCH 25/53] fix lint --- lib/cli/src/config/env.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/cli/src/config/env.rs b/lib/cli/src/config/env.rs index e5dfc052dc9..a328f66d40c 100644 --- a/lib/cli/src/config/env.rs +++ b/lib/cli/src/config/env.rs @@ -141,8 +141,11 @@ impl WasmerEnv { let proxy = self.proxy()?; - let client = - wasmer_api::WasmerClient::new_with_proxy(registry_url, &DEFAULT_WASMER_CLI_USER_AGENT, proxy)?; + let client = wasmer_api::WasmerClient::new_with_proxy( + registry_url, + &DEFAULT_WASMER_CLI_USER_AGENT, + proxy, + )?; let client = if let Some(token) = self.token() { client.with_auth_token(token) From 3437597505b09a2873bb99cf4b635a8b07844b5f Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 22:54:35 +0330 Subject: [PATCH 26/53] replace WASM_ANYREF with WASM_EXTERNREF --- lib/c-api/src/wasm_c_api/types/value.rs | 6 +++--- lib/c-api/src/wasm_c_api/value.rs | 10 +++++----- lib/c-api/tests/wasm-c-api/example/callback.c | 2 +- lib/c-api/tests/wasm-c-api/example/hostref.c | 8 ++++---- lib/c-api/tests/wasm-c-api/example/reflect.c | 2 +- lib/c-api/tests/wasm-c-api/include/wasm.h | 14 +++++++------- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/lib/c-api/src/wasm_c_api/types/value.rs b/lib/c-api/src/wasm_c_api/types/value.rs index ae2189c3117..96712f369c2 100644 --- a/lib/c-api/src/wasm_c_api/types/value.rs +++ b/lib/c-api/src/wasm_c_api/types/value.rs @@ -10,7 +10,7 @@ pub enum wasm_valkind_enum { WASM_I64 = 1, WASM_F32 = 2, WASM_F64 = 3, - WASM_ANYREF = 128, + WASM_EXTERNREF = 128, WASM_FUNCREF = 129, } @@ -22,7 +22,7 @@ impl From for wasm_valkind_enum { Type::F32 => Self::WASM_F32, Type::F64 => Self::WASM_F64, Type::V128 => todo!("no v128 type in Wasm C API yet!"), - Type::ExternRef => Self::WASM_ANYREF, + Type::ExternRef => Self::WASM_EXTERNREF, Type::FuncRef => Self::WASM_FUNCREF, } } @@ -36,7 +36,7 @@ impl From for Type { WASM_I64 => Type::I64, WASM_F32 => Type::F32, WASM_F64 => Type::F64, - WASM_ANYREF => Type::ExternRef, + WASM_EXTERNREF => Type::ExternRef, WASM_FUNCREF => Type::FuncRef, } } diff --git a/lib/c-api/src/wasm_c_api/value.rs b/lib/c-api/src/wasm_c_api/value.rs index 737975fa881..823fec8d861 100644 --- a/lib/c-api/src/wasm_c_api/value.rs +++ b/lib/c-api/src/wasm_c_api/value.rs @@ -9,7 +9,7 @@ use wasmer_api::Value; /// * `WASM_I64`, a 64-bit integer, /// * `WASM_F32`, a 32-bit float, /// * `WASM_F64`, a 64-bit float, -/// * `WASM_ANYREF`, a WebAssembly reference, +/// * `WASM_EXTERNREF`, a WebAssembly reference, /// * `WASM_FUNCREF`, a WebAssembly reference. #[allow(non_camel_case_types)] pub type wasm_valkind_t = u8; @@ -94,7 +94,7 @@ impl std::fmt::Debug for wasm_val_t { Ok(wasm_valkind_enum::WASM_F64) => { ds.field("f64", &unsafe { self.of.float64_t }); } - Ok(wasm_valkind_enum::WASM_ANYREF) => { + Ok(wasm_valkind_enum::WASM_EXTERNREF) => { ds.field("anyref", &unsafe { self.of.wref }); } @@ -150,7 +150,7 @@ pub unsafe extern "C" fn wasm_val_copy( wasm_valkind_enum::WASM_F64 => wasm_val_inner { float64_t: val.of.float64_t, }, - wasm_valkind_enum::WASM_ANYREF => wasm_val_inner { wref: val.of.wref }, + wasm_valkind_enum::WASM_EXTERNREF => wasm_val_inner { wref: val.of.wref }, wasm_valkind_enum::WASM_FUNCREF => wasm_val_inner { wref: val.of.wref }, } }); otherwise ()); @@ -173,7 +173,7 @@ impl TryFrom for wasm_valkind_enum { 1 => wasm_valkind_enum::WASM_I64, 2 => wasm_valkind_enum::WASM_F32, 3 => wasm_valkind_enum::WASM_F64, - 128 => wasm_valkind_enum::WASM_ANYREF, + 128 => wasm_valkind_enum::WASM_EXTERNREF, 129 => wasm_valkind_enum::WASM_FUNCREF, _ => return Err("valkind value out of bounds"), }) @@ -197,7 +197,7 @@ impl TryFrom<&wasm_val_t> for Value { wasm_valkind_enum::WASM_I64 => Value::I64(unsafe { item.of.int64_t }), wasm_valkind_enum::WASM_F32 => Value::F32(unsafe { item.of.float32_t }), wasm_valkind_enum::WASM_F64 => Value::F64(unsafe { item.of.float64_t }), - wasm_valkind_enum::WASM_ANYREF => return Err("ANYREF not supported at this time"), + wasm_valkind_enum::WASM_EXTERNREF => return Err("EXTERNREF not supported at this time"), wasm_valkind_enum::WASM_FUNCREF => return Err("FUNCREF not supported at this time"), }) } diff --git a/lib/c-api/tests/wasm-c-api/example/callback.c b/lib/c-api/tests/wasm-c-api/example/callback.c index 2fddb1634ff..a575397463e 100644 --- a/lib/c-api/tests/wasm-c-api/example/callback.c +++ b/lib/c-api/tests/wasm-c-api/example/callback.c @@ -22,7 +22,7 @@ void wasm_val_print(wasm_val_t val) { case WASM_F64: { printf("%g", val.of.f64); } break; - case WASM_ANYREF: + case WASM_EXTERNREF: case WASM_FUNCREF: { if (val.of.ref == NULL) { printf("null"); diff --git a/lib/c-api/tests/wasm-c-api/example/hostref.c b/lib/c-api/tests/wasm-c-api/example/hostref.c index 1e787ab1902..ae23584033b 100644 --- a/lib/c-api/tests/wasm-c-api/example/hostref.c +++ b/lib/c-api/tests/wasm-c-api/example/hostref.c @@ -159,7 +159,7 @@ int main(int argc, const char* argv[]) { // Create external callback function. printf("Creating callback...\n"); own wasm_functype_t* callback_type = wasm_functype_new_1_1( - wasm_valtype_new(WASM_ANYREF), wasm_valtype_new(WASM_ANYREF)); + wasm_valtype_new(WASM_EXTERNREF), wasm_valtype_new(WASM_EXTERNREF)); own wasm_func_t* callback_func = wasm_func_new(store, callback_type, callback); @@ -207,7 +207,7 @@ int main(int argc, const char* argv[]) { check(wasm_ref_copy(host2), host2); own wasm_val_t val; - val.kind = WASM_ANYREF; + val.kind = WASM_EXTERNREF; val.of.ref = wasm_ref_copy(host1); check(wasm_ref_copy(val.of.ref), host1); own wasm_ref_t* ref = val.of.ref; @@ -225,13 +225,13 @@ int main(int argc, const char* argv[]) { check(call_v_r(global_get), NULL); wasm_global_get(global, &val); - assert(val.kind == WASM_ANYREF); + assert(val.kind == WASM_EXTERNREF); check(val.of.ref, NULL); val.of.ref = host2; wasm_global_set(global, &val); check(call_v_r(global_get), host2); wasm_global_get(global, &val); - assert(val.kind == WASM_ANYREF); + assert(val.kind == WASM_EXTERNREF); check(val.of.ref, host2); printf("Accessing table...\n"); diff --git a/lib/c-api/tests/wasm-c-api/example/reflect.c b/lib/c-api/tests/wasm-c-api/example/reflect.c index d4383185ff0..0a7099fe07e 100644 --- a/lib/c-api/tests/wasm-c-api/example/reflect.c +++ b/lib/c-api/tests/wasm-c-api/example/reflect.c @@ -25,7 +25,7 @@ void print_valtype(const wasm_valtype_t* type) { case WASM_I64: printf("i64"); break; case WASM_F32: printf("f32"); break; case WASM_F64: printf("f64"); break; - case WASM_ANYREF: printf("anyref"); break; + case WASM_EXTERNREF: printf("externref"); break; case WASM_FUNCREF: printf("funcref"); break; } } diff --git a/lib/c-api/tests/wasm-c-api/include/wasm.h b/lib/c-api/tests/wasm-c-api/include/wasm.h index 7b38fe33e16..58115fa386d 100644 --- a/lib/c-api/tests/wasm-c-api/include/wasm.h +++ b/lib/c-api/tests/wasm-c-api/include/wasm.h @@ -186,7 +186,7 @@ enum wasm_valkind_enum { WASM_I64, WASM_F32, WASM_F64, - WASM_ANYREF = 128, + WASM_EXTERNREF = 128, WASM_FUNCREF, }; @@ -195,10 +195,10 @@ WASM_API_EXTERN own wasm_valtype_t* wasm_valtype_new(wasm_valkind_t); WASM_API_EXTERN wasm_valkind_t wasm_valtype_kind(const wasm_valtype_t*); static inline bool wasm_valkind_is_num(wasm_valkind_t k) { - return k < WASM_ANYREF; + return k < WASM_EXTERNREF ; } static inline bool wasm_valkind_is_ref(wasm_valkind_t k) { - return k >= WASM_ANYREF; + return k >= WASM_EXTERNREF ; } static inline bool wasm_valtype_is_num(const wasm_valtype_t* t) { @@ -551,8 +551,8 @@ static inline own wasm_valtype_t* wasm_valtype_new_f64() { return wasm_valtype_new(WASM_F64); } -static inline own wasm_valtype_t* wasm_valtype_new_anyref() { - return wasm_valtype_new(WASM_ANYREF); +static inline own wasm_valtype_t* wasm_valtype_new_externref() { + return wasm_valtype_new(WASM_EXTERNREF); } static inline own wasm_valtype_t* wasm_valtype_new_funcref() { return wasm_valtype_new(WASM_FUNCREF); @@ -712,8 +712,8 @@ static inline void* wasm_val_ptr(const wasm_val_t* val) { #define WASM_I64_VAL(i) {.kind = WASM_I64, .of = {.i64 = i}} #define WASM_F32_VAL(z) {.kind = WASM_F32, .of = {.f32 = z}} #define WASM_F64_VAL(z) {.kind = WASM_F64, .of = {.f64 = z}} -#define WASM_REF_VAL(r) {.kind = WASM_ANYREF, .of = {.ref = r}} -#define WASM_INIT_VAL {.kind = WASM_ANYREF, .of = {.ref = NULL}} +#define WASM_REF_VAL(r) {.kind = WASM_EXTERNREF, .of = {.ref = r}} +#define WASM_INIT_VAL {.kind = WASM_EXTERNREF, .of = {.ref = NULL}} /////////////////////////////////////////////////////////////////////////////// From 9e60dcbcbc5a26c98c6de4284ca96c7eff5ef2e7 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 16 Sep 2024 23:01:06 +0330 Subject: [PATCH 27/53] fix lint --- lib/c-api/src/wasm_c_api/value.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/c-api/src/wasm_c_api/value.rs b/lib/c-api/src/wasm_c_api/value.rs index 823fec8d861..3be2f2a340b 100644 --- a/lib/c-api/src/wasm_c_api/value.rs +++ b/lib/c-api/src/wasm_c_api/value.rs @@ -197,7 +197,9 @@ impl TryFrom<&wasm_val_t> for Value { wasm_valkind_enum::WASM_I64 => Value::I64(unsafe { item.of.int64_t }), wasm_valkind_enum::WASM_F32 => Value::F32(unsafe { item.of.float32_t }), wasm_valkind_enum::WASM_F64 => Value::F64(unsafe { item.of.float64_t }), - wasm_valkind_enum::WASM_EXTERNREF => return Err("EXTERNREF not supported at this time"), + wasm_valkind_enum::WASM_EXTERNREF => { + return Err("EXTERNREF not supported at this time") + } wasm_valkind_enum::WASM_FUNCREF => return Err("FUNCREF not supported at this time"), }) } From 75c23073443ca1d57bce6f2323efcb7e37b673ed Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 17 Sep 2024 02:48:39 +0330 Subject: [PATCH 28/53] add test for aio-http --- tests/integration/cli/Cargo.toml | 2 +- tests/integration/cli/tests/run.rs | 43 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index 76fa9dcf3f3..e96cab8350a 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -17,7 +17,7 @@ hex = "0.4.3" pretty_assertions.workspace = true object = "0.30.0" reqwest = { workspace = true, default-features = false, features = ["json", "blocking", "rustls-tls"] } -tokio = { workspace = true, features = [ "rt", "rt-multi-thread", "macros" ] } +tokio = { workspace = true, features = [ "rt", "rt-multi-thread", "macros", "process" ] } assert_cmd = "2.0.8" predicates = "2.1.5" once_cell = "1.17.1" diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index b43f9d3750f..baa0bd00f9d 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -47,6 +47,49 @@ static CACHE_RUST_LOG: Lazy = Lazy::new(|| { .join(",") }); +#[tokio::test] +async fn aio_http() { + let status = tokio::process::Command::new(get_wasmer_path()) + .kill_on_drop(true) + .arg("package") + .arg("download") + .arg("wasmer-integration-tests/aio-http-hello-world") + .arg("-o") + .arg("aio-http-hello-world.webc") + .arg("--quiet") + .spawn() + .unwrap() + .wait() + .await + .unwrap(); + + assert!(status.success()); + + let mut wasmer = tokio::process::Command::new(get_wasmer_path()) + .kill_on_drop(true) + .arg("run") + .arg("aio-http-hello-world.webc") + .arg("--net") + .stdout(Stdio::null()) + .spawn() + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + + let rsp = reqwest::Client::new() + .get("http://localhost:34343") + .send() + .await + .unwrap(); + + let body = rsp.text().await.unwrap(); + + assert_eq!(body, "Hello, World!"); + + wasmer.kill().await.unwrap(); + wasmer.wait().await.unwrap(); +} + #[test] fn list_cwd() { let package = packages().join("list-cwd"); From c87a42ca0e05c0593c1533983b1a18952d49242a Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Tue, 17 Sep 2024 08:13:48 +0200 Subject: [PATCH 29/53] deps: Upgrade flake dependencies --- flake.lock | 6 +++--- flake.nix | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 6c8dc250472..bad57fad8b4 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1712168706, - "narHash": "sha256-XP24tOobf6GGElMd0ux90FEBalUtw6NkBSVh/RlA6ik=", + "lastModified": 1720535198, + "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "1487bdea619e4a7a53a4590c475deabb5a9d1bfb", + "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 9cd483ac66f..f14e734dba1 100644 --- a/flake.nix +++ b/flake.nix @@ -68,10 +68,11 @@ ]; env.LLVM_SYS_150_PREFIX = pkgs.llvmPackages_15.llvm.dev; - env.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ - pkgs.stdenv.cc.cc - pkgs.openssl.out - ]; + + # shellHook = '' + # LD_LIBRARY_PATH = "${ env.LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc pkgs.openssl.out ] }:$LD_LIBRARY_PATH" + # ''; + }; } ); From e6ddbf25bbaf7f0f906fc52a541fbfe526504f44 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Tue, 17 Sep 2024 08:32:20 +0200 Subject: [PATCH 30/53] build: Update cargo-deny config Also updates the nix flake to pull in the new cargo deny. --- deny.toml | 211 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 120 insertions(+), 91 deletions(-) diff --git a/deny.toml b/deny.toml index 949f5721181..35a9d17436f 100644 --- a/deny.toml +++ b/deny.toml @@ -9,6 +9,11 @@ # The values provided in this template are the default values that will be used # when any section or field is not specified in your own configuration +# Root options + +# The graph table configures how the dependency graph is constructed and thus +# which crates the checks are performed against +[graph] # If 1 or more target triples (and optionally, target_features) are specified, # only the specified targets will be checked when running `cargo deny check`. # This means, if a particular package is only ever used as a target specific @@ -20,87 +25,85 @@ targets = [ # The triple can be any string, but only the target triples built in to # rustc (as of 1.40) can be checked against actual config expressions - #{ triple = "x86_64-unknown-linux-musl" }, + #"x86_64-unknown-linux-musl", # You can also specify which target_features you promise are enabled for a # particular target. target_features are currently not validated against # the actual valid features supported by the target architecture. #{ triple = "wasm32-unknown-unknown", features = ["atomics"] }, ] +# When creating the dependency graph used as the source of truth when checks are +# executed, this field can be used to prune crates from the graph, removing them +# from the view of cargo-deny. This is an extremely heavy hammer, as if a crate +# is pruned from the graph, all of its dependencies will also be pruned unless +# they are connected to another crate in the graph that hasn't been pruned, +# so it should be used with care. The identifiers are [Package ID Specifications] +# (https://doc.rust-lang.org/cargo/reference/pkgid-spec.html) +#exclude = [] +# If true, metadata will be collected with `--all-features`. Note that this can't +# be toggled off if true, if you want to conditionally enable `--all-features` it +# is recommended to pass `--all-features` on the cmd line instead +all-features = false +# If true, metadata will be collected with `--no-default-features`. The same +# caveat with `all-features` applies +no-default-features = false +# If set, these feature will be enabled when collecting metadata. If `--features` +# is specified on the cmd line they will take precedence over this option. +#features = [] + +# The output table provides options for how/if diagnostics are outputted +[output] +# When outputting inclusion graphs in diagnostics that include features, this +# option can be used to specify the depth at which feature edges will be added. +# This option is included since the graphs can be quite large and the addition +# of features from the crate(s) to all of the graph roots can be far too verbose. +# This option can be overridden via `--feature-depth` on the cmd line +feature-depth = 1 # This section is considered when running `cargo deny check advisories` # More documentation for the advisories section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html [advisories] -# The path where the advisory database is cloned/fetched into -db-path = "~/.cargo/advisory-db" +# The path where the advisory databases are cloned/fetched into +#db-path = "$CARGO_HOME/advisory-dbs" # The url(s) of the advisory databases to use -db-urls = ["https://github.com/rustsec/advisory-db"] -# The lint level for security vulnerabilities -vulnerability = "deny" -# The lint level for unmaintained crates -unmaintained = "warn" -# The lint level for crates that have been yanked from their source registry -yanked = "deny" -# The lint level for crates with security notices. Note that as of -# 2019-12-17 there are no security notice advisories in -# https://github.com/rustsec/advisory-db -notice = "warn" +#db-urls = ["https://github.com/rustsec/advisory-db"] # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ #"RUSTSEC-0000-0000", - + #{ id = "RUSTSEC-0000-0000", reason = "you can specify a reason the advisory is ignored" }, + #"a-crate-that-is-yanked@0.1.1", # you can also ignore yanked crate versions if you wish + #{ crate = "a-crate-that-is-yanked@0.1.1", reason = "you can specify why you are ignoring the yanked crate" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. # Setting this to true can be helpful if you have special authentication requirements that cargo-deny does not support. # See Git Authentication for more information about setting up git authentication. -git-fetch-with-cli = true -# Threshold for security vulnerabilities, any vulnerability with a CVSS score -# lower than the range specified will be ignored. Note that ignored advisories -# will still output a note when they are encountered. -# * None - CVSS Score 0.0 -# * Low - CVSS Score 0.1 - 3.9 -# * Medium - CVSS Score 4.0 - 6.9 -# * High - CVSS Score 7.0 - 8.9 -# * Critical - CVSS Score 9.0 - 10.0 -#severity-threshold = +#git-fetch-with-cli = true # This section is considered when running `cargo deny check licenses` # More documentation for the licenses section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html [licenses] -# The lint level for crates which do not have a detectable license -unlicensed = "deny" # List of explicitly allowed licenses # See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.7 short identifier (+ optional exception)]. +# [possible values: any SPDX 3.11 short identifier (+ optional exception)]. allow = [ "MIT", "Apache-2.0", "Apache-2.0 WITH LLVM-exception", + "ISC", + "BSD-2-Clause", + "MPL-2.0", + "0BSD", + "CC0-1.0", + "OpenSSL", + "BSD-3-Clause", + "Unicode-DFS-2016", + # BOOST source license + # NOTE: not to be confused with business source license! + "BSL-1.0", ] -# List of explicitly disallowed licenses -# See https://spdx.org/licenses/ for list of possible licenses -# [possible values: any SPDX 3.7 short identifier (+ optional exception)]. -deny = [ - "AGPL-1.0", - "AGPL-3.0", -] -# Lint level for licenses considered copyleft -copyleft = "warn" -# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses -# * both - The license will be approved if it is both OSI-approved *AND* FSF -# * either - The license will be approved if it is either OSI-approved *OR* FSF -# * osi-only - The license will be approved if is OSI-approved *AND NOT* FSF -# * fsf-only - The license will be approved if is FSF *AND NOT* OSI-approved -# * neither - This predicate is ignored and the default lint level is used -allow-osi-fsf-free = "either" -# Lint level used when no other predicates are matched -# 1. License isn't in the allow or deny lists -# 2. License isn't copyleft -# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither" -default = "deny" # The confidence threshold for detecting a license from license text. # The higher the value, the more closely the license text must be to the # canonical license text of a valid SPDX license file. @@ -111,33 +114,37 @@ confidence-threshold = 0.8 exceptions = [ # Each entry is the crate and version constraint, and its specific allow # list + #{ allow = ["Zlib"], crate = "adler32" }, ] - - # Some crates don't have (easily) machine readable licensing information, # adding a clarification entry for it allows you to manually specify the # licensing information -[[licenses.clarify]] -name = "webc" -version = "*" -expression = "BSL-1.0" -license-files = [ - { path = "LICENSE.txt", hash = 0xa2180a97 } -] +#[[licenses.clarify]] +# The package spec the clarification applies to +#crate = "ring" +# The SPDX expression for the license requirements of the crate +#expression = "MIT AND ISC AND OpenSSL" +# One or more files in the crate's source used as the "source of truth" for +# the license expression. If the contents match, the clarification will be used +# when running the license check, otherwise the clarification will be ignored +# and the crate will be checked normally, which may produce warnings or errors +# depending on the rest of your configuration +#license-files = [ +# Each entry is a crate relative path, and the (opaque) hash of its contents +#{ path = "LICENSE", hash = 0xbd0eed23 } +#] [[licenses.clarify]] -name = "ring" -version = "*" +crate = "ring" expression = "MIT AND ISC AND OpenSSL" -license-files = [ - # Each entry is a crate relative path, and the (opaque) hash of its contents - { path = "LICENSE", hash = 0xbd0eed23 } -] +license-files = [ { path = "LICENSE", hash = 0xbd0eed23 } ] [licenses.private] # If true, ignores workspace crates that aren't published, or are only -# published to private registries +# published to private registries. +# To see how to mark a crate as unpublished (to the official registry), +# visit https://doc.rust-lang.org/cargo/reference/manifest.html#the-publish-field. ignore = false # One or more private registries that you might publish crates to, if a crate # is only published to private registries, and ignore is true, the crate will @@ -160,41 +167,63 @@ wildcards = "allow" # * simplest-path - The path to the version with the fewest edges is highlighted # * all - Both lowest-version and simplest-path are used highlight = "all" +# The default lint level for `default` features for crates that are members of +# the workspace that is being checked. This can be overridden by allowing/denying +# `default` on a crate-by-crate basis if desired. +workspace-default-features = "allow" +# The default lint level for `default` features for external crates that are not +# members of the workspace. This can be overridden by allowing/denying `default` +# on a crate-by-crate basis if desired. +external-default-features = "allow" # List of crates that are allowed. Use with care! allow = [ - #{ name = "ansi_term", version = "=0.11.0" }, + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is allowed" }, ] # List of crates to deny deny = [ - # Each entry the name of a crate and a version range. If version is - # not specified, all versions will be matched. - #{ name = "ansi_term", version = "=0.11.0" }, - # + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason it is banned" }, # Wrapper crates can optionally be specified to allow the crate when it # is a direct dependency of the otherwise banned crate - #{ name = "ansi_term", version = "=0.11.0", wrappers = [] }, + #{ crate = "ansi_term@0.11.0", wrappers = ["this-crate-directly-depends-on-ansi_term"] }, ] + +# List of features to allow/deny +# Each entry the name of a crate and a version range. If version is +# not specified, all versions will be matched. +#[[bans.features]] +#crate = "reqwest" +# Features to not allow +#deny = ["json"] +# Features to allow +#allow = [ +# "rustls", +# "__rustls", +# "__tls", +# "hyper-rustls", +# "rustls", +# "rustls-pemfile", +# "rustls-tls-webpki-roots", +# "tokio-rustls", +# "webpki-roots", +#] +# If true, the allowed features must exactly match the enabled feature set. If +# this is set there is no point setting `deny` +#exact = true + # Certain crates/versions that will be skipped when doing duplicate detection. skip = [ - { name = "webc" }, # licence is not fetched correctly and end up as unknown - { name = "ahash", version = "=0.4.7" }, - { name = "hashbrown", version = "=0.9.1" }, - { name = "cfg-if", version = "=0.1.10" }, - { name = "gimli", version = "=0.25.0" }, - { name = "strsim", version = "=0.8.0" }, - { name = "semver", version = "=0.9.0" }, - { name = "semver", version = "=0.11.0" }, - { name = "semver-parser", version = "=0.7.0" }, - { name = "rustc_version", version = "=0.2.3" }, - { name = "itoa", version = "=0.4.8" }, - { name = "object", version = "=0.27.1" }, + #"ansi_term@0.11.0", + #{ crate = "ansi_term@0.11.0", reason = "you can specify a reason why it can't be updated/removed" }, ] # Similarly to `skip` allows you to skip certain crates during duplicate # detection. Unlike skip, it also includes the entire tree of transitive # dependencies starting at the specified crate, up to a certain depth, which is -# by default infinite +# by default infinite. skip-tree = [ - #{ name = "ansi_term", version = "=0.11.0", depth = 20 }, + #"ansi_term@0.11.0", # will be skipped along with _all_ of its direct and transitive dependencies + #{ crate = "ansi_term@0.11.0", depth = 20 }, ] # This section is considered when running `cargo deny check sources`. @@ -203,20 +232,20 @@ skip-tree = [ [sources] # Lint level for what to happen when a crate from a crate registry that is not # in the allow list is encountered -unknown-registry = "deny" +unknown-registry = "warn" # Lint level for what to happen when a crate from a git repository that is not # in the allow list is encountered -unknown-git = "deny" +unknown-git = "warn" # List of URLs for allowed crate registries. Defaults to the crates.io index # if not specified. If it is specified but empty, no registries are allowed. allow-registry = ["https://github.com/rust-lang/crates.io-index"] # List of URLs for allowed Git repositories allow-git = [] -#[sources.allow-org] +[sources.allow-org] # 1 or more github.com organizations to allow git sources for -#github = [""] +github = [] # 1 or more gitlab.com organizations to allow git sources for -#gitlab = [""] +gitlab = [] # 1 or more bitbucket.org organizations to allow git sources for -#bitbucket = [""] +bitbucket = [] From 16c8a62297e77e3d61ef588d1c4267168f7d432a Mon Sep 17 00:00:00 2001 From: James Kay Date: Wed, 18 Sep 2024 13:32:44 +0100 Subject: [PATCH 31/53] `js::module`: leave warning comment on the `Send` impl Until https://github.com/wasmerio/wasmer/issues/4158 is fixed, let's at least leave a signpost so future readers don't trip on it like I did :) --- lib/api/src/js/module.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/api/src/js/module.rs b/lib/api/src/js/module.rs index 61866abcf5f..adc61d6b08b 100644 --- a/lib/api/src/js/module.rs +++ b/lib/api/src/js/module.rs @@ -44,14 +44,10 @@ pub struct Module { raw_bytes: Option, } -// Module implements `structuredClone` in js, so it's safe it to make it Send. -// https://developer.mozilla.org/en-US/docs/Web/API/structuredClone -// ```js -// const module = new WebAssembly.Module(new Uint8Array([ -// 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00 -// ])); -// structuredClone(module) -// ``` +// XXX +// Do not rely on `Module` being `Send`: it will panic at runtime +// if accessed from multiple threads thanks to [`JsHandle`]. +// See https://github.com/wasmerio/wasmer/issues/4158 for details. unsafe impl Send for Module {} unsafe impl Sync for Module {} From 4d9a3c673dbf21f2068655f06c1c3bb62f0c29e0 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Fri, 20 Sep 2024 12:38:11 +0200 Subject: [PATCH 32/53] chore(wasix): Reduce syscall instrumentation levels Improve tracing performance by reducing the syscall instrumentation levels to "trace". This allows compiling out all the instrumentation with a tracing feature flag in release builds. --- lib/wasix/src/syscalls/legacy/snapshot0.rs | 6 +++--- lib/wasix/src/syscalls/mod.rs | 4 ++-- lib/wasix/src/syscalls/wasi/args_get.rs | 2 +- lib/wasix/src/syscalls/wasi/args_sizes_get.rs | 2 +- lib/wasix/src/syscalls/wasi/environ_get.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_advise.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_allocate.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_close.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_datasync.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_dup.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_event.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_filestat_get.rs | 4 ++-- lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_renumber.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_sync.rs | 2 +- lib/wasix/src/syscalls/wasi/fd_tell.rs | 2 +- lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs | 2 +- lib/wasix/src/syscalls/wasi/path_link.rs | 2 +- lib/wasix/src/syscalls/wasi/path_open.rs | 2 +- lib/wasix/src/syscalls/wasi/path_readlink.rs | 2 +- lib/wasix/src/syscalls/wasi/path_remove_directory.rs | 2 +- lib/wasix/src/syscalls/wasi/path_rename.rs | 2 +- lib/wasix/src/syscalls/wasi/path_symlink.rs | 2 +- lib/wasix/src/syscalls/wasi/path_unlink_file.rs | 2 +- lib/wasix/src/syscalls/wasi/proc_exit.rs | 2 +- lib/wasix/src/syscalls/wasi/proc_raise.rs | 2 +- lib/wasix/src/syscalls/wasi/thread_spawn.rs | 2 +- lib/wasix/src/syscalls/wasix/chdir.rs | 2 +- lib/wasix/src/syscalls/wasix/getcwd.rs | 2 +- lib/wasix/src/syscalls/wasix/port_addr_add.rs | 2 +- lib/wasix/src/syscalls/wasix/port_addr_clear.rs | 2 +- lib/wasix/src/syscalls/wasix/port_addr_list.rs | 2 +- lib/wasix/src/syscalls/wasix/port_addr_remove.rs | 2 +- lib/wasix/src/syscalls/wasix/port_bridge.rs | 2 +- lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs | 2 +- lib/wasix/src/syscalls/wasix/port_gateway_set.rs | 2 +- lib/wasix/src/syscalls/wasix/port_mac.rs | 2 +- lib/wasix/src/syscalls/wasix/port_route_add.rs | 2 +- lib/wasix/src/syscalls/wasix/port_route_clear.rs | 2 +- lib/wasix/src/syscalls/wasix/port_route_list.rs | 2 +- lib/wasix/src/syscalls/wasix/port_route_remove.rs | 2 +- lib/wasix/src/syscalls/wasix/port_unbridge.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_exec.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_exec2.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_fork.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_id.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_parent.rs | 2 +- lib/wasix/src/syscalls/wasix/proc_spawn.rs | 2 +- lib/wasix/src/syscalls/wasix/resolve.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_accept.rs | 4 ++-- lib/wasix/src/syscalls/wasix/sock_addr_local.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_addr_peer.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_bind.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_connect.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_get_opt_flag.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_get_opt_size.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_get_opt_time.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_listen.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_open.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_send_file.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_shutdown.rs | 2 +- lib/wasix/src/syscalls/wasix/sock_status.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_exit.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_join.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_parallelism.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_signal.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_sleep.rs | 2 +- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 2 +- lib/wasix/src/syscalls/wasix/tty_get.rs | 2 +- lib/wasix/src/syscalls/wasix/tty_set.rs | 2 +- 80 files changed, 85 insertions(+), 85 deletions(-) diff --git a/lib/wasix/src/syscalls/legacy/snapshot0.rs b/lib/wasix/src/syscalls/legacy/snapshot0.rs index 215c99602f0..44f38586409 100644 --- a/lib/wasix/src/syscalls/legacy/snapshot0.rs +++ b/lib/wasix/src/syscalls/legacy/snapshot0.rs @@ -16,7 +16,7 @@ use crate::{ }; /// Wrapper around `syscalls::fd_filestat_get` for old Snapshot0 -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn fd_filestat_get( mut ctx: FunctionEnvMut, fd: Fd, @@ -30,7 +30,7 @@ pub fn fd_filestat_get( } /// Wrapper around `syscalls::path_filestat_get` for old Snapshot0 -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn path_filestat_get( mut ctx: FunctionEnvMut, fd: Fd, @@ -50,7 +50,7 @@ pub fn path_filestat_get( /// Wrapper around `syscalls::fd_seek` with extra logic to remap the values /// of `Whence` -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn fd_seek( ctx: FunctionEnvMut, fd: Fd, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 26e1e81d9f6..87cde9b845a 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1255,7 +1255,7 @@ where Ok(Errno::Success) } -#[instrument(level = "debug", skip_all, fields(memory_stack_len = memory_stack.len(), rewind_stack_len = rewind_stack.len(), store_data_len = store_data.len()))] +// NOTE: not tracing-instrumented because [`rewind_ext`] already is. #[must_use = "the action must be passed to the call loop"] pub fn rewind( mut ctx: FunctionEnvMut, @@ -1277,7 +1277,7 @@ where ) } -#[instrument(level = "debug", skip_all, fields(rewind_stack_len = rewind_stack.len(), store_data_len = store_data.len()))] +#[instrument(level = "trace", skip_all, fields(rewind_stack_len = rewind_stack.len(), store_data_len = store_data.len()))] #[must_use = "the action must be passed to the call loop"] pub fn rewind_ext( ctx: &mut FunctionEnvMut, diff --git a/lib/wasix/src/syscalls/wasi/args_get.rs b/lib/wasix/src/syscalls/wasi/args_get.rs index 8a316d92bbe..fe3f22363f9 100644 --- a/lib/wasix/src/syscalls/wasi/args_get.rs +++ b/lib/wasix/src/syscalls/wasi/args_get.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// - `char *argv_buf` /// A pointer to a buffer to write the argument string data. /// -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn args_get( mut ctx: FunctionEnvMut<'_, WasiEnv>, argv: WasmPtr, M>, diff --git a/lib/wasix/src/syscalls/wasi/args_sizes_get.rs b/lib/wasix/src/syscalls/wasi/args_sizes_get.rs index 08ec71c1679..1740e218fa9 100644 --- a/lib/wasix/src/syscalls/wasi/args_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/args_sizes_get.rs @@ -8,7 +8,7 @@ use crate::syscalls::*; /// The number of arguments. /// - `size_t *argv_buf_size` /// The size of the argument string data. -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn args_sizes_get( mut ctx: FunctionEnvMut<'_, WasiEnv>, argc: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasi/environ_get.rs b/lib/wasix/src/syscalls/wasi/environ_get.rs index 0c3b0c60735..042f0ffe536 100644 --- a/lib/wasix/src/syscalls/wasi/environ_get.rs +++ b/lib/wasix/src/syscalls/wasi/environ_get.rs @@ -9,7 +9,7 @@ use crate::{journal::SnapshotTrigger, syscalls::*}; /// A pointer to a buffer to write the environment variable pointers. /// - `char *environ_buf` /// A pointer to a buffer to write the environment variable string data. -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn environ_get( mut ctx: FunctionEnvMut<'_, WasiEnv>, environ: WasmPtr, M>, diff --git a/lib/wasix/src/syscalls/wasi/fd_advise.rs b/lib/wasix/src/syscalls/wasi/fd_advise.rs index bafb7841c4e..d3e79ce7bb4 100644 --- a/lib/wasix/src/syscalls/wasi/fd_advise.rs +++ b/lib/wasix/src/syscalls/wasi/fd_advise.rs @@ -12,7 +12,7 @@ use crate::syscalls::*; /// The length from the offset to which the advice applies /// - `__wasi_advice_t advice` /// The advice to give -#[instrument(level = "debug", skip_all, fields(%fd, %offset, %len, ?advice), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, %offset, %len, ?advice), ret)] pub fn fd_advise( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_allocate.rs b/lib/wasix/src/syscalls/wasi/fd_allocate.rs index 23462cbe48d..ab290193cab 100644 --- a/lib/wasix/src/syscalls/wasi/fd_allocate.rs +++ b/lib/wasix/src/syscalls/wasi/fd_allocate.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// The offset from the start marking the beginning of the allocation /// - `Filesize len` /// The length from the offset marking the end of the allocation -#[instrument(level = "debug", skip_all, fields(%fd, %offset, %len), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, %offset, %len), ret)] pub fn fd_allocate( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_close.rs b/lib/wasix/src/syscalls/wasi/fd_close.rs index 9196d2ade34..6100cc2328a 100644 --- a/lib/wasix/src/syscalls/wasi/fd_close.rs +++ b/lib/wasix/src/syscalls/wasi/fd_close.rs @@ -12,7 +12,7 @@ use crate::syscalls::*; /// If `fd` is a directory /// - `Errno::Badf` /// If `fd` is invalid or not open -#[instrument(level = "debug", skip_all, fields(pid = ctx.data().process.pid().raw(), %fd), ret)] +#[instrument(level = "trace", skip_all, fields(pid = ctx.data().process.pid().raw(), %fd), ret)] pub fn fd_close(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); diff --git a/lib/wasix/src/syscalls/wasi/fd_datasync.rs b/lib/wasix/src/syscalls/wasi/fd_datasync.rs index 75523059cd3..8ab3afbb1ea 100644 --- a/lib/wasix/src/syscalls/wasi/fd_datasync.rs +++ b/lib/wasix/src/syscalls/wasi/fd_datasync.rs @@ -6,7 +6,7 @@ use crate::syscalls::*; /// Inputs: /// - `Fd fd` /// The file descriptor to sync -#[instrument(level = "debug", skip_all, fields(%fd), ret)] +#[instrument(level = "trace", skip_all, fields(%fd), ret)] pub fn fd_datasync(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); diff --git a/lib/wasix/src/syscalls/wasi/fd_dup.rs b/lib/wasix/src/syscalls/wasi/fd_dup.rs index 3dfe826b890..b0d43d6d7dd 100644 --- a/lib/wasix/src/syscalls/wasi/fd_dup.rs +++ b/lib/wasix/src/syscalls/wasi/fd_dup.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// Outputs: /// - `Fd fd` /// The new file handle that is a duplicate of the original -#[instrument(level = "debug", skip_all, fields(%fd, ret_fd = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, ret_fd = field::Empty), ret)] pub fn fd_dup( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_event.rs b/lib/wasix/src/syscalls/wasi/fd_event.rs index 4c67857f5f7..53640876c3a 100644 --- a/lib/wasix/src/syscalls/wasi/fd_event.rs +++ b/lib/wasix/src/syscalls/wasi/fd_event.rs @@ -3,7 +3,7 @@ use crate::{fs::NotificationInner, syscalls::*}; /// ### `fd_event()` /// Creates a file handle for event notifications -#[instrument(level = "debug", skip_all, fields(%initial_val, ret_fd = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%initial_val, ret_fd = field::Empty), ret)] pub fn fd_event( mut ctx: FunctionEnvMut<'_, WasiEnv>, initial_val: u64, diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs index cedd78fecc9..2c7e15d3169 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_flags.rs @@ -8,7 +8,7 @@ use crate::syscalls::*; /// The file descriptor to apply the new flags to /// - `Fdflags flags` /// The flags to apply to `fd` -#[instrument(level = "debug", skip_all, fields(%fd), ret)] +#[instrument(level = "trace", skip_all, fields(%fd), ret)] pub fn fd_fdstat_set_flags( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs index ce9aa91d2dc..c4d241da0e4 100644 --- a/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs +++ b/lib/wasix/src/syscalls/wasi/fd_fdstat_set_rights.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// The rights to apply to `fd` /// - `Rights fs_rights_inheriting` /// The inheriting rights to apply to `fd` -#[instrument(level = "debug", skip_all, fields(%fd), ret)] +#[instrument(level = "trace", skip_all, fields(%fd), ret)] pub fn fd_fdstat_set_rights( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs index e893a35185e..709b6c8037f 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs @@ -10,7 +10,7 @@ use crate::types::wasi::Snapshot0Filestat; /// Output: /// - `Filestat *buf` /// Where the metadata from `fd` will be written -#[instrument(level = "debug", skip_all, fields(%fd), ret)] +#[instrument(level = "trace", skip_all, fields(%fd), ret)] pub fn fd_filestat_get( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, @@ -56,7 +56,7 @@ pub(crate) fn fd_filestat_get_internal( /// Output: /// - `Snapshot0Filestat *buf` /// Where the metadata from `fd` will be written -#[instrument(level = "debug", skip_all, fields(%fd), ret)] +#[instrument(level = "trace", skip_all, fields(%fd), ret)] pub fn fd_filestat_get_old( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs index ddb361744b2..a3b400ac81c 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_size.rs @@ -8,7 +8,7 @@ use crate::syscalls::*; /// File descriptor to adjust /// - `Filesize st_size` /// New size that `fd` will be set to -#[instrument(level = "debug", skip_all, fields(%fd, %st_size), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, %st_size), ret)] pub fn fd_filestat_set_size( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs index 9add80b56d3..56447883832 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs @@ -12,7 +12,7 @@ use crate::syscalls::*; /// Last modified time /// - `Fstflags fst_flags` /// Bit-vector for controlling which times get set -#[instrument(level = "debug", skip_all, fields(%fd, %st_atim, %st_mtim), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, %st_atim, %st_mtim), ret)] pub fn fd_filestat_set_times( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_renumber.rs b/lib/wasix/src/syscalls/wasi/fd_renumber.rs index 55add2ebea7..39e98414566 100644 --- a/lib/wasix/src/syscalls/wasi/fd_renumber.rs +++ b/lib/wasix/src/syscalls/wasi/fd_renumber.rs @@ -8,7 +8,7 @@ use crate::syscalls::*; /// File descriptor to copy /// - `Fd to` /// Location to copy file descriptor to -#[instrument(level = "debug", skip_all, fields(%from, %to), ret)] +#[instrument(level = "trace", skip_all, fields(%from, %to), ret)] pub fn fd_renumber( mut ctx: FunctionEnvMut<'_, WasiEnv>, from: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_sync.rs b/lib/wasix/src/syscalls/wasi/fd_sync.rs index d0600bbdca2..a7aee7bd8e4 100644 --- a/lib/wasix/src/syscalls/wasi/fd_sync.rs +++ b/lib/wasix/src/syscalls/wasi/fd_sync.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// TODO: figure out which errors this should return /// - `Errno::Perm` /// - `Errno::Notcapable` -#[instrument(level = "debug", skip_all, fields(%fd), ret)] +#[instrument(level = "trace", skip_all, fields(%fd), ret)] pub fn fd_sync(mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd) -> Result { wasi_try_ok!(WasiEnv::process_signals_and_exit(&mut ctx)?); diff --git a/lib/wasix/src/syscalls/wasi/fd_tell.rs b/lib/wasix/src/syscalls/wasi/fd_tell.rs index 7f01c7bd6b1..673ec0b89c7 100644 --- a/lib/wasix/src/syscalls/wasi/fd_tell.rs +++ b/lib/wasix/src/syscalls/wasi/fd_tell.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// Output: /// - `Filesize *offset` /// The offset of `fd` relative to the start of the file -#[instrument(level = "debug", skip_all, fields(%fd, offset = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, offset = field::Empty), ret)] pub fn fd_tell( ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs index c471374e20e..6a80c195667 100644 --- a/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/path_filestat_set_times.rs @@ -18,7 +18,7 @@ use crate::syscalls::*; /// The timestamp that the last modified time attribute is set to /// - `Fstflags fst_flags` /// A bitmask controlling which attributes are set -#[instrument(level = "debug", skip_all, fields(%fd, path = field::Empty, %st_atim, %st_mtim), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, path = field::Empty, %st_atim, %st_mtim), ret)] pub fn path_filestat_set_times( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/path_link.rs b/lib/wasix/src/syscalls/wasi/path_link.rs index 9b0388d6cd2..6716f3d2ff7 100644 --- a/lib/wasix/src/syscalls/wasi/path_link.rs +++ b/lib/wasix/src/syscalls/wasi/path_link.rs @@ -18,7 +18,7 @@ use crate::syscalls::*; /// String containing the new file path /// - `u32 old_path_len` /// Length of the `new_path` string -#[instrument(level = "debug", skip_all, fields(%old_fd, %new_fd, old_path = field::Empty, new_path = field::Empty, follow_symlinks = false), ret)] +#[instrument(level = "trace", skip_all, fields(%old_fd, %new_fd, old_path = field::Empty, new_path = field::Empty, follow_symlinks = false), ret)] pub fn path_link( mut ctx: FunctionEnvMut<'_, WasiEnv>, old_fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/path_open.rs b/lib/wasix/src/syscalls/wasi/path_open.rs index 5f8033cb63d..9c8d3c81655 100644 --- a/lib/wasix/src/syscalls/wasi/path_open.rs +++ b/lib/wasix/src/syscalls/wasi/path_open.rs @@ -25,7 +25,7 @@ use crate::syscalls::*; /// The new file descriptor /// Possible Errors: /// - `Errno::Access`, `Errno::Badf`, `Errno::Fault`, `Errno::Fbig?`, `Errno::Inval`, `Errno::Io`, `Errno::Loop`, `Errno::Mfile`, `Errno::Nametoolong?`, `Errno::Nfile`, `Errno::Noent`, `Errno::Notdir`, `Errno::Rofs`, and `Errno::Notcapable` -#[instrument(level = "debug", skip_all, fields(%dirfd, path = field::Empty, follow_symlinks = field::Empty, ret_fd = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%dirfd, path = field::Empty, follow_symlinks = field::Empty, ret_fd = field::Empty), ret)] pub fn path_open( mut ctx: FunctionEnvMut<'_, WasiEnv>, dirfd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/path_readlink.rs b/lib/wasix/src/syscalls/wasi/path_readlink.rs index e83589008f9..194212ac950 100644 --- a/lib/wasix/src/syscalls/wasi/path_readlink.rs +++ b/lib/wasix/src/syscalls/wasi/path_readlink.rs @@ -17,7 +17,7 @@ use crate::syscalls::*; /// Pointer to characters containing the path that the symlink points to /// - `u32 buf_used` /// The number of bytes written to `buf` -#[instrument(level = "debug", skip_all, fields(%dir_fd, path = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%dir_fd, path = field::Empty), ret)] pub fn path_readlink( ctx: FunctionEnvMut<'_, WasiEnv>, dir_fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs index 2fc0fe2fecf..a8d335e403c 100644 --- a/lib/wasix/src/syscalls/wasi/path_remove_directory.rs +++ b/lib/wasix/src/syscalls/wasi/path_remove_directory.rs @@ -2,7 +2,7 @@ use super::*; use crate::syscalls::*; /// Returns Errno::Notemtpy if directory is not empty -#[instrument(level = "debug", skip_all, fields(%fd, path = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, path = field::Empty), ret)] pub fn path_remove_directory( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index 147c8f209ac..37ab9145f42 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -16,7 +16,7 @@ use crate::syscalls::*; /// Pointer to UTF8 bytes, the new file name /// - `u32 new_path_len` /// The number of bytes to read from `new_path` -#[instrument(level = "debug", skip_all, fields(%old_fd, %new_fd, old_path = field::Empty, new_path = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%old_fd, %new_fd, old_path = field::Empty, new_path = field::Empty), ret)] pub fn path_rename( mut ctx: FunctionEnvMut<'_, WasiEnv>, old_fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/path_symlink.rs b/lib/wasix/src/syscalls/wasi/path_symlink.rs index 66a45988777..112e78fbdd5 100644 --- a/lib/wasix/src/syscalls/wasi/path_symlink.rs +++ b/lib/wasix/src/syscalls/wasi/path_symlink.rs @@ -14,7 +14,7 @@ use crate::syscalls::*; /// Array of UTF-8 bytes representing the target path /// - `u32 new_path_len` /// The number of bytes to read from `new_path` -#[instrument(level = "debug", skip_all, fields(%fd, old_path = field::Empty, new_path = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, old_path = field::Empty, new_path = field::Empty), ret)] pub fn path_symlink( mut ctx: FunctionEnvMut<'_, WasiEnv>, old_path: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs index c79a5812c37..16a78c5daf2 100644 --- a/lib/wasix/src/syscalls/wasi/path_unlink_file.rs +++ b/lib/wasix/src/syscalls/wasi/path_unlink_file.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// Array of UTF-8 bytes representing the path /// - `u32 path_len` /// The number of bytes in the `path` array -#[instrument(level = "debug", skip_all, fields(%fd, path = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, path = field::Empty), ret)] pub fn path_unlink_file( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/proc_exit.rs b/lib/wasix/src/syscalls/wasi/proc_exit.rs index 58b83d54b4d..1afb7a4b29c 100644 --- a/lib/wasix/src/syscalls/wasi/proc_exit.rs +++ b/lib/wasix/src/syscalls/wasi/proc_exit.rs @@ -8,7 +8,7 @@ use crate::syscalls::*; /// Inputs: /// - `ExitCode` /// Exit code to return to the operating system -#[instrument(level = "debug", skip_all)] +#[instrument(level = "trace", skip_all)] pub fn proc_exit( mut ctx: FunctionEnvMut<'_, WasiEnv>, code: ExitCode, diff --git a/lib/wasix/src/syscalls/wasi/proc_raise.rs b/lib/wasix/src/syscalls/wasi/proc_raise.rs index 6cd8cdf9f0f..98efef0cb25 100644 --- a/lib/wasix/src/syscalls/wasi/proc_raise.rs +++ b/lib/wasix/src/syscalls/wasi/proc_raise.rs @@ -7,7 +7,7 @@ use crate::syscalls::*; /// Inputs: /// - `Signal` /// Signal to be raised for this process -#[instrument(level = "debug", skip_all, fields(sig), ret)] +#[instrument(level = "trace", skip_all, fields(sig), ret)] pub fn proc_raise(mut ctx: FunctionEnvMut<'_, WasiEnv>, sig: Signal) -> Result { let env = ctx.data(); env.process.signal_process(sig); diff --git a/lib/wasix/src/syscalls/wasi/thread_spawn.rs b/lib/wasix/src/syscalls/wasi/thread_spawn.rs index be8e9de02a3..36b81e3a29d 100644 --- a/lib/wasix/src/syscalls/wasi/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasi/thread_spawn.rs @@ -16,7 +16,7 @@ use wasmer_wasix_types::wasi::ThreadStart; /// /// Returns the thread index of the newly created thread /// (indices always start from the same value as `pid` and increments in steps) -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn thread_spawn( mut ctx: FunctionEnvMut<'_, WasiEnv>, start_ptr: WasmPtr, M>, diff --git a/lib/wasix/src/syscalls/wasix/chdir.rs b/lib/wasix/src/syscalls/wasix/chdir.rs index beb4b93a75d..5c2d643139e 100644 --- a/lib/wasix/src/syscalls/wasix/chdir.rs +++ b/lib/wasix/src/syscalls/wasix/chdir.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `chdir()` /// Sets the current working directory -#[instrument(level = "debug", skip_all, fields(name = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(name = field::Empty), ret)] pub fn chdir( mut ctx: FunctionEnvMut<'_, WasiEnv>, path: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/getcwd.rs b/lib/wasix/src/syscalls/wasix/getcwd.rs index 27250221a85..faadee9dbae 100644 --- a/lib/wasix/src/syscalls/wasix/getcwd.rs +++ b/lib/wasix/src/syscalls/wasix/getcwd.rs @@ -5,7 +5,7 @@ use crate::syscalls::*; /// Returns the current working directory /// If the path exceeds the size of the buffer then this function /// will return ERANGE -#[instrument(level = "debug", skip_all, fields(path = field::Empty, max_path_len = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(path = field::Empty, max_path_len = field::Empty), ret)] pub fn getcwd( ctx: FunctionEnvMut<'_, WasiEnv>, path: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/port_addr_add.rs b/lib/wasix/src/syscalls/wasix/port_addr_add.rs index 5a9d5909a66..6df252f004f 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_add.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `addr` - Address to be added -#[instrument(level = "debug", skip_all, fields(ip = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(ip = field::Empty), ret)] pub fn port_addr_add( mut ctx: FunctionEnvMut<'_, WasiEnv>, ip: WasmPtr<__wasi_cidr_t, M>, diff --git a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs index 9ceb6c45c66..e8959c62fab 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `port_addr_clear()` /// Clears all the addresses on the local port -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn port_addr_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_addr_clear_internal(&mut ctx)?); diff --git a/lib/wasix/src/syscalls/wasix/port_addr_list.rs b/lib/wasix/src/syscalls/wasix/port_addr_list.rs index 54390855415..8072c712712 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_list.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_list.rs @@ -14,7 +14,7 @@ use crate::syscalls::*; /// ## Return /// /// The number of addresses returned. -#[instrument(level = "debug", skip_all, fields(naddrs = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(naddrs = field::Empty), ret)] pub fn port_addr_list( mut ctx: FunctionEnvMut<'_, WasiEnv>, addrs_ptr: WasmPtr<__wasi_cidr_t, M>, diff --git a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs index eb689243576..001a4374b9b 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs @@ -7,7 +7,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `addr` - Address to be removed -#[instrument(level = "debug", skip_all, fields(ip = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(ip = field::Empty), ret)] pub fn port_addr_remove( mut ctx: FunctionEnvMut<'_, WasiEnv>, ip: WasmPtr<__wasi_addr_t, M>, diff --git a/lib/wasix/src/syscalls/wasix/port_bridge.rs b/lib/wasix/src/syscalls/wasix/port_bridge.rs index e686ec5d078..6bd79e70e58 100644 --- a/lib/wasix/src/syscalls/wasix/port_bridge.rs +++ b/lib/wasix/src/syscalls/wasix/port_bridge.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// * `network` - Fully qualified identifier for the network /// * `token` - Access token used to authenticate with the network /// * `security` - Level of encryption to encapsulate the network connection with -#[instrument(level = "debug", skip_all, fields(network = field::Empty, ?security), ret)] +#[instrument(level = "trace", skip_all, fields(network = field::Empty, ?security), ret)] pub fn port_bridge( mut ctx: FunctionEnvMut<'_, WasiEnv>, network: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs b/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs index 1415f76c609..e380ad2c541 100644 --- a/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs +++ b/lib/wasix/src/syscalls/wasix/port_dhcp_acquire.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `port_dhcp_acquire()` /// Acquires a set of IP addresses using DHCP -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn port_dhcp_acquire(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_dhcp_acquire_internal(&mut ctx)?); diff --git a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs index fc0549ea2c1..92070f96d69 100644 --- a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs +++ b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs @@ -7,7 +7,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `addr` - Address of the default gateway -#[instrument(level = "debug", skip_all, fields(ip = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(ip = field::Empty), ret)] pub fn port_gateway_set( mut ctx: FunctionEnvMut<'_, WasiEnv>, ip: WasmPtr<__wasi_addr_t, M>, diff --git a/lib/wasix/src/syscalls/wasix/port_mac.rs b/lib/wasix/src/syscalls/wasix/port_mac.rs index 93b13c544c9..3d6ebcf0fd4 100644 --- a/lib/wasix/src/syscalls/wasix/port_mac.rs +++ b/lib/wasix/src/syscalls/wasix/port_mac.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `port_mac()` /// Returns the MAC address of the local port -#[instrument(level = "debug", skip_all, fields(max = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(max = field::Empty), ret)] pub fn port_mac( mut ctx: FunctionEnvMut<'_, WasiEnv>, ret_mac: WasmPtr<__wasi_hardwareaddress_t, M>, diff --git a/lib/wasix/src/syscalls/wasix/port_route_add.rs b/lib/wasix/src/syscalls/wasix/port_route_add.rs index 00e9f781c34..3dd0cadc3a3 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_add.rs @@ -5,7 +5,7 @@ use crate::syscalls::*; /// ### `port_route_add()` /// Adds a new route to the local port -#[instrument(level = "debug", skip_all, fields(cidr = field::Empty, via_router = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(cidr = field::Empty, via_router = field::Empty), ret)] pub fn port_route_add( mut ctx: FunctionEnvMut<'_, WasiEnv>, cidr: WasmPtr<__wasi_cidr_t, M>, diff --git a/lib/wasix/src/syscalls/wasix/port_route_clear.rs b/lib/wasix/src/syscalls/wasix/port_route_clear.rs index 3eca5de4208..f2d2de02cb4 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_clear.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `port_route_clear()` /// Clears all the routes in the local port -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn port_route_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_route_clear_internal(&mut ctx)?); diff --git a/lib/wasix/src/syscalls/wasix/port_route_list.rs b/lib/wasix/src/syscalls/wasix/port_route_list.rs index 31b0ee397de..6fba94d4406 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_list.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_list.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `routes` - The buffer where routes will be stored -#[instrument(level = "debug", skip_all, fields(nroutes = field::Empty, max_routes = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(nroutes = field::Empty, max_routes = field::Empty), ret)] pub fn port_route_list( mut ctx: FunctionEnvMut<'_, WasiEnv>, routes_ptr: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/port_route_remove.rs b/lib/wasix/src/syscalls/wasix/port_route_remove.rs index 0b1b5da2923..9e0ddbd456e 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_remove.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `port_route_remove()` /// Removes an existing route from the local port -#[instrument(level = "debug", skip_all, fields(ip = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(ip = field::Empty), ret)] pub fn port_route_remove( mut ctx: FunctionEnvMut<'_, WasiEnv>, ip: WasmPtr<__wasi_addr_t, M>, diff --git a/lib/wasix/src/syscalls/wasix/port_unbridge.rs b/lib/wasix/src/syscalls/wasix/port_unbridge.rs index 4495cd04862..c083511119b 100644 --- a/lib/wasix/src/syscalls/wasix/port_unbridge.rs +++ b/lib/wasix/src/syscalls/wasix/port_unbridge.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `port_unbridge()` /// Disconnects from a remote network -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn port_unbridge(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result { wasi_try_ok!(port_unbridge_internal(&mut ctx)?); diff --git a/lib/wasix/src/syscalls/wasix/proc_exec.rs b/lib/wasix/src/syscalls/wasix/proc_exec.rs index d5c433298ec..fbf9fc66bfb 100644 --- a/lib/wasix/src/syscalls/wasix/proc_exec.rs +++ b/lib/wasix/src/syscalls/wasix/proc_exec.rs @@ -17,7 +17,7 @@ use crate::{ /// ## Return /// /// Returns a bus process id that can be used to invoke calls -#[instrument(level = "debug", skip_all, fields(name = field::Empty, %args_len), ret)] +#[instrument(level = "trace", skip_all, fields(name = field::Empty, %args_len), ret)] pub fn proc_exec( mut ctx: FunctionEnvMut<'_, WasiEnv>, name: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/proc_exec2.rs b/lib/wasix/src/syscalls/wasix/proc_exec2.rs index 3d909b466b6..66fda51e1a2 100644 --- a/lib/wasix/src/syscalls/wasix/proc_exec2.rs +++ b/lib/wasix/src/syscalls/wasix/proc_exec2.rs @@ -18,7 +18,7 @@ use crate::{ /// ## Return /// /// Returns a bus process id that can be used to invoke calls -#[instrument(level = "debug", skip_all, fields(name = field::Empty, %args_len), ret)] +#[instrument(level = "trace", skip_all, fields(name = field::Empty, %args_len), ret)] pub fn proc_exec2( mut ctx: FunctionEnvMut<'_, WasiEnv>, name: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/proc_fork.rs b/lib/wasix/src/syscalls/wasix/proc_fork.rs index 31092c86e54..0761a2cf701 100644 --- a/lib/wasix/src/syscalls/wasix/proc_fork.rs +++ b/lib/wasix/src/syscalls/wasix/proc_fork.rs @@ -19,7 +19,7 @@ pub(crate) struct ForkResult { /// Forks the current process into a new subprocess. If the function /// returns a zero then its the new subprocess. If it returns a positive /// number then its the current process and the $pid represents the child. -#[instrument(level = "debug", skip_all, fields(pid = ctx.data().process.pid().raw()), ret)] +#[instrument(level = "trace", skip_all, fields(pid = ctx.data().process.pid().raw()), ret)] pub fn proc_fork( mut ctx: FunctionEnvMut<'_, WasiEnv>, mut copy_memory: Bool, diff --git a/lib/wasix/src/syscalls/wasix/proc_id.rs b/lib/wasix/src/syscalls/wasix/proc_id.rs index bbbc25fb0c9..3c32c91759f 100644 --- a/lib/wasix/src/syscalls/wasix/proc_id.rs +++ b/lib/wasix/src/syscalls/wasix/proc_id.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `proc_id()` /// Returns the handle of the current process -#[instrument(level = "debug", skip_all, fields(pid = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(pid = field::Empty), ret)] pub fn proc_id(ctx: FunctionEnvMut<'_, WasiEnv>, ret_pid: WasmPtr) -> Errno { let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; diff --git a/lib/wasix/src/syscalls/wasix/proc_parent.rs b/lib/wasix/src/syscalls/wasix/proc_parent.rs index f1efee72731..fe652b9dafa 100644 --- a/lib/wasix/src/syscalls/wasix/proc_parent.rs +++ b/lib/wasix/src/syscalls/wasix/proc_parent.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `proc_parent()` /// Returns the parent handle of the supplied process -#[instrument(level = "debug", skip_all, fields(%pid, parent = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%pid, parent = field::Empty), ret)] pub fn proc_parent( ctx: FunctionEnvMut<'_, WasiEnv>, pid: Pid, diff --git a/lib/wasix/src/syscalls/wasix/proc_spawn.rs b/lib/wasix/src/syscalls/wasix/proc_spawn.rs index 10158c742ab..d46d989d046 100644 --- a/lib/wasix/src/syscalls/wasix/proc_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/proc_spawn.rs @@ -23,7 +23,7 @@ use crate::syscalls::*; /// ## Return /// /// Returns a bus process id that can be used to invoke calls -#[instrument(level = "debug", skip_all, fields(name = field::Empty, working_dir = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(name = field::Empty, working_dir = field::Empty), ret)] pub fn proc_spawn( mut ctx: FunctionEnvMut<'_, WasiEnv>, name: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/resolve.rs b/lib/wasix/src/syscalls/wasix/resolve.rs index 2e40e6e6829..51d0b1074dc 100644 --- a/lib/wasix/src/syscalls/wasix/resolve.rs +++ b/lib/wasix/src/syscalls/wasix/resolve.rs @@ -19,7 +19,7 @@ use crate::syscalls::*; /// ## Return /// /// The number of IP addresses returned during the DNS resolution. -#[instrument(level = "debug", skip_all, fields(host = field::Empty, %port), ret)] +#[instrument(level = "trace", skip_all, fields(host = field::Empty, %port), ret)] pub fn resolve( mut ctx: FunctionEnvMut<'_, WasiEnv>, host: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/sock_accept.rs b/lib/wasix/src/syscalls/wasix/sock_accept.rs index 0c317aec8d1..2ccbaa10086 100644 --- a/lib/wasix/src/syscalls/wasix/sock_accept.rs +++ b/lib/wasix/src/syscalls/wasix/sock_accept.rs @@ -15,7 +15,7 @@ use crate::{net::socket::TimeType, syscalls::*}; /// ## Return /// /// New socket connection -#[instrument(level = "debug", skip_all, fields(%sock, fd = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, fd = field::Empty), ret)] pub fn sock_accept( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, @@ -57,7 +57,7 @@ pub fn sock_accept( /// ## Return /// /// New socket connection -#[instrument(level = "debug", skip_all, fields(%sock, fd = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, fd = field::Empty), ret)] pub fn sock_accept_v2( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_addr_local.rs b/lib/wasix/src/syscalls/wasix/sock_addr_local.rs index ac329bfd50c..f4b35f477bb 100644 --- a/lib/wasix/src/syscalls/wasix/sock_addr_local.rs +++ b/lib/wasix/src/syscalls/wasix/sock_addr_local.rs @@ -12,7 +12,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `fd` - Socket that the address is bound to -#[instrument(level = "debug", skip_all, fields(%sock, addr = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, addr = field::Empty), ret)] pub fn sock_addr_local( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_addr_peer.rs b/lib/wasix/src/syscalls/wasix/sock_addr_peer.rs index c49aea0d072..d0f261bba01 100644 --- a/lib/wasix/src/syscalls/wasix/sock_addr_peer.rs +++ b/lib/wasix/src/syscalls/wasix/sock_addr_peer.rs @@ -12,7 +12,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `fd` - Socket that the address is bound to -#[instrument(level = "debug", skip_all, fields(%sock, addr = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, addr = field::Empty), ret)] pub fn sock_addr_peer( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_bind.rs b/lib/wasix/src/syscalls/wasix/sock_bind.rs index bdea8822e3f..0675ab13dfd 100644 --- a/lib/wasix/src/syscalls/wasix/sock_bind.rs +++ b/lib/wasix/src/syscalls/wasix/sock_bind.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// /// * `fd` - File descriptor of the socket to be bind /// * `addr` - Address to bind the socket to -#[instrument(level = "debug", skip_all, fields(%sock, addr = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, addr = field::Empty), ret)] pub fn sock_bind( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_connect.rs b/lib/wasix/src/syscalls/wasix/sock_connect.rs index 3942d3dbcb4..5c3981e3b92 100644 --- a/lib/wasix/src/syscalls/wasix/sock_connect.rs +++ b/lib/wasix/src/syscalls/wasix/sock_connect.rs @@ -13,7 +13,7 @@ use crate::syscalls::*; /// /// * `fd` - Socket descriptor /// * `addr` - Address of the socket to connect to -#[instrument(level = "debug", skip_all, fields(%sock, addr = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, addr = field::Empty), ret)] pub fn sock_connect( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_get_opt_flag.rs b/lib/wasix/src/syscalls/wasix/sock_get_opt_flag.rs index 54083606e32..83f1f4d5e47 100644 --- a/lib/wasix/src/syscalls/wasix/sock_get_opt_flag.rs +++ b/lib/wasix/src/syscalls/wasix/sock_get_opt_flag.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// /// * `fd` - Socket descriptor /// * `sockopt` - Socket option to be retrieved -#[instrument(level = "debug", skip_all, fields(%sock, %opt), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %opt), ret)] pub fn sock_get_opt_flag( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_get_opt_size.rs b/lib/wasix/src/syscalls/wasix/sock_get_opt_size.rs index cedb2cdf56c..dd741284fe6 100644 --- a/lib/wasix/src/syscalls/wasix/sock_get_opt_size.rs +++ b/lib/wasix/src/syscalls/wasix/sock_get_opt_size.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// /// * `fd` - Socket descriptor /// * `sockopt` - Socket option to be retrieved -#[instrument(level = "debug", skip_all, fields(%sock, %opt), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %opt), ret)] pub fn sock_get_opt_size( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_get_opt_time.rs b/lib/wasix/src/syscalls/wasix/sock_get_opt_time.rs index 02e8386ad01..eaced230fb0 100644 --- a/lib/wasix/src/syscalls/wasix/sock_get_opt_time.rs +++ b/lib/wasix/src/syscalls/wasix/sock_get_opt_time.rs @@ -8,7 +8,7 @@ use crate::{net::socket::TimeType, syscalls::*}; /// /// * `fd` - Socket descriptor /// * `sockopt` - Socket option to be retrieved -#[instrument(level = "debug", skip_all, fields(%sock, %opt), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %opt), ret)] pub fn sock_get_opt_time( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs index 56ef67e9299..05aa022586b 100644 --- a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs +++ b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v4.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// * `fd` - Socket descriptor /// * `multiaddr` - Multicast group to joined /// * `interface` - Interface that will join -#[instrument(level = "debug", skip_all, fields(%sock), ret)] +#[instrument(level = "trace", skip_all, fields(%sock), ret)] pub fn sock_join_multicast_v4( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs index 420ddfbbd21..f70b827fb50 100644 --- a/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs +++ b/lib/wasix/src/syscalls/wasix/sock_join_multicast_v6.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// * `fd` - Socket descriptor /// * `multiaddr` - Multicast group to joined /// * `interface` - Interface that will join -#[instrument(level = "debug", skip_all, fields(%sock, %iface), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %iface), ret)] pub fn sock_join_multicast_v6( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs index b668c628aa4..12d58489cd0 100644 --- a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs +++ b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v4.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// * `fd` - Socket descriptor /// * `multiaddr` - Multicast group to leave /// * `interface` - Interface that will left -#[instrument(level = "debug", skip_all, fields(%sock), ret)] +#[instrument(level = "trace", skip_all, fields(%sock), ret)] pub fn sock_leave_multicast_v4( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs index 886bfd563e5..33e7888ee03 100644 --- a/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs +++ b/lib/wasix/src/syscalls/wasix/sock_leave_multicast_v6.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// * `fd` - Socket descriptor /// * `multiaddr` - Multicast group to leave /// * `interface` - Interface that will left -#[instrument(level = "debug", skip_all, fields(%sock, %iface), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %iface), ret)] pub fn sock_leave_multicast_v6( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_listen.rs b/lib/wasix/src/syscalls/wasix/sock_listen.rs index c706bd8e2d1..dace9bc0f5e 100644 --- a/lib/wasix/src/syscalls/wasix/sock_listen.rs +++ b/lib/wasix/src/syscalls/wasix/sock_listen.rs @@ -13,7 +13,7 @@ use crate::{journal::SnapshotTrigger, syscalls::*}; /// /// * `fd` - File descriptor of the socket to be bind /// * `backlog` - Maximum size of the queue for pending connections -#[instrument(level = "debug", skip_all, fields(%sock, %backlog), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %backlog), ret)] pub fn sock_listen( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_open.rs b/lib/wasix/src/syscalls/wasix/sock_open.rs index da6de577780..407b0f1cd63 100644 --- a/lib/wasix/src/syscalls/wasix/sock_open.rs +++ b/lib/wasix/src/syscalls/wasix/sock_open.rs @@ -20,7 +20,7 @@ use crate::{net::socket::SocketProperties, syscalls::*}; /// ## Return /// /// The file descriptor of the socket that has been opened. -#[instrument(level = "debug", skip_all, fields(?af, ?ty, ?pt, sock = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(?af, ?ty, ?pt, sock = field::Empty), ret)] pub fn sock_open( mut ctx: FunctionEnvMut<'_, WasiEnv>, af: Addressfamily, diff --git a/lib/wasix/src/syscalls/wasix/sock_send_file.rs b/lib/wasix/src/syscalls/wasix/sock_send_file.rs index 75500152e66..14dd2e5866b 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_file.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_file.rs @@ -15,7 +15,7 @@ use crate::{net::socket::TimeType, syscalls::*, WasiInodes}; /// ## Return /// /// Number of bytes transmitted. -#[instrument(level = "debug", skip_all, fields(%sock, %in_fd, %offset, %count, nsent = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %in_fd, %offset, %count, nsent = field::Empty), ret)] pub fn sock_send_file( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs index 993779520a1..ef0d78cb347 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_flag.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// * `fd` - Socket descriptor /// * `sockopt` - Socket option to be set /// * `flag` - Value to set the option to -#[instrument(level = "debug", skip_all, fields(%sock, %opt, %flag), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %opt, %flag), ret)] pub fn sock_set_opt_flag( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs index a9bdb842b49..73e8a1fd312 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_size.rs @@ -10,7 +10,7 @@ use crate::{net::socket::TimeType, syscalls::*}; /// * `fd` - Socket descriptor /// * `opt` - Socket option to be set /// * `size` - Buffer size -#[instrument(level = "debug", skip_all, fields(%sock, %opt, %size), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %opt, %size), ret)] pub fn sock_set_opt_size( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs b/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs index 6196cc5e4a4..c6adeabf6ce 100644 --- a/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs +++ b/lib/wasix/src/syscalls/wasix/sock_set_opt_time.rs @@ -9,7 +9,7 @@ use crate::{net::socket::TimeType, syscalls::*}; /// * `fd` - Socket descriptor /// * `sockopt` - Socket option to be set /// * `time` - Value to set the time to -#[instrument(level = "debug", skip_all, fields(%sock, %opt, time = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, %opt, time = field::Empty), ret)] pub fn sock_set_opt_time( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_shutdown.rs b/lib/wasix/src/syscalls/wasix/sock_shutdown.rs index 8ef89634e2a..2c1cc72516f 100644 --- a/lib/wasix/src/syscalls/wasix/sock_shutdown.rs +++ b/lib/wasix/src/syscalls/wasix/sock_shutdown.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `how` - Which channels on the socket to shut down. -#[instrument(level = "debug", skip_all, fields(%sock), ret)] +#[instrument(level = "trace", skip_all, fields(%sock), ret)] pub fn sock_shutdown( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_status.rs b/lib/wasix/src/syscalls/wasix/sock_status.rs index 81a7b960927..659a7d34cd5 100644 --- a/lib/wasix/src/syscalls/wasix/sock_status.rs +++ b/lib/wasix/src/syscalls/wasix/sock_status.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `sock_status()` /// Returns the current status of a socket -#[instrument(level = "debug", skip_all, fields(%sock, status = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(%sock, status = field::Empty), ret)] pub fn sock_status( mut ctx: FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/thread_exit.rs b/lib/wasix/src/syscalls/wasix/thread_exit.rs index 9282f440a29..e6f348c7ead 100644 --- a/lib/wasix/src/syscalls/wasix/thread_exit.rs +++ b/lib/wasix/src/syscalls/wasix/thread_exit.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `rval` - The exit code returned by the process. -#[instrument(level = "debug", skip_all, fields(%exitcode), ret)] +#[instrument(level = "trace", skip_all, fields(%exitcode), ret)] pub fn thread_exit(ctx: FunctionEnvMut<'_, WasiEnv>, exitcode: ExitCode) -> Result<(), WasiError> { tracing::debug!(tid=%ctx.data().thread.id(), %exitcode); Err(WasiError::Exit(exitcode)) diff --git a/lib/wasix/src/syscalls/wasix/thread_join.rs b/lib/wasix/src/syscalls/wasix/thread_join.rs index 9b9c87816b1..511e5d0e47b 100644 --- a/lib/wasix/src/syscalls/wasix/thread_join.rs +++ b/lib/wasix/src/syscalls/wasix/thread_join.rs @@ -10,7 +10,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `tid` - Handle of the thread to wait on -//#[instrument(level = "debug", skip_all, fields(%join_tid), ret)] +//#[instrument(level = "trace", skip_all, fields(%join_tid), ret)] pub fn thread_join( ctx: FunctionEnvMut<'_, WasiEnv>, join_tid: Tid, diff --git a/lib/wasix/src/syscalls/wasix/thread_parallelism.rs b/lib/wasix/src/syscalls/wasix/thread_parallelism.rs index c8c97587e46..51c57707562 100644 --- a/lib/wasix/src/syscalls/wasix/thread_parallelism.rs +++ b/lib/wasix/src/syscalls/wasix/thread_parallelism.rs @@ -4,7 +4,7 @@ use crate::syscalls::*; /// ### `thread_parallelism()` /// Returns the available parallelism which is normally the /// number of available cores that can run concurrently -#[instrument(level = "debug", skip_all, fields(parallelism = field::Empty), ret)] +#[instrument(level = "trace", skip_all, fields(parallelism = field::Empty), ret)] pub fn thread_parallelism( ctx: FunctionEnvMut<'_, WasiEnv>, ret_parallelism: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/thread_signal.rs b/lib/wasix/src/syscalls/wasix/thread_signal.rs index 87000502c11..b2c27b6b593 100644 --- a/lib/wasix/src/syscalls/wasix/thread_signal.rs +++ b/lib/wasix/src/syscalls/wasix/thread_signal.rs @@ -7,7 +7,7 @@ use crate::syscalls::*; /// Inputs: /// - `Signal` /// Signal to be raised for this process -#[instrument(level = "debug", skip_all, fields(%tid, ?sig), ret)] +#[instrument(level = "trace", skip_all, fields(%tid, ?sig), ret)] pub fn thread_signal( mut ctx: FunctionEnvMut<'_, WasiEnv>, tid: Tid, diff --git a/lib/wasix/src/syscalls/wasix/thread_sleep.rs b/lib/wasix/src/syscalls/wasix/thread_sleep.rs index 4925d4d0c79..93031f70d12 100644 --- a/lib/wasix/src/syscalls/wasix/thread_sleep.rs +++ b/lib/wasix/src/syscalls/wasix/thread_sleep.rs @@ -9,7 +9,7 @@ use crate::syscalls::*; /// ## Parameters /// /// * `duration` - Amount of time that the thread should sleep -#[instrument(level = "debug", skip_all, fields(%duration), ret)] +#[instrument(level = "trace", skip_all, fields(%duration), ret)] pub fn thread_sleep( mut ctx: FunctionEnvMut<'_, WasiEnv>, duration: Timestamp, diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index d40b11afbca..3f47e16fed2 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -30,7 +30,7 @@ use wasmer_wasix_types::wasi::ThreadStart; /// /// Returns the thread index of the newly created thread /// (indices always start from the same value as `pid` and increments in steps) -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn thread_spawn_v2( mut ctx: FunctionEnvMut<'_, WasiEnv>, start_ptr: WasmPtr, M>, diff --git a/lib/wasix/src/syscalls/wasix/tty_get.rs b/lib/wasix/src/syscalls/wasix/tty_get.rs index 937917a5afe..93d28dd7408 100644 --- a/lib/wasix/src/syscalls/wasix/tty_get.rs +++ b/lib/wasix/src/syscalls/wasix/tty_get.rs @@ -3,7 +3,7 @@ use crate::syscalls::*; /// ### `tty_get()` /// Retrieves the current state of the TTY -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn tty_get( ctx: FunctionEnvMut<'_, WasiEnv>, tty_state: WasmPtr, diff --git a/lib/wasix/src/syscalls/wasix/tty_set.rs b/lib/wasix/src/syscalls/wasix/tty_set.rs index 24866d9f5b6..6738b09a23b 100644 --- a/lib/wasix/src/syscalls/wasix/tty_set.rs +++ b/lib/wasix/src/syscalls/wasix/tty_set.rs @@ -3,7 +3,7 @@ use crate::{syscalls::*, WasiTtyState}; /// ### `tty_set()` /// Updates the properties of the rect -#[instrument(level = "debug", skip_all, ret)] +#[instrument(level = "trace", skip_all, ret)] pub fn tty_set( mut ctx: FunctionEnvMut<'_, WasiEnv>, tty_state: WasmPtr, From 1eb8471f1803e81c7decf795390e24ecdf1f7f1b Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 20 Sep 2024 18:16:10 +0330 Subject: [PATCH 33/53] transfer the file size when renamed --- lib/wasix/src/syscalls/wasi/fd_filestat_get.rs | 6 +++++- lib/wasix/src/syscalls/wasi/path_rename.rs | 12 +++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs index 709b6c8037f..6d65e53dc4d 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_get.rs @@ -10,7 +10,7 @@ use crate::types::wasi::Snapshot0Filestat; /// Output: /// - `Filestat *buf` /// Where the metadata from `fd` will be written -#[instrument(level = "trace", skip_all, fields(%fd), ret)] +#[instrument(level = "trace", skip_all, fields(%fd, size = field::Empty, mtime = field::Empty), ret)] pub fn fd_filestat_get( mut ctx: FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, @@ -18,6 +18,10 @@ pub fn fd_filestat_get( ) -> Errno { let stat = wasi_try!(fd_filestat_get_internal(&mut ctx, fd)); + // These two values have proved to be helpful in multiple investigations + Span::current().record("size", stat.st_size); + Span::current().record("mtime", stat.st_mtim); + let env = ctx.data(); let (memory, _) = unsafe { env.get_memory_and_wasi_state(&ctx, 0) }; let buf = buf.deref(&memory); diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index 37ab9145f42..3fc1d238ed5 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -77,9 +77,10 @@ pub fn path_rename_internal( .fs .get_inode_at_path(inodes, source_fd, source_path, true)); // Create the destination inode if the file exists. - let _ = state - .fs - .get_inode_at_path(inodes, target_fd, target_path, true); + let target_inode = + wasi_try_ok!(state + .fs + .get_inode_at_path(inodes, target_fd, target_path, true)); let (source_parent_inode, source_entry_name) = wasi_try_ok!(state.fs.get_parent_inode_at_path( inodes, source_fd, @@ -201,6 +202,8 @@ pub fn path_rename_internal( } } + let source_size = source_entry.stat.read().unwrap().st_size; + if need_create { let mut guard = target_parent_inode.write(); if let Kind::Dir { entries, .. } = guard.deref_mut() { @@ -212,5 +215,8 @@ pub fn path_rename_internal( } } + // The target entry is created, one way or the other + target_inode.stat.write().unwrap().st_size = source_size; + Ok(Errno::Success) } From ea1f3b24453b6213caf46db3c0f675d83bd4d229 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 20 Sep 2024 18:32:59 +0330 Subject: [PATCH 34/53] fix assuming target exists --- lib/wasix/src/syscalls/wasi/path_rename.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/wasix/src/syscalls/wasi/path_rename.rs b/lib/wasix/src/syscalls/wasi/path_rename.rs index 3fc1d238ed5..c334d6ff250 100644 --- a/lib/wasix/src/syscalls/wasi/path_rename.rs +++ b/lib/wasix/src/syscalls/wasi/path_rename.rs @@ -77,10 +77,9 @@ pub fn path_rename_internal( .fs .get_inode_at_path(inodes, source_fd, source_path, true)); // Create the destination inode if the file exists. - let target_inode = - wasi_try_ok!(state - .fs - .get_inode_at_path(inodes, target_fd, target_path, true)); + let _ = state + .fs + .get_inode_at_path(inodes, target_fd, target_path, true); let (source_parent_inode, source_entry_name) = wasi_try_ok!(state.fs.get_parent_inode_at_path( inodes, source_fd, @@ -216,6 +215,10 @@ pub fn path_rename_internal( } // The target entry is created, one way or the other + let target_inode = + wasi_try_ok!(state + .fs + .get_inode_at_path(inodes, target_fd, target_path, true)); target_inode.stat.write().unwrap().st_size = source_size; Ok(Errno::Success) From de4ffd18c84388d13d584ad884d2c15ee8f9d8b5 Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Mon, 23 Sep 2024 19:49:54 +0400 Subject: [PATCH 35/53] Fix --entrypoint not working for WASI(X) modules --- lib/cli/src/commands/run/mod.rs | 4 ++++ lib/wasix/src/runners/wasi.rs | 14 ++++++++++++++ lib/wasix/src/runners/wasi_common.rs | 5 +++++ lib/wasix/src/state/builder.rs | 24 +++++++++++++++++++++++- 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index eefc629f72a..9a414c5a58d 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -425,6 +425,10 @@ impl Run { .with_forward_host_env(self.wasi.forward_host_env) .with_capabilities(self.wasi.capabilities()); + if let Some(ref entrypoint) = self.entrypoint { + runner.with_entrypoint(entrypoint); + } + #[cfg(feature = "journal")] { for trigger in self.wasi.snapshot_on.iter().cloned() { diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index f69f7e9ae1d..f02358de565 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -34,6 +34,20 @@ impl WasiRunner { WasiRunner::default() } + /// Returns the current entrypoint for this `WasiRunner` + pub fn entrypoint(&self) -> Option { + self.wasi.entrypoint.clone() + } + + /// Builder method to set the name of the entrypoint function for this `WasiRunner` + pub fn with_entrypoint(&mut self, entrypoint: S) -> &mut Self + where + S: Into, + { + self.wasi.entrypoint = Some(entrypoint.into()); + self + } + /// Returns the current arguments for this `WasiRunner` pub fn get_args(&self) -> Vec { self.wasi.args.clone() diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 82b2135f98a..f265d275911 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -32,6 +32,7 @@ pub struct MappedCommand { #[derive(Derivative, Default, Clone)] #[derivative(Debug)] pub(crate) struct CommonWasiOptions { + pub(crate) entrypoint: Option, pub(crate) args: Vec, pub(crate) env: HashMap, pub(crate) forward_host_env: bool, @@ -57,6 +58,10 @@ impl CommonWasiOptions { wasi: &WasiAnnotation, root_fs: Option, ) -> Result<(), anyhow::Error> { + if let Some(ref entrypoint) = self.entrypoint { + builder.set_entrypoint(entrypoint); + } + let root_fs = root_fs.unwrap_or_else(|| { RootFileSystemBuilder::default() .with_tmp(!self.is_tmp_mapped) diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index b35bf444d16..65d81d5c0bc 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -49,6 +49,8 @@ use super::env::WasiEnvInit; /// ``` #[derive(Default)] pub struct WasiEnvBuilder { + /// Name of entrypoint function. Defaults to running `_start` if not specified. + pub(super) entrypoint: Option, /// Command line arguments. pub(super) args: Vec, /// Environment variables. @@ -97,6 +99,7 @@ impl std::fmt::Debug for WasiEnvBuilder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // TODO: update this when stable f.debug_struct("WasiEnvBuilder") + .field("entrypoint", &self.entrypoint) .field("args", &self.args) .field("envs", &self.envs) .field("preopens", &self.preopens) @@ -240,6 +243,21 @@ impl WasiEnvBuilder { &mut self.envs } + pub fn entrypoint(mut self, entrypoint: S) -> Self + where + S: AsRef, + { + self.set_entrypoint(entrypoint); + self + } + + pub fn set_entrypoint(&mut self, entrypoint: S) + where + S: AsRef, + { + self.entrypoint = Some(entrypoint.as_ref().to_owned()); + } + /// Add an argument. /// /// Arguments must not contain the nul (0x0) byte @@ -1024,6 +1042,8 @@ impl WasiEnvBuilder { ); } + let entrypoint = self.entrypoint.clone(); + let (instance, env) = self.instantiate_ext(module, module_hash, store)?; // Bootstrap the process @@ -1036,7 +1056,9 @@ impl WasiEnvBuilder { .map_err(|exit| WasiRuntimeError::Wasi(WasiError::Exit(exit)))?; } - let start = instance.exports.get_function("_start")?; + let start = instance + .exports + .get_function(entrypoint.as_deref().unwrap_or("_start"))?; env.data(&store).thread.set_status_running(); let result = crate::run_wasi_func_start(start, store); From 3eba516711a0167df9730daa4341eec6f42e6935 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Mon, 23 Sep 2024 18:44:35 +0200 Subject: [PATCH 36/53] fix(middlewares): Fix error in metering middleware --- lib/middlewares/src/metering.rs | 109 ++++++++++++++++++++++---------- 1 file changed, 74 insertions(+), 35 deletions(-) diff --git a/lib/middlewares/src/metering.rs b/lib/middlewares/src/metering.rs index 18bc7997146..1207269a2da 100644 --- a/lib/middlewares/src/metering.rs +++ b/lib/middlewares/src/metering.rs @@ -124,6 +124,10 @@ pub enum MeteringPoints { impl u64 + Send + Sync> Metering { /// Creates a `Metering` middleware. + /// + /// When providing a cost function, you should consider that branching operations do + /// additional work to track the metering points and probably need to have a higher cost. + /// To find out which operations are affected by this, you can call [`is_accounting`]. pub fn new(initial_limit: u64, cost_function: F) -> Self { Self { initial_limit, @@ -198,6 +202,42 @@ impl u64 + Send + Sync + 'static> ModuleMiddleware for Meter } } +/// Returns `true` if and only if the given operator is an accounting operator. +/// Accounting operators do additional work to track the metering points. +pub fn is_accounting(operator: &Operator) -> bool { + // Possible sources and targets of a branch. + matches!( + operator, + Operator::Loop { .. } // loop headers are branch targets + | Operator::End // block ends are branch targets + | Operator::If { .. } // branch source, "if" can branch to else branch + | Operator::Else // "else" is the "end" of an if branch + | Operator::Br { .. } // branch source + | Operator::BrTable { .. } // branch source + | Operator::BrIf { .. } // branch source + | Operator::Call { .. } // function call - branch source + | Operator::CallIndirect { .. } // function call - branch source + | Operator::Return // end of function - branch source + // exceptions proposal + | Operator::Throw { .. } // branch source + | Operator::ThrowRef // branch source + | Operator::Rethrow { .. } // branch source + | Operator::Delegate { .. } // branch source + | Operator::Catch { .. } // branch target + // tail_call proposal + | Operator::ReturnCall { .. } // branch source + | Operator::ReturnCallIndirect { .. } // branch source + // gc proposal + | Operator::BrOnCast { .. } // branch source + | Operator::BrOnCastFail { .. } // branch source + // function_references proposal + | Operator::CallRef { .. } // branch source + | Operator::ReturnCallRef { .. } // branch source + | Operator::BrOnNull { .. } // branch source + | Operator::BrOnNonNull { .. } // branch source + ) +} + impl u64 + Send + Sync> fmt::Debug for FunctionMetering { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("FunctionMetering") @@ -218,41 +258,40 @@ impl u64 + Send + Sync> FunctionMiddleware for FunctionMeter // corner cases. self.accumulated_cost += (self.cost_function)(&operator); - // Possible sources and targets of a branch. Finalize the cost of the previous basic block and perform necessary checks. - match operator { - Operator::Loop { .. } // loop headers are branch targets - | Operator::End // block ends are branch targets - | Operator::Else // "else" is the "end" of an if branch - | Operator::Br { .. } // branch source - | Operator::BrTable { .. } // branch source - | Operator::BrIf { .. } // branch source - | Operator::Call { .. } // function call - branch source - | Operator::CallIndirect { .. } // function call - branch source - | Operator::Return // end of function - branch source - => { - if self.accumulated_cost > 0 { - state.extend(&[ - // if unsigned(globals[remaining_points_index]) < unsigned(self.accumulated_cost) { throw(); } - Operator::GlobalGet { global_index: self.global_indexes.remaining_points().as_u32() }, - Operator::I64Const { value: self.accumulated_cost as i64 }, - Operator::I64LtU, - Operator::If { blockty: WpTypeOrFuncType::Empty }, - Operator::I32Const { value: 1 }, - Operator::GlobalSet { global_index: self.global_indexes.points_exhausted().as_u32() }, - Operator::Unreachable, - Operator::End, - - // globals[remaining_points_index] -= self.accumulated_cost; - Operator::GlobalGet { global_index: self.global_indexes.remaining_points().as_u32() }, - Operator::I64Const { value: self.accumulated_cost as i64 }, - Operator::I64Sub, - Operator::GlobalSet { global_index: self.global_indexes.remaining_points().as_u32() }, - ]); - - self.accumulated_cost = 0; - } - } - _ => {} + // Finalize the cost of the previous basic block and perform necessary checks. + if is_accounting(&operator) && self.accumulated_cost > 0 { + state.extend(&[ + // if unsigned(globals[remaining_points_index]) < unsigned(self.accumulated_cost) { throw(); } + Operator::GlobalGet { + global_index: self.global_indexes.remaining_points().as_u32(), + }, + Operator::I64Const { + value: self.accumulated_cost as i64, + }, + Operator::I64LtU, + Operator::If { + blockty: WpTypeOrFuncType::Empty, + }, + Operator::I32Const { value: 1 }, + Operator::GlobalSet { + global_index: self.global_indexes.points_exhausted().as_u32(), + }, + Operator::Unreachable, + Operator::End, + // globals[remaining_points_index] -= self.accumulated_cost; + Operator::GlobalGet { + global_index: self.global_indexes.remaining_points().as_u32(), + }, + Operator::I64Const { + value: self.accumulated_cost as i64, + }, + Operator::I64Sub, + Operator::GlobalSet { + global_index: self.global_indexes.remaining_points().as_u32(), + }, + ]); + + self.accumulated_cost = 0; } state.push_operator(operator); From 1485c0134c5d3a062f881ef255faf1e56ae57296 Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Tue, 24 Sep 2024 12:23:33 +0400 Subject: [PATCH 37/53] Remove 'invoke' alias from --entrypoint, add separate --invoke arg for single WASM modules (breaking change to Wasmer CLI args) --- lib/cli/src/commands/run/mod.rs | 20 ++++++++++++-------- lib/wasix/src/runners/wasi.rs | 12 ++++++------ lib/wasix/src/runners/wasi_common.rs | 6 +++--- lib/wasix/src/state/builder.rs | 18 +++++++++--------- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 9a414c5a58d..3966aaa6cc1 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -76,9 +76,12 @@ pub struct Run { /// Set the default stack size (default is 1048576) #[clap(long = "stack-size")] stack_size: Option, - /// The function or command to invoke. - #[clap(short, long, aliases = &["command", "invoke", "command-name"])] + /// The entrypoint module for webc packages. + #[clap(short, long, aliases = &["command", "command-name"])] entrypoint: Option, + /// The function to invoke. + #[clap(short, long)] + invoke: Option, /// Generate a coredump at this path if a WebAssembly trap occurs #[clap(name = "COREDUMP_PATH", long)] coredump_on_trap: Option, @@ -377,19 +380,19 @@ impl Run { let instance = Instance::new(store, module, &imports) .context("Unable to instantiate the WebAssembly module")?; - let entrypoint = match &self.entrypoint { + let entry_function = match &self.invoke { Some(entry) => { instance.exports .get_function(entry) - .with_context(|| format!("The module doesn't contain a \"{entry}\" function"))? + .with_context(|| format!("The module doesn't export a function named \"{entry}\""))? }, None => { instance.exports.get_function("_start") - .context("The module doesn't contain a \"_start\" function. Either implement it or specify an entrypoint function.")? + .context("The module doesn't export a \"_start\" function. Either implement it or specify an entry function with --invoke")? } }; - let return_values = invoke_function(&instance, store, entrypoint, &self.args)?; + let return_values = invoke_function(&instance, store, entry_function, &self.args)?; println!( "{}", @@ -425,8 +428,8 @@ impl Run { .with_forward_host_env(self.wasi.forward_host_env) .with_capabilities(self.wasi.capabilities()); - if let Some(ref entrypoint) = self.entrypoint { - runner.with_entrypoint(entrypoint); + if let Some(ref entry_function) = self.invoke { + runner.with_entry_function(entry_function); } #[cfg(feature = "journal")] @@ -523,6 +526,7 @@ impl Run { wcgi: WcgiOptions::default(), stack_size: None, entrypoint: Some(original_executable.to_string()), + invoke: None, coredump_on_trap: None, input: PackageSource::infer(executable)?, args: args.to_vec(), diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index f02358de565..b8301be192f 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -34,17 +34,17 @@ impl WasiRunner { WasiRunner::default() } - /// Returns the current entrypoint for this `WasiRunner` - pub fn entrypoint(&self) -> Option { - self.wasi.entrypoint.clone() + /// Returns the current entry function for this `WasiRunner` + pub fn entry_function(&self) -> Option { + self.wasi.entry_function.clone() } - /// Builder method to set the name of the entrypoint function for this `WasiRunner` - pub fn with_entrypoint(&mut self, entrypoint: S) -> &mut Self + /// Builder method to set the name of the entry function for this `WasiRunner` + pub fn with_entry_function(&mut self, entry_function: S) -> &mut Self where S: Into, { - self.wasi.entrypoint = Some(entrypoint.into()); + self.wasi.entry_function = Some(entry_function.into()); self } diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index f265d275911..85ff26b548c 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -32,7 +32,7 @@ pub struct MappedCommand { #[derive(Derivative, Default, Clone)] #[derivative(Debug)] pub(crate) struct CommonWasiOptions { - pub(crate) entrypoint: Option, + pub(crate) entry_function: Option, pub(crate) args: Vec, pub(crate) env: HashMap, pub(crate) forward_host_env: bool, @@ -58,8 +58,8 @@ impl CommonWasiOptions { wasi: &WasiAnnotation, root_fs: Option, ) -> Result<(), anyhow::Error> { - if let Some(ref entrypoint) = self.entrypoint { - builder.set_entrypoint(entrypoint); + if let Some(ref entry_function) = self.entry_function { + builder.set_entry_function(entry_function); } let root_fs = root_fs.unwrap_or_else(|| { diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index 65d81d5c0bc..ec64455f1f6 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -49,8 +49,8 @@ use super::env::WasiEnvInit; /// ``` #[derive(Default)] pub struct WasiEnvBuilder { - /// Name of entrypoint function. Defaults to running `_start` if not specified. - pub(super) entrypoint: Option, + /// Name of entry function. Defaults to running `_start` if not specified. + pub(super) entry_function: Option, /// Command line arguments. pub(super) args: Vec, /// Environment variables. @@ -99,7 +99,7 @@ impl std::fmt::Debug for WasiEnvBuilder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // TODO: update this when stable f.debug_struct("WasiEnvBuilder") - .field("entrypoint", &self.entrypoint) + .field("entry_function", &self.entry_function) .field("args", &self.args) .field("envs", &self.envs) .field("preopens", &self.preopens) @@ -243,19 +243,19 @@ impl WasiEnvBuilder { &mut self.envs } - pub fn entrypoint(mut self, entrypoint: S) -> Self + pub fn entry_function(mut self, entry_function: S) -> Self where S: AsRef, { - self.set_entrypoint(entrypoint); + self.set_entry_function(entry_function); self } - pub fn set_entrypoint(&mut self, entrypoint: S) + pub fn set_entry_function(&mut self, entry_function: S) where S: AsRef, { - self.entrypoint = Some(entrypoint.as_ref().to_owned()); + self.entry_function = Some(entry_function.as_ref().to_owned()); } /// Add an argument. @@ -1042,7 +1042,7 @@ impl WasiEnvBuilder { ); } - let entrypoint = self.entrypoint.clone(); + let entry_function = self.entry_function.clone(); let (instance, env) = self.instantiate_ext(module, module_hash, store)?; @@ -1058,7 +1058,7 @@ impl WasiEnvBuilder { let start = instance .exports - .get_function(entrypoint.as_deref().unwrap_or("_start"))?; + .get_function(entry_function.as_deref().unwrap_or("_start"))?; env.data(&store).thread.set_status_running(); let result = crate::run_wasi_func_start(start, store); From 8f3f3ade04c7a24369690a03a13f50086202893d Mon Sep 17 00:00:00 2001 From: Arshia001 Date: Tue, 24 Sep 2024 14:09:13 +0400 Subject: [PATCH 38/53] Revert "Add new --invoke arg for choosing entry function for single WASM modules + fix --invoke not working for WASI(X) modules" --- lib/cli/src/commands/run/mod.rs | 20 ++++++-------------- lib/wasix/src/runners/wasi.rs | 14 -------------- lib/wasix/src/runners/wasi_common.rs | 5 ----- lib/wasix/src/state/builder.rs | 24 +----------------------- 4 files changed, 7 insertions(+), 56 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 3966aaa6cc1..eefc629f72a 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -76,12 +76,9 @@ pub struct Run { /// Set the default stack size (default is 1048576) #[clap(long = "stack-size")] stack_size: Option, - /// The entrypoint module for webc packages. - #[clap(short, long, aliases = &["command", "command-name"])] + /// The function or command to invoke. + #[clap(short, long, aliases = &["command", "invoke", "command-name"])] entrypoint: Option, - /// The function to invoke. - #[clap(short, long)] - invoke: Option, /// Generate a coredump at this path if a WebAssembly trap occurs #[clap(name = "COREDUMP_PATH", long)] coredump_on_trap: Option, @@ -380,19 +377,19 @@ impl Run { let instance = Instance::new(store, module, &imports) .context("Unable to instantiate the WebAssembly module")?; - let entry_function = match &self.invoke { + let entrypoint = match &self.entrypoint { Some(entry) => { instance.exports .get_function(entry) - .with_context(|| format!("The module doesn't export a function named \"{entry}\""))? + .with_context(|| format!("The module doesn't contain a \"{entry}\" function"))? }, None => { instance.exports.get_function("_start") - .context("The module doesn't export a \"_start\" function. Either implement it or specify an entry function with --invoke")? + .context("The module doesn't contain a \"_start\" function. Either implement it or specify an entrypoint function.")? } }; - let return_values = invoke_function(&instance, store, entry_function, &self.args)?; + let return_values = invoke_function(&instance, store, entrypoint, &self.args)?; println!( "{}", @@ -428,10 +425,6 @@ impl Run { .with_forward_host_env(self.wasi.forward_host_env) .with_capabilities(self.wasi.capabilities()); - if let Some(ref entry_function) = self.invoke { - runner.with_entry_function(entry_function); - } - #[cfg(feature = "journal")] { for trigger in self.wasi.snapshot_on.iter().cloned() { @@ -526,7 +519,6 @@ impl Run { wcgi: WcgiOptions::default(), stack_size: None, entrypoint: Some(original_executable.to_string()), - invoke: None, coredump_on_trap: None, input: PackageSource::infer(executable)?, args: args.to_vec(), diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index b8301be192f..f69f7e9ae1d 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -34,20 +34,6 @@ impl WasiRunner { WasiRunner::default() } - /// Returns the current entry function for this `WasiRunner` - pub fn entry_function(&self) -> Option { - self.wasi.entry_function.clone() - } - - /// Builder method to set the name of the entry function for this `WasiRunner` - pub fn with_entry_function(&mut self, entry_function: S) -> &mut Self - where - S: Into, - { - self.wasi.entry_function = Some(entry_function.into()); - self - } - /// Returns the current arguments for this `WasiRunner` pub fn get_args(&self) -> Vec { self.wasi.args.clone() diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index 85ff26b548c..82b2135f98a 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -32,7 +32,6 @@ pub struct MappedCommand { #[derive(Derivative, Default, Clone)] #[derivative(Debug)] pub(crate) struct CommonWasiOptions { - pub(crate) entry_function: Option, pub(crate) args: Vec, pub(crate) env: HashMap, pub(crate) forward_host_env: bool, @@ -58,10 +57,6 @@ impl CommonWasiOptions { wasi: &WasiAnnotation, root_fs: Option, ) -> Result<(), anyhow::Error> { - if let Some(ref entry_function) = self.entry_function { - builder.set_entry_function(entry_function); - } - let root_fs = root_fs.unwrap_or_else(|| { RootFileSystemBuilder::default() .with_tmp(!self.is_tmp_mapped) diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index ec64455f1f6..b35bf444d16 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -49,8 +49,6 @@ use super::env::WasiEnvInit; /// ``` #[derive(Default)] pub struct WasiEnvBuilder { - /// Name of entry function. Defaults to running `_start` if not specified. - pub(super) entry_function: Option, /// Command line arguments. pub(super) args: Vec, /// Environment variables. @@ -99,7 +97,6 @@ impl std::fmt::Debug for WasiEnvBuilder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // TODO: update this when stable f.debug_struct("WasiEnvBuilder") - .field("entry_function", &self.entry_function) .field("args", &self.args) .field("envs", &self.envs) .field("preopens", &self.preopens) @@ -243,21 +240,6 @@ impl WasiEnvBuilder { &mut self.envs } - pub fn entry_function(mut self, entry_function: S) -> Self - where - S: AsRef, - { - self.set_entry_function(entry_function); - self - } - - pub fn set_entry_function(&mut self, entry_function: S) - where - S: AsRef, - { - self.entry_function = Some(entry_function.as_ref().to_owned()); - } - /// Add an argument. /// /// Arguments must not contain the nul (0x0) byte @@ -1042,8 +1024,6 @@ impl WasiEnvBuilder { ); } - let entry_function = self.entry_function.clone(); - let (instance, env) = self.instantiate_ext(module, module_hash, store)?; // Bootstrap the process @@ -1056,9 +1036,7 @@ impl WasiEnvBuilder { .map_err(|exit| WasiRuntimeError::Wasi(WasiError::Exit(exit)))?; } - let start = instance - .exports - .get_function(entry_function.as_deref().unwrap_or("_start"))?; + let start = instance.exports.get_function("_start")?; env.data(&store).thread.set_status_running(); let result = crate::run_wasi_func_start(start, store); From 1ccda3351606189d077106a6b1754448625668b1 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 26 Sep 2024 08:53:16 +0200 Subject: [PATCH 39/53] feat(wasix): InMemorySource: add named packages as hashed too Register named packages as hashed packages as well to make them available by hash. --- lib/wasix/src/runtime/resolver/in_memory_source.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/wasix/src/runtime/resolver/in_memory_source.rs index 03553b2c042..727378118b2 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/wasix/src/runtime/resolver/in_memory_source.rs @@ -68,9 +68,20 @@ impl InMemorySource { } /// Add a new [`PackageSummary`] to the [`InMemorySource`]. + /// + /// Named packages are also made accessible by their hash. pub fn add(&mut self, summary: PackageSummary) { match summary.pkg.id.clone() { PackageId::Named(ident) => { + // Also add the package as a hashed package. + let pkg_hash = PackageHash::Sha256(wasmer_config::hash::Sha256Hash( + summary.dist.webc_sha256.as_bytes(), + )); + if !self.hash_packages.contains_key(&pkg_hash) { + self.hash_packages.insert(pkg_hash, summary.clone()); + } + + // Add the named package. let summaries = self .named_packages .entry(ident.full_name.clone()) From d266ffbed065d352f312d08a7e3b52be557b4c5c Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 26 Sep 2024 08:54:08 +0200 Subject: [PATCH 40/53] feat(wasix): Add InMemorySource::{is_empty,len} convenience helpers --- lib/wasix/src/runtime/resolver/in_memory_source.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/wasix/src/runtime/resolver/in_memory_source.rs index 727378118b2..dbe3ef78b7d 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/wasix/src/runtime/resolver/in_memory_source.rs @@ -118,6 +118,17 @@ impl InMemorySource { PackageId::Hash(hash) => self.hash_packages.get(hash), } } + + pub fn is_empty(&self) -> bool { + self.named_packages.is_empty() && self.hash_packages.is_empty() + } + + /// Returns the number of packages in the source. + pub fn len(&self) -> usize { + // Only need to count the hash packages, + // as the named packages are also always added as hashed. + self.hash_packages.len() + } } #[async_trait::async_trait] From f1a08790cd3a709f6b3f8e212cfe6338b672da4f Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 26 Sep 2024 08:54:33 +0200 Subject: [PATCH 41/53] feat(wasix): Allow result merging in the MultiSource Make it possible to merge results from multiple sources together, ignoring already found packages. Results from all sources are combined. This is usually what you want, so it's on by default. --- .../src/runtime/resolver/multi_source.rs | 96 ++++++++++++++++--- 1 file changed, 85 insertions(+), 11 deletions(-) diff --git a/lib/wasix/src/runtime/resolver/multi_source.rs b/lib/wasix/src/runtime/resolver/multi_source.rs index 2dc3af3c4c3..7178a8d25dc 100644 --- a/lib/wasix/src/runtime/resolver/multi_source.rs +++ b/lib/wasix/src/runtime/resolver/multi_source.rs @@ -25,7 +25,7 @@ pub struct MultiSource { } impl MultiSource { - pub const fn new() -> Self { + pub fn new() -> Self { MultiSource { sources: Vec::new(), strategy: MultiSourceStrategy::default(), @@ -51,19 +51,47 @@ impl MultiSource { impl Source for MultiSource { #[tracing::instrument(level = "debug", skip_all, fields(%package))] async fn query(&self, package: &PackageSource) -> Result, QueryError> { + let mut output = Vec::::new(); + for source in &self.sources { match source.query(package).await { - Ok(summaries) => return Ok(summaries), - Err(QueryError::Unsupported) if self.strategy.continue_if_unsupported => continue, - Err(QueryError::NotFound) if self.strategy.continue_if_not_found => continue, - Err(QueryError::NoMatches { .. }) if self.strategy.continue_if_no_matches => { + Ok(mut summaries) => { + if self.strategy.merge_results { + // Extend matches, but skip already found versions. + summaries.retain(|new| { + !output.iter().any(|existing| new.pkg.id == existing.pkg.id) + }); + output.extend(summaries); + } else { + return Ok(summaries); + } + } + Err(QueryError::Unsupported) + if self.strategy.continue_if_unsupported || self.strategy.merge_results => + { + continue + } + Err(QueryError::NotFound) + if self.strategy.continue_if_not_found || self.strategy.merge_results => + { + continue + } + Err(QueryError::NoMatches { .. }) + if self.strategy.continue_if_no_matches || self.strategy.merge_results => + { continue } Err(e) => return Err(e), } } - Err(QueryError::NotFound) + if !output.is_empty() { + output.sort_by(|a, b| a.pkg.id.cmp(&b.pkg.id)); + + Ok(output) + } else { + Err(QueryError::NotFound) + } } } @@ -86,20 +114,66 @@ pub struct MultiSourceStrategy { /// /// This flag is **disabled** by default. pub continue_if_no_matches: bool, + + /// Merge results from all sources into a single result. + /// + /// True by default. + pub merge_results: bool, } -impl MultiSourceStrategy { - pub const fn default() -> Self { +impl Default for MultiSourceStrategy { + fn default() -> Self { MultiSourceStrategy { continue_if_unsupported: true, continue_if_not_found: true, continue_if_no_matches: true, + merge_results: true, } } } -impl Default for MultiSourceStrategy { - fn default() -> Self { - MultiSourceStrategy::default() +#[cfg(test)] +mod tests { + use wasmer_config::package::PackageId; + + use super::super::{DistributionInfo, InMemorySource, PackageInfo, WebcHash}; + use super::*; + + /// Test that the `MultiSource` can merge results from multiple sources. + #[tokio::test] + async fn test_multi_source_merge() { + let id1 = PackageId::new_named("ns/pkg", "0.0.1".parse().unwrap()); + let pkg1 = PackageSummary { + pkg: PackageInfo { + id: id1.clone(), + commands: Vec::new(), + entrypoint: None, + dependencies: Vec::new(), + filesystem: Vec::new(), + }, + dist: DistributionInfo { + webc: "https://example.com/ns/pkg/0.0.1".parse().unwrap(), + webc_sha256: WebcHash([0u8; 32]), + }, + }; + + let id2 = PackageId::new_named("ns/pkg", "0.0.2".parse().unwrap()); + let mut pkg2 = pkg1.clone(); + pkg2.pkg.id = id2.clone(); + + let mut mem1 = InMemorySource::new(); + mem1.add(pkg1); + + let mut mem2 = InMemorySource::new(); + mem2.add(pkg2); + + let mut multi = MultiSource::new(); + multi.add_source(mem1); + multi.add_source(mem2); + + let summaries = multi.query(&"ns/pkg".parse().unwrap()).await.unwrap(); + assert_eq!(summaries.len(), 2); + assert_eq!(summaries[0].pkg.id, id1); + assert_eq!(summaries[1].pkg.id, id2); } } From 1834827aa6e8cb81ab1d7f9904b095c842c99a3c Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 26 Sep 2024 09:29:42 +0200 Subject: [PATCH 42/53] chore: Formatting/linting fixes --- lib/wasix/src/runners/wasi.rs | 6 +++--- lib/wasix/src/runtime/package_loader/builtin_loader.rs | 2 +- lib/wasix/src/runtime/resolver/in_memory_source.rs | 6 +++--- lib/wasix/src/runtime/resolver/multi_source.rs | 6 ++++++ lib/wasix/tests/runners.rs | 3 +-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index f69f7e9ae1d..f775ef87e79 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -482,7 +482,7 @@ mod tests { let fs = &init.state.fs.root_fs; - fs.read_dir(&std::path::Path::new("/host")).unwrap(); + fs.read_dir(std::path::Path::new("/host")).unwrap(); } #[cfg(all(feature = "host-fs", feature = "sys"))] @@ -535,7 +535,7 @@ mod tests { let fs = &init.state.fs.root_fs; - fs.read_dir(&std::path::Path::new("/host")).unwrap(); - fs.read_dir(&std::path::Path::new("/settings")).unwrap(); + fs.read_dir(std::path::Path::new("/host")).unwrap(); + fs.read_dir(std::path::Path::new("/settings")).unwrap(); } } diff --git a/lib/wasix/src/runtime/package_loader/builtin_loader.rs b/lib/wasix/src/runtime/package_loader/builtin_loader.rs index be4ed7de8e5..5c7e7125910 100644 --- a/lib/wasix/src/runtime/package_loader/builtin_loader.rs +++ b/lib/wasix/src/runtime/package_loader/builtin_loader.rs @@ -704,7 +704,7 @@ mod test { let path = dir.path(); let contents = "fail"; - let correct_hash = WebcHash::sha256(&contents); + let correct_hash = WebcHash::sha256(contents); let used_hash = WebcHash::parse_hex("0000a28ea38a000f3a3328cb7fabe330638d3258affe1a869e3f92986222d997") .unwrap(); diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/wasix/src/runtime/resolver/in_memory_source.rs index dbe3ef78b7d..1505cf12099 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/wasix/src/runtime/resolver/in_memory_source.rs @@ -77,9 +77,9 @@ impl InMemorySource { let pkg_hash = PackageHash::Sha256(wasmer_config::hash::Sha256Hash( summary.dist.webc_sha256.as_bytes(), )); - if !self.hash_packages.contains_key(&pkg_hash) { - self.hash_packages.insert(pkg_hash, summary.clone()); - } + self.hash_packages + .entry(pkg_hash) + .or_insert_with(|| summary.clone()); // Add the named package. let summaries = self diff --git a/lib/wasix/src/runtime/resolver/multi_source.rs b/lib/wasix/src/runtime/resolver/multi_source.rs index 7178a8d25dc..e876b8225bf 100644 --- a/lib/wasix/src/runtime/resolver/multi_source.rs +++ b/lib/wasix/src/runtime/resolver/multi_source.rs @@ -24,6 +24,12 @@ pub struct MultiSource { strategy: MultiSourceStrategy, } +impl Default for MultiSource { + fn default() -> Self { + Self::new() + } +} + impl MultiSource { pub fn new() -> Self { MultiSource { diff --git a/lib/wasix/tests/runners.rs b/lib/wasix/tests/runners.rs index b58c41e5998..cd77428bdd0 100644 --- a/lib/wasix/tests/runners.rs +++ b/lib/wasix/tests/runners.rs @@ -236,9 +236,8 @@ fn client() -> Client { } builder }; - let client = builder.build().unwrap(); - client + builder.build().unwrap() } #[cfg(not(target_os = "windows"))] From 7ca5d6a54e2f574f2aa3588bb219e9032c615902 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 26 Sep 2024 12:01:55 +0200 Subject: [PATCH 43/53] chore(wasix): Extend QueryError to hold the queried package For better error reporting. --- .../src/runtime/resolver/backend_source.rs | 51 ++++++++++++--- .../src/runtime/resolver/filesystem_source.rs | 60 +++++++++++------ .../src/runtime/resolver/in_memory_source.rs | 10 ++- lib/wasix/src/runtime/resolver/inputs.rs | 6 +- .../src/runtime/resolver/multi_source.rs | 8 ++- lib/wasix/src/runtime/resolver/source.rs | 64 +++++++++++++------ lib/wasix/src/runtime/resolver/web_source.rs | 31 ++++++--- 7 files changed, 165 insertions(+), 65 deletions(-) diff --git a/lib/wasix/src/runtime/resolver/backend_source.rs b/lib/wasix/src/runtime/resolver/backend_source.rs index 00fd3491b54..22590e39935 100644 --- a/lib/wasix/src/runtime/resolver/backend_source.rs +++ b/lib/wasix/src/runtime/resolver/backend_source.rs @@ -217,7 +217,7 @@ impl BackendSource { async fn query_by_hash( &self, hash: &PackageHash, - ) -> Result, QueryError> { + ) -> Result, anyhow::Error> { // FIXME: implementing caching! let Some(data) = self.query_graphql_by_hash(hash).await? else { @@ -241,22 +241,34 @@ impl Source for BackendSource { ), PackageSource::Ident(PackageIdent::Hash(hash)) => { // TODO: implement caching! - match self.query_by_hash(hash).await? { - Some(summary) => return Ok(vec![summary]), - None => { + match self.query_by_hash(hash).await { + Ok(Some(summary)) => return Ok(vec![summary]), + Ok(None) => { return Err(QueryError::NoMatches { + query: package.clone(), archived_versions: Vec::new(), }); } + Err(error) => { + return Err(QueryError::Other { + query: package.clone(), + error, + }); + } } } - _ => return Err(QueryError::Unsupported), + _ => { + return Err(QueryError::Unsupported { + query: package.clone(), + }) + } }; if let Some(cache) = &self.cache { match cache.lookup_cached_query(&package_name) { Ok(Some(cached)) => { if let Ok(cached) = matching_package_summaries( + package, cached, &version_constraint, self.preferred_webc_version, @@ -276,7 +288,13 @@ impl Source for BackendSource { } } - let response = self.query_graphql_named(&package_name).await?; + let response = self + .query_graphql_named(&package_name) + .await + .map_err(|error| QueryError::Other { + query: package.clone(), + error, + })?; if let Some(cache) = &self.cache { if let Err(e) = cache.update(&package_name, &response) { @@ -288,11 +306,18 @@ impl Source for BackendSource { } } - matching_package_summaries(response, &version_constraint, self.preferred_webc_version) + matching_package_summaries( + package, + response, + &version_constraint, + self.preferred_webc_version, + ) } } +#[allow(clippy::result_large_err)] fn matching_package_summaries( + query: &PackageSource, response: WebQuery, version_constraint: &VersionReq, preferred_webc_version: webc::Version, @@ -304,7 +329,12 @@ fn matching_package_summaries( package_name, versions, .. - } = response.data.get_package.ok_or(QueryError::NotFound)?; + } = response + .data + .get_package + .ok_or_else(|| QueryError::NotFound { + query: query.clone(), + })?; let mut archived_versions = Vec::new(); for pkg_version in versions { @@ -349,7 +379,10 @@ fn matching_package_summaries( } if summaries.is_empty() { - Err(QueryError::NoMatches { archived_versions }) + Err(QueryError::NoMatches { + query: query.clone(), + archived_versions, + }) } else { Ok(summaries) } diff --git a/lib/wasix/src/runtime/resolver/filesystem_source.rs b/lib/wasix/src/runtime/resolver/filesystem_source.rs index 471fa18b9b9..59b7a415f0b 100644 --- a/lib/wasix/src/runtime/resolver/filesystem_source.rs +++ b/lib/wasix/src/runtime/resolver/filesystem_source.rs @@ -10,29 +10,14 @@ use crate::runtime::resolver::{ #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct FileSystemSource {} -#[async_trait::async_trait] -impl Source for FileSystemSource { - #[tracing::instrument(level = "debug", skip_all, fields(%package))] - async fn query(&self, package: &PackageSource) -> Result, QueryError> { - let path = match package { - PackageSource::Path(path) => { - let path = std::path::PathBuf::from(path); - path.canonicalize().with_context(|| { - format!( - "Unable to get the canonical form for \"{}\"", - path.display() - ) - })? - } - _ => return Err(QueryError::Unsupported), - }; - - let webc_sha256 = crate::block_in_place(|| WebcHash::for_file(&path)) +impl FileSystemSource { + async fn load_path(path: &std::path::Path) -> Result, anyhow::Error> { + let webc_sha256 = crate::block_in_place(|| WebcHash::for_file(path)) .with_context(|| format!("Unable to hash \"{}\"", path.display()))?; - let container = crate::block_in_place(|| Container::from_disk(&path)) + let container = crate::block_in_place(|| Container::from_disk(path)) .with_context(|| format!("Unable to parse \"{}\"", path.display()))?; - let url = crate::runtime::resolver::utils::url_from_file_path(&path) + let url = crate::runtime::resolver::utils::url_from_file_path(path) .ok_or_else(|| anyhow::anyhow!("Unable to turn \"{}\" into a URL", path.display()))?; let id = PackageInfo::package_id_from_manifest(container.manifest()) @@ -52,3 +37,38 @@ impl Source for FileSystemSource { Ok(vec![summary]) } } + +#[async_trait::async_trait] +impl Source for FileSystemSource { + #[tracing::instrument(level = "debug", skip_all, fields(%package))] + async fn query(&self, package: &PackageSource) -> Result, QueryError> { + let path = match package { + PackageSource::Path(path) => { + let path = std::path::PathBuf::from(path); + path.canonicalize() + .with_context(|| { + format!( + "Unable to get the canonical form for \"{}\"", + path.display() + ) + }) + .map_err(|error| QueryError::Other { + query: package.clone(), + error, + })? + } + _ => { + return Err(QueryError::Unsupported { + query: package.clone(), + }) + } + }; + + Self::load_path(&path) + .await + .map_err(|error| QueryError::Other { + query: package.clone(), + error, + }) + } +} diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/wasix/src/runtime/resolver/in_memory_source.rs index 1505cf12099..1e97b0da0c4 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/wasix/src/runtime/resolver/in_memory_source.rs @@ -157,13 +157,16 @@ impl Source for InMemorySource { if matches.is_empty() { return Err(QueryError::NoMatches { + query: package.clone(), archived_versions: Vec::new(), }); } Ok(matches) } - None => Err(QueryError::NotFound), + None => Err(QueryError::NotFound { + query: package.clone(), + }), } } PackageSource::Ident(PackageIdent::Hash(hash)) => self @@ -171,9 +174,12 @@ impl Source for InMemorySource { .get(hash) .map(|x| vec![x.clone()]) .ok_or_else(|| QueryError::NoMatches { + query: package.clone(), archived_versions: Vec::new(), }), - PackageSource::Url(_) | PackageSource::Path(_) => Err(QueryError::Unsupported), + PackageSource::Url(_) | PackageSource::Path(_) => Err(QueryError::Unsupported { + query: package.clone(), + }), } } } diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/wasix/src/runtime/resolver/inputs.rs index db0cea3b03f..8b5114b8126 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/wasix/src/runtime/resolver/inputs.rs @@ -2,7 +2,7 @@ use std::{ fmt::{self, Display, Formatter}, fs::File, io::{BufRead, BufReader, Read}, - path::{Path, PathBuf}, + path::Path, }; use anyhow::Error; @@ -290,9 +290,9 @@ impl WebcHash { Ok(Self(hash)) } - pub fn for_file(path: &PathBuf) -> Result { + pub fn for_file(path: &Path) -> Result { // check for a hash at the file location - let mut path_hash = path.clone(); + let mut path_hash = path.to_owned(); path_hash.set_extension("webc.sha256"); if let Ok(mut file) = File::open(&path_hash) { let mut hash = Vec::new(); diff --git a/lib/wasix/src/runtime/resolver/multi_source.rs b/lib/wasix/src/runtime/resolver/multi_source.rs index e876b8225bf..be18b912030 100644 --- a/lib/wasix/src/runtime/resolver/multi_source.rs +++ b/lib/wasix/src/runtime/resolver/multi_source.rs @@ -72,12 +72,12 @@ impl Source for MultiSource { return Ok(summaries); } } - Err(QueryError::Unsupported) + Err(QueryError::Unsupported { .. }) if self.strategy.continue_if_unsupported || self.strategy.merge_results => { continue } - Err(QueryError::NotFound) + Err(QueryError::NotFound { .. }) if self.strategy.continue_if_not_found || self.strategy.merge_results => { continue @@ -96,7 +96,9 @@ impl Source for MultiSource { Ok(output) } else { - Err(QueryError::NotFound) + Err(QueryError::NotFound { + query: package.clone(), + }) } } } diff --git a/lib/wasix/src/runtime/resolver/source.rs b/lib/wasix/src/runtime/resolver/source.rs index a3bb5b8d74a..ac7158ae866 100644 --- a/lib/wasix/src/runtime/resolver/source.rs +++ b/lib/wasix/src/runtime/resolver/source.rs @@ -34,9 +34,13 @@ pub trait Source: Sync + Debug { left_version.cmp(&right_version) }) .ok_or(QueryError::NoMatches { + query: pkg.clone(), archived_versions: Vec::new(), }), - _ => candidates.into_iter().next().ok_or(QueryError::NotFound), + _ => candidates + .into_iter() + .next() + .ok_or_else(|| QueryError::NotFound { query: pkg.clone() }), } } } @@ -54,44 +58,65 @@ where #[derive(Debug)] pub enum QueryError { - Unsupported, - NotFound, + Unsupported { + query: PackageSource, + }, + NotFound { + query: PackageSource, + }, NoMatches { + query: PackageSource, archived_versions: Vec, }, - Timeout, - Other(anyhow::Error), + Timeout { + query: PackageSource, + }, + Other { + query: PackageSource, + error: anyhow::Error, + }, } -impl From for QueryError { - fn from(value: anyhow::Error) -> Self { - Self::Other(value) +impl QueryError { + pub fn query(&self) -> &PackageSource { + match self { + Self::Unsupported { query } + | Self::NotFound { query } + | Self::NoMatches { query, .. } + | Self::Timeout { query } + | Self::Other { query, .. } => query, + } } } impl Display for QueryError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "failed to query package '{}': ", self.query())?; + match self { - Self::Unsupported => f.write_str("This type of package specifier isn't supported"), - Self::NotFound => f.write_str("Not found"), - Self::Timeout => f.write_str("Timed out"), - Self::NoMatches { archived_versions } => match archived_versions.as_slice() { + Self::Unsupported { .. } => f.write_str("unsupported package specifier"), + Self::NotFound { .. } => f.write_str("not found"), + Self::Timeout { .. } => f.write_str("timeout"), + Self::NoMatches { + query: _, + archived_versions, + } => match archived_versions.as_slice() { [] => f.write_str( - "The package was found, but no published versions matched the constraint", + "the package was found, but no published versions matched the constraint", ), [version] => write!( f, - "The only version satisfying the constraint, {version}, is archived" + "the only version satisfying the constraint, {version}, is archived" ), [first, rest @ ..] => { let num_others = rest.len(); write!( f, - "Unable to satisfy the request. Version {first}, and {num_others} are all archived" + "unable to satisfy the request - version {first}, and {num_others} are all archived" ) } }, - Self::Other(e) => Display::fmt(e, f), + Self::Other { error: e, query: _ } => Display::fmt(e, f), } } } @@ -99,8 +124,11 @@ impl Display for QueryError { impl std::error::Error for QueryError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { - Self::Other(e) => Some(&**e), - Self::Unsupported | Self::NotFound | Self::NoMatches { .. } | Self::Timeout => None, + Self::Other { error, query: _ } => Some(&**error), + Self::Unsupported { .. } + | Self::NotFound { .. } + | Self::NoMatches { .. } + | Self::Timeout { .. } => None, } } } diff --git a/lib/wasix/src/runtime/resolver/web_source.rs b/lib/wasix/src/runtime/resolver/web_source.rs index 0660e752d90..f92a6c1354c 100644 --- a/lib/wasix/src/runtime/resolver/web_source.rs +++ b/lib/wasix/src/runtime/resolver/web_source.rs @@ -224,17 +224,8 @@ impl WebSource { Ok((body, etag)) } -} - -#[async_trait::async_trait] -impl Source for WebSource { - #[tracing::instrument(level = "debug", skip_all, fields(%package))] - async fn query(&self, package: &PackageSource) -> Result, QueryError> { - let url = match package { - PackageSource::Url(url) => url, - _ => return Err(QueryError::Unsupported), - }; + async fn load_url(&self, url: &Url) -> Result, anyhow::Error> { let local_path = self .get_locally_cached_file(url) .await @@ -263,6 +254,26 @@ impl Source for WebSource { } } +#[async_trait::async_trait] +impl Source for WebSource { + #[tracing::instrument(level = "debug", skip_all, fields(%package))] + async fn query(&self, package: &PackageSource) -> Result, QueryError> { + let url = match package { + PackageSource::Url(url) => url, + _ => { + return Err(QueryError::Unsupported { + query: package.clone(), + }) + } + }; + + self.load_url(url).await.map_err(|error| QueryError::Other { + query: package.clone(), + error, + }) + } +} + fn sha256(bytes: &[u8]) -> String { let mut hasher = Sha256::default(); hasher.update(bytes); From 23806b40feb51cdeefda8d7be5dd734a7855910e Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 26 Sep 2024 12:40:41 +0200 Subject: [PATCH 44/53] docs: Add comment on MultiSource error behaviour --- lib/wasix/src/runtime/resolver/multi_source.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/wasix/src/runtime/resolver/multi_source.rs b/lib/wasix/src/runtime/resolver/multi_source.rs index be18b912030..ed450f5d24b 100644 --- a/lib/wasix/src/runtime/resolver/multi_source.rs +++ b/lib/wasix/src/runtime/resolver/multi_source.rs @@ -87,6 +87,9 @@ impl Source for MultiSource { { continue } + // Generic errors do not respect the `merge_results` strategy + // flag, because unexpected errors should be bubbled to the + // caller. Err(e) => return Err(e), } } From 9ebdaff863e230d80ccf8a9d91c4ed63002e32da Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Mon, 30 Sep 2024 17:09:49 +0200 Subject: [PATCH 45/53] Make QueryError cloneable --- .../src/runtime/resolver/backend_source.rs | 10 ++-------- .../src/runtime/resolver/filesystem_source.rs | 10 ++-------- lib/wasix/src/runtime/resolver/source.rs | 20 +++++++++++++++---- lib/wasix/src/runtime/resolver/web_source.rs | 7 +++---- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/lib/wasix/src/runtime/resolver/backend_source.rs b/lib/wasix/src/runtime/resolver/backend_source.rs index 22590e39935..1d943ffc347 100644 --- a/lib/wasix/src/runtime/resolver/backend_source.rs +++ b/lib/wasix/src/runtime/resolver/backend_source.rs @@ -250,10 +250,7 @@ impl Source for BackendSource { }); } Err(error) => { - return Err(QueryError::Other { - query: package.clone(), - error, - }); + return Err(QueryError::new_other(error, package)); } } } @@ -291,10 +288,7 @@ impl Source for BackendSource { let response = self .query_graphql_named(&package_name) .await - .map_err(|error| QueryError::Other { - query: package.clone(), - error, - })?; + .map_err(|error| QueryError::new_other(error, package))?; if let Some(cache) = &self.cache { if let Err(e) = cache.update(&package_name, &response) { diff --git a/lib/wasix/src/runtime/resolver/filesystem_source.rs b/lib/wasix/src/runtime/resolver/filesystem_source.rs index 59b7a415f0b..1927f8af02a 100644 --- a/lib/wasix/src/runtime/resolver/filesystem_source.rs +++ b/lib/wasix/src/runtime/resolver/filesystem_source.rs @@ -52,10 +52,7 @@ impl Source for FileSystemSource { path.display() ) }) - .map_err(|error| QueryError::Other { - query: package.clone(), - error, - })? + .map_err(|error| QueryError::new_other(error, package))? } _ => { return Err(QueryError::Unsupported { @@ -66,9 +63,6 @@ impl Source for FileSystemSource { Self::load_path(&path) .await - .map_err(|error| QueryError::Other { - query: package.clone(), - error, - }) + .map_err(|error| QueryError::new_other(error, package)) } } diff --git a/lib/wasix/src/runtime/resolver/source.rs b/lib/wasix/src/runtime/resolver/source.rs index ac7158ae866..1992750f1ca 100644 --- a/lib/wasix/src/runtime/resolver/source.rs +++ b/lib/wasix/src/runtime/resolver/source.rs @@ -1,4 +1,7 @@ -use std::fmt::{Debug, Display}; +use std::{ + fmt::{Debug, Display}, + sync::Arc, +}; use wasmer_config::package::{PackageIdent, PackageSource}; @@ -56,7 +59,7 @@ where } } -#[derive(Debug)] +#[derive(Clone, Debug)] pub enum QueryError { Unsupported { query: PackageSource, @@ -73,7 +76,9 @@ pub enum QueryError { }, Other { query: PackageSource, - error: anyhow::Error, + // Arc to make it cloneable + // Cloning is important for some use-cases. + error: Arc, }, } @@ -87,6 +92,13 @@ impl QueryError { | Self::Other { query, .. } => query, } } + + pub fn new_other(err: anyhow::Error, query: &PackageSource) -> Self { + Self::Other { + query: query.clone(), + error: Arc::new(err), + } + } } impl Display for QueryError { @@ -124,7 +136,7 @@ impl Display for QueryError { impl std::error::Error for QueryError { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { match self { - Self::Other { error, query: _ } => Some(&**error), + Self::Other { error, query: _ } => Some(&***error), Self::Unsupported { .. } | Self::NotFound { .. } | Self::NoMatches { .. } diff --git a/lib/wasix/src/runtime/resolver/web_source.rs b/lib/wasix/src/runtime/resolver/web_source.rs index f92a6c1354c..75371bef1e4 100644 --- a/lib/wasix/src/runtime/resolver/web_source.rs +++ b/lib/wasix/src/runtime/resolver/web_source.rs @@ -267,10 +267,9 @@ impl Source for WebSource { } }; - self.load_url(url).await.map_err(|error| QueryError::Other { - query: package.clone(), - error, - }) + self.load_url(url) + .await + .map_err(|error| QueryError::new_other(error, package)) } } From f3e39d1fe3cf8fd3a4f951a4adab8e4747ccc36b Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Mon, 30 Sep 2024 17:11:01 +0200 Subject: [PATCH 46/53] chore: Fix lint --- tests/compilers/serialize.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compilers/serialize.rs b/tests/compilers/serialize.rs index fce0451402a..bc60a2b9376 100644 --- a/tests/compilers/serialize.rs +++ b/tests/compilers/serialize.rs @@ -62,7 +62,7 @@ fn test_deserialize(config: crate::Config) -> Result<()> { ); let f0 = Function::new(&mut store, &func_type, |params| { let param_0: i64 = params[0].unwrap_i32() as i64; - let param_1: i64 = params[1].unwrap_i64() as i64; + let param_1: i64 = params[1].unwrap_i64(); let param_2: i64 = params[2].unwrap_i32() as i64; let param_3: i64 = params[3].unwrap_f32() as i64; let param_4: i64 = params[4].unwrap_f64() as i64; From e4244c1f77e3ae674b4ced603dfb7c4d335a325b Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Tue, 1 Oct 2024 13:57:23 +0200 Subject: [PATCH 47/53] Various fixes to get wasmer-js compiling --- lib/backend-api/src/client.rs | 9 ++++++++- lib/virtual-net/src/client.rs | 1 + lib/virtual-net/src/rx_tx.rs | 1 + lib/virtual-net/src/server.rs | 1 + 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/backend-api/src/client.rs b/lib/backend-api/src/client.rs index d3579939f72..ac2cd13fa52 100644 --- a/lib/backend-api/src/client.rs +++ b/lib/backend-api/src/client.rs @@ -5,6 +5,10 @@ use crate::GraphQLApiFailure; use anyhow::{bail, Context as _}; use cynic::{http::CynicReqwestError, GraphQlResponse, Operation}; use url::Url; +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] +use reqwest::Proxy; +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] +struct Proxy; /// API client for the Wasmer API. /// @@ -82,7 +86,7 @@ impl WasmerClient { pub fn new_with_proxy( graphql_endpoint: Url, user_agent: &str, - proxy: Option, + proxy: Option, ) -> Result { let builder = { #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] @@ -93,11 +97,14 @@ impl WasmerClient { .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(90)); + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] if let Some(proxy) = proxy { builder.proxy(proxy) } else { builder } + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + builder }; let client = builder.build().context("failed to create reqwest client")?; diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 5cb31ede648..773c86749f1 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -23,6 +23,7 @@ use futures_util::stream::FuturesOrdered; use futures_util::Sink; use futures_util::Stream; use futures_util::StreamExt; +#[cfg(feature = "hyper")] use hyper_util::rt::tokio::TokioIo; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; diff --git a/lib/virtual-net/src/rx_tx.rs b/lib/virtual-net/src/rx_tx.rs index 6d8391b37b2..17fbf21a431 100644 --- a/lib/virtual-net/src/rx_tx.rs +++ b/lib/virtual-net/src/rx_tx.rs @@ -7,6 +7,7 @@ use std::{ use crate::Result; use futures_util::{future::BoxFuture, Future, Sink, SinkExt, Stream}; +#[cfg(feature = "hyper")] use hyper_util::rt::tokio::TokioIo; use serde::Serialize; #[cfg(feature = "tokio-tungstenite")] diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index 73859d428e4..2f0bcc44471 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -17,6 +17,7 @@ use std::net::IpAddr; use std::task::Waker; use std::time::Duration; +#[cfg(feature = "hyper")] use hyper_util::rt::tokio::TokioIo; use std::{ collections::HashMap, From 3a1a28442c112c8304dc806a04a9924ca424ab42 Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Tue, 1 Oct 2024 12:04:40 -0700 Subject: [PATCH 48/53] Update lib/backend-api/src/client.rs --- lib/backend-api/src/client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backend-api/src/client.rs b/lib/backend-api/src/client.rs index ac2cd13fa52..2abfc8ad176 100644 --- a/lib/backend-api/src/client.rs +++ b/lib/backend-api/src/client.rs @@ -4,9 +4,9 @@ use std::time::Duration; use crate::GraphQLApiFailure; use anyhow::{bail, Context as _}; use cynic::{http::CynicReqwestError, GraphQlResponse, Operation}; -use url::Url; #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] use reqwest::Proxy; +use url::Url; #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] struct Proxy; From 2b66a3a735bbc383be24f3cd0c5a0c86779a35ff Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Wed, 2 Oct 2024 20:15:17 +0330 Subject: [PATCH 49/53] add support for exec-name and transfer args and envs after execve --- Cargo.lock | 4 +- Cargo.toml | 2 +- lib/wasix/src/bin_factory/mod.rs | 61 ++++++++++++++++++- .../src/os/command/builtins/cmd_wasmer.rs | 4 +- lib/wasix/src/state/builder.rs | 2 +- lib/wasix/src/state/env.rs | 2 +- lib/wasix/src/state/mod.rs | 4 +- lib/wasix/src/syscalls/mod.rs | 2 +- lib/wasix/src/syscalls/wasi/args_get.rs | 4 ++ lib/wasix/src/syscalls/wasi/args_sizes_get.rs | 10 ++- lib/wasix/src/syscalls/wasix/proc_spawn.rs | 2 +- 11 files changed, 83 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f49dec42510..275ca45fd7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7377,9 +7377,9 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c48441419be082f8d2537c84d8b1f502624d77bc08fbbd09ab17cadfe7f0ac53" +checksum = "cdea84cf234555864ca9b7a5084c1a99dbdf2d148035f62a09b19ce5606532c1" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index f106fae036f..d8b8d13b216 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,7 @@ wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.0.1", default-features = false, features = ["package"] } +webc = { version = "6.1.0", default-features = false, features = ["package"] } shared-buffer = "0.1.4" # Third-party crates diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index 0cb8cee8e0d..f4ce8cc290f 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -12,7 +12,7 @@ use exec::spawn_exec_wasm; use virtual_fs::{AsyncReadExt, FileSystem}; use wasmer::FunctionEnvMut; use wasmer_wasix_types::wasi::Errno; -use webc::Container; +use webc::{metadata::annotations::Wasi, Container}; mod binary_package; mod exec; @@ -25,6 +25,7 @@ pub use self::{ }; use crate::{ os::{command::Commands, task::TaskJoinHandle}, + state::conv_env_vars, Runtime, SpawnError, WasiEnv, }; @@ -92,6 +93,64 @@ impl BinFactory { spawn_exec_wasm(&bytes, name.as_str(), env, &self.runtime).await } Executable::BinaryPackage(pkg) => { + // Get the command that is going to be executed + let cmd = if let Some(cmd) = pkg.get_command(name.as_str()) { + cmd + } else if let Some(cmd) = pkg.get_entrypoint_command() { + cmd + } else { + tracing::error!( + command=name, + pkg=%pkg.id, + "Unable to spawn a command because its package has no entrypoint", + ); + env.on_exit(Some(Errno::Noexec.into())).await; + return Err(SpawnError::MissingEntrypoint { + package_id: pkg.id.clone(), + }); + }; + + // Transfter the args, env vars, and exec_name that was specified in wasmer.toml for this command + // when spawning it as part of syscalls such as proc_exec2 + if let Ok(Some(Wasi { + main_args, + env: env_vars, + exec_name, + .. + })) = cmd.metadata().wasi() + { + if let Some(env_vars) = env_vars { + let env_vars = env_vars + .into_iter() + .map(|env_var| { + let (k, v) = env_var.split_once('=').unwrap(); + + (k.to_string(), v.as_bytes().to_vec()) + }) + .collect::>(); + + let env_vars = conv_env_vars(env_vars); + + env.state + .envs + .lock() + .unwrap() + .extend_from_slice(env_vars.as_slice()); + } + + if let Some(args) = main_args { + env.state + .args + .lock() + .unwrap() + .extend_from_slice(args.as_slice()); + } + + if let Some(exec_name) = exec_name { + env.state.args.lock().unwrap()[0] = exec_name; + } + } + spawn_exec(pkg, name.as_str(), store, env, &self.runtime).await } } diff --git a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs index c0a0536907d..5060063c466 100644 --- a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs +++ b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs @@ -69,7 +69,7 @@ impl CmdWasmer { // Set the arguments of the environment by replacing the state let mut state = env.state.fork(); args.insert(0, what.clone()); - state.args = args; + state.args = std::sync::Mutex::new(args); env.state = Arc::new(state); if let Ok(binary) = self.get_package(&what).await { @@ -130,7 +130,7 @@ impl VirtualCommand for CmdWasmer { ) -> Result { // Read the command we want to run let env_inner = env.as_ref().ok_or(SpawnError::UnknownError)?; - let args = env_inner.state.args.clone(); + let args = env_inner.state.args.lock().unwrap().clone(); let mut args = args.iter().map(|s| s.as_str()); let _alias = args.next(); let cmd = args.next(); diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index b35bf444d16..e862ddc5312 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -867,7 +867,7 @@ impl WasiEnvBuilder { fs: wasi_fs, secret: rand::thread_rng().gen::<[u8; 32]>(), inodes, - args: self.args.clone(), + args: std::sync::Mutex::new(self.args.clone()), preopen: self.vfs_preopens.clone(), futexs: Default::default(), clock_offset: Default::default(), diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 452109aa718..c51f53aac80 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -295,7 +295,7 @@ impl WasiEnvInit { clock_offset: std::sync::Mutex::new( self.state.clock_offset.lock().unwrap().clone(), ), - args: self.state.args.clone(), + args: std::sync::Mutex::new(self.state.args.lock().unwrap().clone()), envs: std::sync::Mutex::new(self.state.envs.lock().unwrap().deref().clone()), preopen: self.state.preopen.clone(), }, diff --git a/lib/wasix/src/state/mod.rs b/lib/wasix/src/state/mod.rs index 6d1dce47069..b37dc034c0c 100644 --- a/lib/wasix/src/state/mod.rs +++ b/lib/wasix/src/state/mod.rs @@ -133,7 +133,7 @@ pub(crate) struct WasiState { pub inodes: WasiInodes, pub futexs: Mutex, pub clock_offset: Mutex>, - pub args: Vec, + pub args: Mutex>, pub envs: Mutex>>, // TODO: should not be here, since this requires active work to resolve. @@ -255,7 +255,7 @@ impl WasiState { inodes: self.inodes.clone(), futexs: Default::default(), clock_offset: Mutex::new(self.clock_offset.lock().unwrap().clone()), - args: self.args.clone(), + args: Mutex::new(self.args.lock().unwrap().clone()), envs: Mutex::new(self.envs.lock().unwrap().clone()), preopen: self.preopen.clone(), } diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 87cde9b845a..6289c9b7486 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1495,7 +1495,7 @@ pub(crate) fn _prepare_wasi( // Swap out the arguments with the new ones if let Some(args) = args { let mut wasi_state = wasi_env.state.fork(); - wasi_state.args = args; + *wasi_state.args.lock().unwrap() = args; wasi_env.state = Arc::new(wasi_state); } diff --git a/lib/wasix/src/syscalls/wasi/args_get.rs b/lib/wasix/src/syscalls/wasi/args_get.rs index fe3f22363f9..70066237645 100644 --- a/lib/wasix/src/syscalls/wasi/args_get.rs +++ b/lib/wasix/src/syscalls/wasi/args_get.rs @@ -21,6 +21,8 @@ pub fn args_get( let args = state .args + .lock() + .unwrap() .iter() .map(|a| a.as_bytes().to_vec()) .collect::>(); @@ -30,6 +32,8 @@ pub fn args_get( "args:\n{}", state .args + .lock() + .unwrap() .iter() .enumerate() .map(|(i, v)| format!("{:>20}: {}", i, v)) diff --git a/lib/wasix/src/syscalls/wasi/args_sizes_get.rs b/lib/wasix/src/syscalls/wasi/args_sizes_get.rs index 1740e218fa9..28fef44a7a5 100644 --- a/lib/wasix/src/syscalls/wasi/args_sizes_get.rs +++ b/lib/wasix/src/syscalls/wasi/args_sizes_get.rs @@ -20,8 +20,14 @@ pub fn args_sizes_get( let argc = argc.deref(&memory); let argv_buf_size = argv_buf_size.deref(&memory); - let argc_val: M::Offset = wasi_try!(state.args.len().try_into().map_err(|_| Errno::Overflow)); - let argv_buf_size_val: usize = state.args.iter().map(|v| v.len() + 1).sum(); + let argc_val: M::Offset = wasi_try!(state + .args + .lock() + .unwrap() + .len() + .try_into() + .map_err(|_| Errno::Overflow)); + let argv_buf_size_val: usize = state.args.lock().unwrap().iter().map(|v| v.len() + 1).sum(); let argv_buf_size_val: M::Offset = wasi_try!(argv_buf_size_val.try_into().map_err(|_| Errno::Overflow)); wasi_try_mem!(argc.write(argc_val)); diff --git a/lib/wasix/src/syscalls/wasix/proc_spawn.rs b/lib/wasix/src/syscalls/wasix/proc_spawn.rs index d46d989d046..4d22eec4e7a 100644 --- a/lib/wasix/src/syscalls/wasix/proc_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/proc_spawn.rs @@ -117,7 +117,7 @@ pub fn proc_spawn_internal( let child_process = child_env.process.clone(); if let Some(args) = args { let mut child_state = env.state.fork(); - child_state.args = args; + child_state.args = std::sync::Mutex::new(args); child_env.state = Arc::new(child_state); } From 9dd9a630f4883ab4b7a4b87a5ec9205085614fca Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Thu, 3 Oct 2024 01:14:17 +0330 Subject: [PATCH 50/53] fix running packages using built-in wasmer --- lib/cli/src/commands/run/mod.rs | 21 +------ lib/wasix/src/bin_factory/binary_package.rs | 19 ++++++ lib/wasix/src/bin_factory/mod.rs | 44 +------------ .../src/os/command/builtins/cmd_wasmer.rs | 18 ++++++ lib/wasix/src/runners/wasi.rs | 8 ++- lib/wasix/src/state/env.rs | 61 ++++++++++++++++--- 6 files changed, 101 insertions(+), 70 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index eefc629f72a..4b68a7a1196 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -219,7 +219,7 @@ impl Run { ) -> Result<(), Error> { let id = match self.entrypoint.as_deref() { Some(cmd) => cmd, - None => infer_webc_entrypoint(pkg)?, + None => pkg.infer_entrypoint()?, }; let cmd = pkg .get_command(id) @@ -570,25 +570,6 @@ fn parse_value(s: &str, ty: wasmer_types::Type) -> Result { Ok(value) } -fn infer_webc_entrypoint(pkg: &BinaryPackage) -> Result<&str, Error> { - if let Some(entrypoint) = pkg.entrypoint_cmd.as_deref() { - return Ok(entrypoint); - } - - match pkg.commands.as_slice() { - [] => bail!("The WEBC file doesn't contain any executable commands"), - [one] => Ok(one.name()), - [..] => { - let mut commands: Vec<_> = pkg.commands.iter().map(|cmd| cmd.name()).collect(); - commands.sort(); - bail!( - "Unable to determine the WEBC file's entrypoint. Please choose one of {:?}", - commands, - ); - } - } -} - /// The input that was passed in via the command-line. #[derive(Debug, Clone, PartialEq)] enum PackageSource { diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 4fcf0405411..c2311125bd9 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -224,6 +224,25 @@ impl BinaryPackage { } }) } + + pub fn infer_entrypoint(&self) -> Result<&str, anyhow::Error> { + if let Some(entrypoint) = self.entrypoint_cmd.as_deref() { + return Ok(entrypoint); + } + + match self.commands.as_slice() { + [] => anyhow::bail!("The WEBC file doesn't contain any executable commands"), + [one] => Ok(one.name()), + [..] => { + let mut commands: Vec<_> = self.commands.iter().map(|cmd| cmd.name()).collect(); + commands.sort(); + anyhow::bail!( + "Unable to determine the WEBC file's entrypoint. Please choose one of {:?}", + commands, + ); + } + } + } } #[cfg(test)] diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index f4ce8cc290f..c074676eb44 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -12,7 +12,7 @@ use exec::spawn_exec_wasm; use virtual_fs::{AsyncReadExt, FileSystem}; use wasmer::FunctionEnvMut; use wasmer_wasix_types::wasi::Errno; -use webc::{metadata::annotations::Wasi, Container}; +use webc::Container; mod binary_package; mod exec; @@ -25,7 +25,6 @@ pub use self::{ }; use crate::{ os::{command::Commands, task::TaskJoinHandle}, - state::conv_env_vars, Runtime, SpawnError, WasiEnv, }; @@ -110,46 +109,7 @@ impl BinFactory { }); }; - // Transfter the args, env vars, and exec_name that was specified in wasmer.toml for this command - // when spawning it as part of syscalls such as proc_exec2 - if let Ok(Some(Wasi { - main_args, - env: env_vars, - exec_name, - .. - })) = cmd.metadata().wasi() - { - if let Some(env_vars) = env_vars { - let env_vars = env_vars - .into_iter() - .map(|env_var| { - let (k, v) = env_var.split_once('=').unwrap(); - - (k.to_string(), v.as_bytes().to_vec()) - }) - .collect::>(); - - let env_vars = conv_env_vars(env_vars); - - env.state - .envs - .lock() - .unwrap() - .extend_from_slice(env_vars.as_slice()); - } - - if let Some(args) = main_args { - env.state - .args - .lock() - .unwrap() - .extend_from_slice(args.as_slice()); - } - - if let Some(exec_name) = exec_name { - env.state.args.lock().unwrap()[0] = exec_name; - } - } + env.prepare_spawn(cmd); spawn_exec(pkg, name.as_str(), store, env, &self.runtime).await } diff --git a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs index 5060063c466..82706144c31 100644 --- a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs +++ b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs @@ -73,6 +73,24 @@ impl CmdWasmer { env.state = Arc::new(state); if let Ok(binary) = self.get_package(&what).await { + // Infer the command that is going to be executed + let cmd_name: &str = + binary + .infer_entrypoint() + .map_err(|_| SpawnError::MissingEntrypoint { + package_id: binary.id.clone(), + })?; + + let cmd = binary + .get_command(cmd_name) + .ok_or_else(|| SpawnError::NotFound { + message: format!("{cmd_name} command in package: {}", binary.id), + })?; + + env.prepare_spawn(cmd); + + env.use_package_async(&binary).await.unwrap(); + // Now run the module spawn_exec(binary, name, store, env, &self.runtime).await } else { diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index f775ef87e79..bf4818b5387 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -338,9 +338,15 @@ impl crate::runners::Runner for WasiRunner { .annotation("wasi")? .unwrap_or_else(|| Wasi::new(command_name)); + let exec_name = if let Some(exec_name) = wasi.exec_name.as_ref() { + exec_name + } else { + command_name + }; + #[allow(unused_mut)] let mut env = self - .prepare_webc_env(command_name, &wasi, Some(pkg), Arc::clone(&runtime), None) + .prepare_webc_env(exec_name, &wasi, Some(pkg), Arc::clone(&runtime), None) .context("Unable to prepare the WASI environment")?; #[cfg(feature = "journal")] diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index c51f53aac80..15306311fe2 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -21,11 +21,12 @@ use wasmer_wasix_types::{ wasi::{Errno, ExitCode, Snapshot0Clockid}, wasix::ThreadStartType, }; +use webc::metadata::annotations::Wasi; #[cfg(feature = "journal")] use crate::journal::{DynJournal, JournalEffector, SnapshotTrigger}; use crate::{ - bin_factory::{BinFactory, BinaryPackage}, + bin_factory::{BinFactory, BinaryPackage, BinaryPackageCommand}, capabilities::Capabilities, fs::{WasiFsRoot, WasiInodes}, import_object_for_all_wasi_versions, @@ -42,7 +43,7 @@ use crate::{ use wasmer_types::ModuleHash; pub(crate) use super::handles::*; -use super::WasiState; +use super::{conv_env_vars, WasiState}; /// Various [`TypedFunction`] and [`Global`] handles for an active WASI(X) instance. /// @@ -1079,6 +1080,10 @@ impl WasiEnv { (state, inodes) } + pub fn use_package(&self, pkg: &BinaryPackage) -> Result<(), WasiStateCreationError> { + InlineWaker::block_on(self.use_package_async(pkg)) + } + /// Make all the commands in a [`BinaryPackage`] available to the WASI /// instance. /// @@ -1090,13 +1095,16 @@ impl WasiEnv { /// /// [cmd-atom]: crate::bin_factory::BinaryPackageCommand::atom() /// [pkg-fs]: crate::bin_factory::BinaryPackage::webc_fs - pub fn use_package(&self, pkg: &BinaryPackage) -> Result<(), WasiStateCreationError> { + pub async fn use_package_async( + &self, + pkg: &BinaryPackage, + ) -> Result<(), WasiStateCreationError> { tracing::trace!(package=%pkg.id, "merging package dependency into wasi environment"); let root_fs = &self.state.fs.root_fs; // We first need to merge the filesystem in the package into the // main file system, if it has not been merged already. - if let Err(e) = InlineWaker::block_on(self.state.fs.conditional_union(pkg)) { + if let Err(e) = self.state.fs.conditional_union(pkg).await { tracing::warn!( error = &e as &dyn std::error::Error, "Unable to merge the package's filesystem into the main one", @@ -1146,9 +1154,7 @@ impl WasiEnv { } WasiFsRoot::Backing(fs) => { let mut f = fs.new_open_options().create(true).write(true).open(path)?; - if let Err(e) = - InlineWaker::block_on(f.copy_reference(Box::new(StaticFile::new(atom)))) - { + if let Err(e) = f.copy_reference(Box::new(StaticFile::new(atom))).await { tracing::warn!( error = &e as &dyn std::error::Error, "Unable to copy file reference", @@ -1304,4 +1310,45 @@ impl WasiEnv { Box::pin(async {}) } } + + pub fn prepare_spawn(&self, cmd: &BinaryPackageCommand) { + if let Ok(Some(Wasi { + main_args, + env: env_vars, + exec_name, + .. + })) = cmd.metadata().wasi() + { + if let Some(env_vars) = env_vars { + let env_vars = env_vars + .into_iter() + .map(|env_var| { + let (k, v) = env_var.split_once('=').unwrap(); + + (k.to_string(), v.as_bytes().to_vec()) + }) + .collect::>(); + + let env_vars = conv_env_vars(env_vars); + + self.state + .envs + .lock() + .unwrap() + .extend_from_slice(env_vars.as_slice()); + } + + if let Some(args) = main_args { + self.state + .args + .lock() + .unwrap() + .extend_from_slice(args.as_slice()); + } + + if let Some(exec_name) = exec_name { + self.state.args.lock().unwrap()[0] = exec_name; + } + } + } } From 5bc60872da115a6ec1a242ca072912095c1c76d8 Mon Sep 17 00:00:00 2001 From: manunio Date: Tue, 1 Oct 2024 21:22:42 +0530 Subject: [PATCH 51/53] oss-fuzz: add cifuzz This PR adds [cifuzz](https://google.github.io/oss-fuzz/getting-started/continuous-integration/) action workflow which is a service provided by oss-fuzz where this project already runs, this helps in catching shallow bugs, regression or build breakage by running fuzzers on PR for ~5 minutes. --- .github/workflows/cifuzz.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/cifuzz.yaml diff --git a/.github/workflows/cifuzz.yaml b/.github/workflows/cifuzz.yaml new file mode 100644 index 00000000000..6cf8ce9a8e8 --- /dev/null +++ b/.github/workflows/cifuzz.yaml @@ -0,0 +1,24 @@ +name: CIFuzz +on: [pull_request] +jobs: + Fuzzing: + runs-on: ubuntu-latest + steps: + - name: Build Fuzzers + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'wasmer' + language: rust + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'wasmer' + language: rust + fuzz-seconds: 300 + - name: Upload Crash + uses: actions/upload-artifact@v3 + if: failure() && steps.build.outcome == 'success' + with: + name: artifacts + path: ./out/artifacts From 8fa66ad71e023e6fa0b87a926db477bf690dedad Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Thu, 3 Oct 2024 13:06:02 +0330 Subject: [PATCH 52/53] enable built-in wasmer to run wasm files and webcs --- lib/wasix/src/bin_factory/mod.rs | 4 +- .../src/os/command/builtins/cmd_wasmer.rs | 90 +++++++++++++------ 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index c074676eb44..fa90ffd8d7e 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -8,7 +8,6 @@ use std::{ }; use anyhow::Context; -use exec::spawn_exec_wasm; use virtual_fs::{AsyncReadExt, FileSystem}; use wasmer::FunctionEnvMut; use wasmer_wasix_types::wasi::Errno; @@ -20,7 +19,8 @@ mod exec; pub use self::{ binary_package::*, exec::{ - run_exec, spawn_exec, spawn_exec_module, spawn_load_module, spawn_load_wasm, spawn_union_fs, + run_exec, spawn_exec, spawn_exec_module, spawn_exec_wasm, spawn_load_module, + spawn_load_wasm, spawn_union_fs, }, }; use crate::{ diff --git a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs index 82706144c31..20e4d26a769 100644 --- a/lib/wasix/src/os/command/builtins/cmd_wasmer.rs +++ b/lib/wasix/src/os/command/builtins/cmd_wasmer.rs @@ -1,12 +1,15 @@ -use std::{any::Any, sync::Arc}; +use std::{any::Any, path::PathBuf, sync::Arc}; use crate::{ + bin_factory::spawn_exec_wasm, os::task::{OwnedTaskStatus, TaskJoinHandle}, runtime::task_manager::InlineWaker, SpawnError, }; +use virtual_fs::{AsyncReadExt, FileSystem}; use wasmer::{FunctionEnvMut, Store}; use wasmer_wasix_types::wasi::Errno; +use webc::Container; use crate::{ bin_factory::{spawn_exec, BinaryPackage}, @@ -57,6 +60,11 @@ impl CmdWasmer { what: Option, mut args: Vec, ) -> Result { + pub enum Executable { + Wasm(bytes::Bytes), + BinaryPackage(BinaryPackage), + } + // If the first argument is a '--' then skip it if args.first().map(|a| a.as_str()) == Some("--") { args = args.into_iter().skip(1).collect(); @@ -72,40 +80,66 @@ impl CmdWasmer { state.args = std::sync::Mutex::new(args); env.state = Arc::new(state); - if let Ok(binary) = self.get_package(&what).await { - // Infer the command that is going to be executed - let cmd_name: &str = - binary - .infer_entrypoint() - .map_err(|_| SpawnError::MissingEntrypoint { - package_id: binary.id.clone(), - })?; + let file_path = if what.starts_with('/') { + PathBuf::from(&what) + } else { + // convert relative path to absolute path + let cwd = env.state.fs.current_dir.lock().unwrap().clone(); - let cmd = binary - .get_command(cmd_name) - .ok_or_else(|| SpawnError::NotFound { - message: format!("{cmd_name} command in package: {}", binary.id), - })?; + PathBuf::from(cwd).join(&what) + }; - env.prepare_spawn(cmd); + let fs = env.fs_root(); + let f = fs.new_open_options().read(true).open(&file_path); + let executable = if let Ok(mut file) = f { + let mut data = Vec::with_capacity(file.size() as usize); + file.read_to_end(&mut data).await.unwrap(); - env.use_package_async(&binary).await.unwrap(); + let bytes: bytes::Bytes = data.into(); - // Now run the module - spawn_exec(binary, name, store, env, &self.runtime).await - } else { - let _ = unsafe { - stderr_write( - parent_ctx, - format!("package not found - {}\r\n", what).as_bytes(), - ) + if let Ok(container) = Container::from_bytes(bytes.clone()) { + let pkg = BinaryPackage::from_webc(&container, &*self.runtime) + .await + .unwrap(); + + Executable::BinaryPackage(pkg) + } else { + Executable::Wasm(bytes) } - .await; + } else if let Ok(pkg) = self.get_package(&what).await { + Executable::BinaryPackage(pkg) + } else { + let _ = unsafe { stderr_write(parent_ctx, HELP_RUN.as_bytes()) }.await; + let handle = + OwnedTaskStatus::new_finished_with_code(Errno::Success.into()).handle(); + return Ok(handle); + }; + + match executable { + Executable::BinaryPackage(binary) => { + // Infer the command that is going to be executed + let cmd_name: &str = + binary + .infer_entrypoint() + .map_err(|_| SpawnError::MissingEntrypoint { + package_id: binary.id.clone(), + })?; + + let cmd = binary + .get_command(cmd_name) + .ok_or_else(|| SpawnError::NotFound { + message: format!("{cmd_name} command in package: {}", binary.id), + })?; - let handle = OwnedTaskStatus::new_finished_with_code(Errno::Noent.into()).handle(); - Ok(handle) + env.prepare_spawn(cmd); + + env.use_package_async(&binary).await.unwrap(); + + // Now run the module + spawn_exec(binary, name, store, env, &self.runtime).await + } + Executable::Wasm(bytes) => spawn_exec_wasm(&bytes, name, env, &self.runtime).await, } - // Get the binary } else { let _ = unsafe { stderr_write(parent_ctx, HELP_RUN.as_bytes()) }.await; let handle = OwnedTaskStatus::new_finished_with_code(Errno::Success.into()).handle(); From 3ccff3d9ed9cb0dd843f7ca9afc4195a1a90a6b9 Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Thu, 3 Oct 2024 06:52:34 -0700 Subject: [PATCH 53/53] Apply suggestions from code review --- lib/wasix/src/bin_factory/binary_package.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index c2311125bd9..051b9268c6a 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -231,13 +231,13 @@ impl BinaryPackage { } match self.commands.as_slice() { - [] => anyhow::bail!("The WEBC file doesn't contain any executable commands"), + [] => anyhow::bail!("The package doesn't contain any executable commands"), [one] => Ok(one.name()), [..] => { let mut commands: Vec<_> = self.commands.iter().map(|cmd| cmd.name()).collect(); commands.sort(); anyhow::bail!( - "Unable to determine the WEBC file's entrypoint. Please choose one of {:?}", + "Unable to determine the package's entrypoint. Please choose one of {:?}", commands, ); }