From 588812ca0e450581736c30b1f6ef0e69aafd4a00 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 30 Jun 2023 16:11:49 +1000 Subject: [PATCH 01/52] Added ignore rules on vendored dependencies --- lib/wasi-web/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/wasi-web/.gitignore b/lib/wasi-web/.gitignore index bac5fc96dcd..3ff3a0b1647 100644 --- a/lib/wasi-web/.gitignore +++ b/lib/wasi-web/.gitignore @@ -3,6 +3,8 @@ dist .parcel-cache .cache .vscode +.cargo target pkg wapm/*.wasm +vendor From a0161716490793e0fa3f53ad40de451a452fa9e9 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 1 Jul 2023 14:17:13 +1000 Subject: [PATCH 02/52] Added dependencies needed for VPN functionality --- Cargo.lock | 1114 ++++++++++++++++++++++++++++-------- lib/cli/Cargo.toml | 2 +- lib/virtual-net/Cargo.toml | 9 + lib/wasix/Cargo.toml | 1 + 4 files changed, 873 insertions(+), 253 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5a0d505f5f..980d315885a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" dependencies = [ "gimli 0.27.3", ] @@ -17,6 +17,18 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if 1.0.0", + "cipher 0.3.0", + "cpufeatures", + "opaque-debug", +] + [[package]] name = "ahash" version = "0.7.6" @@ -75,7 +87,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -214,7 +226,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -225,7 +237,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -245,7 +257,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -256,16 +268,16 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backtrace" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide 0.6.2", - "object 0.30.4", + "miniz_oxide", + "object 0.31.1", "rustc-demangle", ] @@ -275,6 +287,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.2" @@ -307,9 +325,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" [[package]] name = "bitvec" @@ -418,6 +436,16 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.4.0" @@ -481,7 +509,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b922faaf31122819ec80c4047cc684c6979a087366c069611e33649bf98e18d" dependencies = [ "heck 0.4.1", - "indexmap", + "indexmap 1.9.3", "log", "proc-macro2", "quote", @@ -524,7 +552,16 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "winapi", + "winapi 0.3.9", +] + +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", ] [[package]] @@ -550,9 +587,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.5" +version = "4.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2686c4115cb0810d9a984776e197823d08ec94f176549a89a9efded477c456dc" +checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a" dependencies = [ "clap_builder", "clap_derive", @@ -565,19 +602,18 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eef05769009513df2eb1c3b4613e7fad873a14c600ff025b08f250f59fee7de" dependencies = [ - "clap 4.3.5", + "clap 4.3.10", "log", ] [[package]] name = "clap_builder" -version = "4.3.5" +version = "4.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e53afce1efce6ed1f633cf0e57612fe51db54a1ee4fd8f8503d078fe02d69ae" +checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d" dependencies = [ "anstream", "anstyle", - "bitflags 1.3.2", "clap_lex", "strsim", ] @@ -591,7 +627,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -600,6 +636,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -614,7 +659,7 @@ checksum = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" dependencies = [ "atty", "lazy_static", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -625,7 +670,7 @@ checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" dependencies = [ "atty", "lazy_static", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -676,14 +721,14 @@ dependencies = [ "lazy_static", "libc", "log", - "miow", + "miow 0.3.7", "regex", "rustfix", "serde", "serde_derive", "serde_json", "tester", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -801,7 +846,7 @@ dependencies = [ "hashbrown 0.12.3", "log", "regalloc2", - "smallvec", + "smallvec 1.10.0", "target-lexicon 0.12.8", ] @@ -829,9 +874,9 @@ dependencies = [ "cranelift-entity", "fxhash", "hashbrown 0.12.3", - "indexmap", + "indexmap 1.9.3", "log", - "smallvec", + "smallvec 1.10.0", ] [[package]] @@ -849,7 +894,7 @@ dependencies = [ "cranelift-codegen", "hashbrown 0.12.3", "log", - "smallvec", + "smallvec 1.10.0", "target-lexicon 0.12.8", ] @@ -926,7 +971,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -936,8 +992,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-epoch 0.9.15", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", ] [[package]] @@ -948,11 +1019,33 @@ checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if 1.0.0", - "crossbeam-utils", + "crossbeam-utils 0.8.16", "memoffset 0.9.0", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -971,11 +1064,11 @@ dependencies = [ "bitflags 1.3.2", "crossterm_winapi", "libc", - "mio", + "mio 0.8.8", "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -984,7 +1077,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1028,6 +1121,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher 0.3.0", +] + [[package]] name = "cty" version = "0.2.2" @@ -1118,7 +1220,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1140,7 +1242,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core 0.20.1", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1151,11 +1253,17 @@ checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if 1.0.0", "hashbrown 0.12.3", - "lock_api", + "lock_api 0.4.10", "once_cell", "parking_lot_core 0.9.8", ] +[[package]] +name = "data-encoding" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" + [[package]] name = "derivative" version = "2.2.0" @@ -1175,7 +1283,7 @@ checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1240,7 +1348,7 @@ checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1251,7 +1359,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1384,9 +1492,15 @@ dependencies = [ "darling 0.20.1", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + [[package]] name = "erased-serde" version = "0.3.25" @@ -1483,7 +1597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", - "miniz_oxide 0.7.1", + "miniz_oxide", ] [[package]] @@ -1531,12 +1645,34 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags 1.3.2", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.28" @@ -1593,7 +1729,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1675,7 +1811,7 @@ checksum = "e77ac7b51b8e6313251737fcef4b1c01a2ea102bde68415b62c0ee9268fec357" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1685,7 +1821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 1.9.3", "stable_deref_trait", ] @@ -1781,19 +1917,19 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", - "tokio", + "tokio 1.29.1", "tokio-util", "tracing", ] @@ -1887,15 +2023,6 @@ dependencies = [ "libc", ] -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.1" @@ -1923,7 +2050,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "itoa", ] @@ -1934,7 +2061,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes", + "bytes 1.4.0", "http", "pin-project-lite", ] @@ -1969,11 +2096,11 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -1985,7 +2112,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio", + "tokio 1.29.1", "tower-service", "tracing", "want", @@ -2000,7 +2127,7 @@ dependencies = [ "http", "hyper", "rustls", - "tokio", + "tokio 1.29.1", "tokio-rustls", ] @@ -2010,10 +2137,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes", + "bytes 1.4.0", "hyper", "native-tls", - "tokio", + "tokio 1.29.1", "tokio-native-tls", ] @@ -2084,6 +2211,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "indicatif" version = "0.17.5" @@ -2161,9 +2298,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.29.0" +version = "1.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a28d25139df397cbca21408bb742cf6837e04cdbebf1b07b760caf971d6a972" +checksum = "28491f7753051e5704d4d0ae7860d45fae3238d7d235bc4289dcd45c48d3cec3" dependencies = [ "console", "lazy_static", @@ -2206,21 +2343,29 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "is-terminal" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" dependencies = [ "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", + "rustix 0.38.1", "windows-sys 0.48.0", ] @@ -2233,7 +2378,7 @@ dependencies = [ "cfg-if 0.1.10", "libc", "redox_syscall 0.1.57", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2280,6 +2425,25 @@ dependencies = [ "serde", ] +[[package]] +name = "keccak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" +dependencies = [ + "cpufeatures", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2316,9 +2480,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libfuzzer-sys" @@ -2373,6 +2537,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + [[package]] name = "llvm-sys" version = "140.1.1" @@ -2386,6 +2556,15 @@ dependencies = [ "semver 1.0.17", ] +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "lock_api" version = "0.4.10" @@ -2461,6 +2640,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md5" version = "0.7.0" @@ -2491,6 +2676,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -2542,7 +2736,7 @@ checksum = "c66a1fdd7e946fe33fe9725012e25836bba3655769bee9ee347cce7de3f396df" dependencies = [ "cc", "dlib", - "futures", + "futures 0.3.28", "instant", "js-sys", "lazy_static", @@ -2556,7 +2750,7 @@ dependencies = [ "wayland-client", "wayland-cursor", "wayland-protocols", - "winapi", + "winapi 0.3.9", "x11-dl", ] @@ -2579,20 +2773,30 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] -name = "miniz_oxide" -version = "0.7.1" +name = "mio" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" dependencies = [ - "adler", + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", ] [[package]] @@ -2607,13 +2811,36 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + [[package]] name = "miow" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2640,6 +2867,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "net2" +version = "0.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + [[package]] name = "nix" version = "0.24.3" @@ -2700,7 +2938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2720,11 +2958,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.1", "libc", ] @@ -2763,7 +3001,7 @@ checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" dependencies = [ "crc32fast", "hashbrown 0.11.2", - "indexmap", + "indexmap 1.9.3", "memchr", ] @@ -2777,6 +3015,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -2789,6 +3036,12 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + [[package]] name = "openssl" version = "0.10.55" @@ -2812,7 +3065,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2851,7 +3104,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2860,6 +3113,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.3", + "rustc_version 0.2.3", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2867,7 +3131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api", + "lock_api 0.4.10", "parking_lot_core 0.8.6", ] @@ -2877,10 +3141,25 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api", + "lock_api 0.4.10", "parking_lot_core 0.9.8", ] +[[package]] +name = "parking_lot_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version 0.2.3", + "smallvec 0.6.14", + "winapi 0.3.9", +] + [[package]] name = "parking_lot_core" version = "0.8.6" @@ -2891,8 +3170,8 @@ dependencies = [ "instant", "libc", "redox_syscall 0.2.16", - "smallvec", - "winapi", + "smallvec 1.10.0", + "winapi 0.3.9", ] [[package]] @@ -2904,8 +3183,8 @@ dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.3.5", - "smallvec", - "windows-targets 0.48.0", + "smallvec 1.10.0", + "windows-targets 0.48.1", ] [[package]] @@ -2972,7 +3251,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2993,27 +3272,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] name = "pin-project" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +checksum = "6e138fdd8263907a2b0e1b4e80b7e58c721126479b6e6eedfb1b402acea7b9bd" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +checksum = "d1fef411b303e3e12d534fb6e7852de82da56edd937d895125821fb7c09436c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3187,9 +3466,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] @@ -3251,9 +3530,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] @@ -3320,8 +3599,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", + "crossbeam-deque 0.8.3", + "crossbeam-utils 0.8.16", "num_cpus", ] @@ -3375,7 +3654,7 @@ dependencies = [ "fxhash", "log", "slice-group-by", - "smallvec", + "smallvec 1.10.0", ] [[package]] @@ -3419,7 +3698,7 @@ dependencies = [ "bitflags 1.3.2", "libc", "mach", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3444,8 +3723,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "async-compression", - "base64", - "bytes", + "base64 0.21.2", + "bytes 1.4.0", "encoding_rs", "futures-core", "futures-util", @@ -3469,7 +3748,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio", + "tokio 1.29.1", "tokio-native-tls", "tokio-rustls", "tokio-util", @@ -3495,7 +3774,7 @@ dependencies = [ "spin 0.5.2", "untrusted", "web-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3507,7 +3786,7 @@ dependencies = [ "bitvec", "bytecheck", "hashbrown 0.12.3", - "indexmap", + "indexmap 1.9.3", "ptr_meta", "rend", "rkyv_derive", @@ -3535,7 +3814,7 @@ checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" dependencies = [ "libc", "rtoolbox", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3545,7 +3824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3559,7 +3838,7 @@ dependencies = [ "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "smallvec", + "smallvec 1.10.0", ] [[package]] @@ -3609,15 +3888,28 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.37.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "62f25693a73057a1b4cb56179dd3c7ea21a7c6c5ee7d85781f5749b46f34b79c" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustix" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc6396159432b5c8490d4e301d8c705f61860b8b6c863bf79942ce5401968f3" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.3", "windows-sys 0.48.0", ] @@ -3635,11 +3927,11 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64", + "base64 0.21.2", ] [[package]] @@ -3700,7 +3992,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -3747,6 +4039,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3921,7 +4219,7 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -3937,9 +4235,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.97" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" dependencies = [ "itoa", "ryu", @@ -3957,9 +4255,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" dependencies = [ "serde", ] @@ -3982,7 +4280,7 @@ version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ - "indexmap", + "indexmap 1.9.3", "ryu", "serde", "yaml-rust", @@ -3990,11 +4288,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.21" +version = "0.9.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" dependencies = [ - "indexmap", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -4032,6 +4330,17 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.0" @@ -4049,6 +4358,16 @@ dependencies = [ "digest", ] +[[package]] +name = "sha3" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" +dependencies = [ + "digest", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -4064,7 +4383,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cf61602ee61e2f83dd016b3e6387245291cf728ea071c378b35088125b4d995" dependencies = [ - "bytes", + "bytes 1.4.0", "memmap2 0.6.2", ] @@ -4100,7 +4419,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio", + "mio 0.8.8", "signal-hook", ] @@ -4140,6 +4459,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -4153,7 +4481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4168,7 +4496,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api", + "lock_api 0.4.10", ] [[package]] @@ -4243,7 +4571,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1", + "sha1 0.6.1", "syn 1.0.109", ] @@ -4300,9 +4628,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" dependencies = [ "proc-macro2", "quote", @@ -4348,7 +4676,7 @@ dependencies = [ "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", - "rustix", + "rustix 0.37.21", "windows-sys 0.48.0", ] @@ -4360,7 +4688,7 @@ checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ "dirs-next", "rustversion", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4370,7 +4698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4455,7 +4783,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -4480,7 +4808,7 @@ dependencies = [ "stdweb", "time-macros 0.1.1", "version_check", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4574,14 +4902,39 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.2" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + +[[package]] +name = "tokio" +version = "1.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", - "bytes", + "backtrace", + "bytes 1.4.0", "libc", - "mio", + "mio 0.8.8", "num_cpus", "pin-project-lite", "signal-hook-registry", @@ -4590,6 +4943,78 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "tokio-io", +] + +[[package]] +name = "tokio-core" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87b1395334443abca552f63d4f61d0486f12377c2ba8b368e523f89e828cffd4" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "log", + "mio 0.6.23", + "scoped-tls 0.1.2", + "tokio 0.1.22", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-timer", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.31", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.31", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -4598,7 +5023,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -4608,7 +5033,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio", + "tokio 1.29.1", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", ] [[package]] @@ -4618,7 +5062,105 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", - "tokio", + "tokio 1.29.1", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "mio 0.6.23", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque 0.7.4", + "crossbeam-queue", + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c" +dependencies = [ + "futures-util", + "log", + "tokio 1.29.1", + "tungstenite", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", + "mio 0.6.23", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "libc", + "log", + "mio 0.6.23", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", ] [[package]] @@ -4627,11 +5169,11 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-core", "futures-sink", "pin-project-lite", - "tokio", + "tokio 1.29.1", "tracing", ] @@ -4646,9 +5188,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" +checksum = "1ebafdf5ad1220cb59e7d17cf4d2c72015297b75b19a10472f99b89225089240" dependencies = [ "serde", "serde_spanned", @@ -4658,20 +5200,20 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.10" +version = "0.19.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" +checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_spanned", "toml_datetime", @@ -4688,7 +5230,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio", + "tokio 1.29.1", "tower-layer", "tower-service", "tracing", @@ -4700,8 +5242,8 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ - "bitflags 2.3.2", - "bytes", + "bitflags 2.3.3", + "bytes 1.4.0", "futures-core", "futures-util", "http", @@ -4747,7 +5289,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -4795,7 +5337,7 @@ dependencies = [ "serde", "serde_json", "sharded-slab", - "smallvec", + "smallvec 1.10.0", "thread_local", "tracing", "tracing-core", @@ -4814,7 +5356,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec", + "smallvec 1.10.0", "thread_local", "tracing", "tracing-core", @@ -4853,6 +5395,38 @@ dependencies = [ "termcolor", ] +[[package]] +name = "tun-tap" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27165dce5c30856f7e11754ff8d3bb6fc965fba6f25f31ecbddc7c23f99e1615" +dependencies = [ + "cc", + "futures 0.1.31", + "libc", + "mio 0.6.23", + "tokio-core", +] + +[[package]] +name = "tungstenite" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15fba1a6d6bb030745759a9a2a588bfe8490fc8b4751a277db3a0be1c9ebbf67" +dependencies = [ + "byteorder", + "bytes 1.4.0", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1 0.10.5", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.16.0" @@ -4982,6 +5556,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8parse" version = "0.2.1" @@ -4990,9 +5570,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.3.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa2982af2eec27de306107c027578ff7f423d65f7250e40ce0fea8f45248b81" +checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" dependencies = [ "getrandom", "serde", @@ -5043,13 +5623,13 @@ version = "0.6.0" dependencies = [ "anyhow", "async-trait", - "bytes", + "bytes 1.4.0", "derivative", "filetime", "fs_extra", - "futures", + "futures 0.3.28", "getrandom", - "indexmap", + "indexmap 1.9.3", "lazy_static", "libc", "pin-project-lite", @@ -5058,7 +5638,7 @@ dependencies = [ "slab", "tempfile", "thiserror", - "tokio", + "tokio 1.29.1", "tracing", "typetag", "webc", @@ -5069,9 +5649,9 @@ name = "virtual-io" version = "0.1.0" dependencies = [ "async-trait", - "bytes", + "bytes 1.4.0", "derivative", - "mio", + "mio 0.8.8", "socket2", "thiserror", "tracing", @@ -5081,15 +5661,23 @@ dependencies = [ name = "virtual-net" version = "0.3.0" dependencies = [ + "aes", "async-trait", - "bytes", + "base64 0.13.1", + "blake3", + "bytes 1.4.0", + "ctr", "derivative", "libc", - "mio", + "mio 0.8.8", + "rand_chacha", + "sha3", "socket2", "thiserror", - "tokio", + "tokio 1.29.1", + "tokio-tungstenite", "tracing", + "tun-tap", "virtual-io", ] @@ -5244,9 +5832,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d816eaf89496a0eefecb79f43463b07205701165c17790fa6f0eab5ae72bc39" dependencies = [ "anyhow", - "base64", + "base64 0.21.2", "flate2", - "indexmap", + "indexmap 1.9.3", "json5", "nuke-dir", "rand", @@ -5255,7 +5843,7 @@ dependencies = [ "serde_json", "sha2", "tar", - "toml 0.7.4", + "toml 0.7.5", "tracing", "url", "validator", @@ -5387,7 +5975,7 @@ checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" dependencies = [ "console_error_panic_hook", "js-sys", - "scoped-tls", + "scoped-tls 1.0.1", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", @@ -5464,7 +6052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7e95fdeed16adeffed44efdc7ccf27d4f57ff2e99de417c75bcee7dee09049b" dependencies = [ "arbitrary", - "indexmap", + "indexmap 1.9.3", "leb128", "wasm-encoder 0.4.1", ] @@ -5487,11 +6075,11 @@ name = "wasmer" version = "4.0.0" dependencies = [ "anyhow", - "bytes", + "bytes 1.4.0", "cfg-if 1.0.0", "derivative", "hashbrown 0.11.2", - "indexmap", + "indexmap 1.9.3", "js-sys", "macro-wasmer-universal-test", "more-asserts", @@ -5516,7 +6104,7 @@ dependencies = [ "wasmparser 0.83.0", "wasmparser 0.95.0", "wat", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5527,7 +6115,7 @@ checksum = "96d3d713e203593b245f700f42f23ec632e6cf8c01be3ed2ce828b6948a261e4" dependencies = [ "anyhow", "cynic", - "futures", + "futures 0.3.28", "reqwest", "serde", "serde_json", @@ -5567,7 +6155,7 @@ dependencies = [ "libc", "paste", "thiserror", - "tokio", + "tokio 1.29.1", "typetag", "virtual-fs", "wasmer", @@ -5625,19 +6213,19 @@ dependencies = [ "assert_cmd 2.0.11", "async-trait", "atty", - "bytes", + "bytes 1.4.0", "bytesize", "cargo_metadata", "cfg-if 1.0.0", "chrono", - "clap 4.3.5", + "clap 4.3.10", "colored 2.0.0", "dialoguer", "dirs", "distance", "flate2", "hex", - "indexmap", + "indexmap 1.9.3", "indicatif", "isatty", "libc", @@ -5660,7 +6248,7 @@ dependencies = [ "tempfile", "thiserror", "tldextract", - "tokio", + "tokio 1.29.1", "toml 0.5.11", "tracing", "tracing-subscriber 0.3.17", @@ -5706,13 +6294,13 @@ dependencies = [ "region", "serde", "serde_bytes", - "smallvec", + "smallvec 1.10.0", "thiserror", "wasmer-object", "wasmer-types", "wasmer-vm", "wasmparser 0.95.0", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5723,7 +6311,7 @@ dependencies = [ "atty", "bytesize", "cfg-if 1.0.0", - "clap 4.3.5", + "clap 4.3.10", "colored 2.0.0", "distance", "fern", @@ -5748,7 +6336,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec", + "smallvec 1.10.0", "target-lexicon 0.12.8", "tracing", "wasmer-compiler", @@ -5770,7 +6358,7 @@ dependencies = [ "regex", "rustc_version 0.4.0", "semver 1.0.17", - "smallvec", + "smallvec 1.10.0", "target-lexicon 0.12.8", "wasmer-compiler", "wasmer-types", @@ -5790,7 +6378,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec", + "smallvec 1.10.0", "target-lexicon 0.12.8", "wasmer-compiler", "wasmer-types", @@ -5803,11 +6391,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "401220ca92419731ffb670b6d6f8442fc32b110f6e9bb5658c4cfa658c48ab32" dependencies = [ "anyhow", - "clap 4.3.5", + "clap 4.3.10", "clap-verbosity-flag", "comfy-table", "dialoguer", - "futures", + "futures 0.3.28", "is-terminal", "log", "once_cell", @@ -5818,8 +6406,8 @@ dependencies = [ "serde_json", "serde_yaml 0.8.26", "time 0.3.22", - "tokio", - "toml 0.7.4", + "tokio 1.29.1", + "toml 0.7.5", "tracing", "tracing-subscriber 0.3.17", "url", @@ -5925,7 +6513,7 @@ dependencies = [ "derivative", "dirs", "flate2", - "futures", + "futures 0.3.28", "hex", "insta", "md5", @@ -5940,7 +6528,7 @@ dependencies = [ "tar", "target-lexicon 0.12.8", "tempfile", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -5980,7 +6568,7 @@ dependencies = [ "futures-util", "graphql_client", "hex", - "indexmap", + "indexmap 1.9.3", "indicatif", "lazy_static", "log", @@ -5998,11 +6586,11 @@ dependencies = [ "thiserror", "time 0.3.22", "tldextract", - "tokio", + "tokio 1.29.1", "toml 0.5.11", "url", "wasmer-toml", - "wasmer-wasm-interface 4.0.0-beta.3", + "wasmer-wasm-interface 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "wasmparser 0.51.4", "webc", "whoami", @@ -6013,7 +6601,7 @@ name = "wasmer-registry" version = "5.2.0" dependencies = [ "anyhow", - "clap 4.3.5", + "clap 4.3.10", "console", "dirs", "filetime", @@ -6021,7 +6609,7 @@ dependencies = [ "futures-util", "graphql_client", "hex", - "indexmap", + "indexmap 1.9.3", "indicatif", "lazy_static", "log", @@ -6040,7 +6628,7 @@ dependencies = [ "thiserror", "time 0.3.22", "tldextract", - "tokio", + "tokio 1.29.1", "toml 0.5.11", "url", "wasmer-toml", @@ -6070,12 +6658,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4232db0aff83ed6208d541ddcf1bf72730673528be8c4fe13c6369060f6e05a7" dependencies = [ "anyhow", - "indexmap", + "indexmap 1.9.3", "semver 1.0.17", "serde", "serde_cbor", "serde_json", - "serde_yaml 0.9.21", + "serde_yaml 0.9.22", "thiserror", "toml 0.5.11", ] @@ -6087,7 +6675,7 @@ dependencies = [ "bytecheck", "enum-iterator", "enumset", - "indexmap", + "indexmap 1.9.3", "memoffset 0.6.5", "more-asserts", "rkyv", @@ -6109,7 +6697,7 @@ dependencies = [ "derivative", "enum-iterator", "fnv", - "indexmap", + "indexmap 1.9.3", "lazy_static", "libc", "mach", @@ -6121,7 +6709,7 @@ dependencies = [ "thiserror", "tracing", "wasmer-types", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6131,13 +6719,13 @@ dependencies = [ "anyhow", "async-trait", "bincode", - "bytes", + "bytes 1.4.0", "cfg-if 1.0.0", "chrono", "cooked-waker", "dashmap", "derivative", - "futures", + "futures 0.3.28", "getrandom", "heapless", "hex", @@ -6165,7 +6753,7 @@ dependencies = [ "term_size", "termios", "thiserror", - "tokio", + "tokio 1.29.1", "tower", "tower-http", "tracing", @@ -6190,7 +6778,7 @@ dependencies = [ "wcgi-host", "webc", "weezl", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6232,26 +6820,26 @@ dependencies = [ [[package]] name = "wasmer-wasm-interface" -version = "4.0.0-beta.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d70bdc140529bd98fec735cc4fc0fde15632fd29350445fdc3413b11c8b41f" +version = "4.0.0" dependencies = [ + "bincode", "either", "nom 5.1.3", "serde", "wasmparser 0.51.4", + "wat", ] [[package]] name = "wasmer-wasm-interface" version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17603c436eb15548721c2eb78001daba77e5193eab8680c1be5fb0f37879658e" dependencies = [ - "bincode", "either", "nom 5.1.3", "serde", "wasmparser 0.51.4", - "wat", ] [[package]] @@ -6259,11 +6847,11 @@ name = "wasmer-wast" version = "4.0.0" dependencies = [ "anyhow", - "futures", + "futures 0.3.28", "serde", "tempfile", "thiserror", - "tokio", + "tokio 1.29.1", "virtual-fs", "wasmer", "wasmer-wasix", @@ -6286,7 +6874,7 @@ dependencies = [ "tempfile", "test-generator", "test-log", - "tokio", + "tokio 1.29.1", "tracing", "tracing-subscriber 0.3.17", "wasi-test-generator", @@ -6321,7 +6909,7 @@ version = "0.95.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ea896273ea99b15132414be1da01ab0d8836415083298ecaffbe308eaac87a" dependencies = [ - "indexmap", + "indexmap 1.9.3", "url", ] @@ -6331,7 +6919,7 @@ version = "0.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29e3ac9b780c7dda0cac7a52a5d6d2d6707cc6e3451c9db209b6c758f40d7acb" dependencies = [ - "indexmap", + "indexmap 1.9.3", "semver 1.0.17", ] @@ -6394,7 +6982,7 @@ dependencies = [ "downcast-rs", "libc", "nix 0.24.3", - "scoped-tls", + "scoped-tls 1.0.1", "wayland-commons", "wayland-scanner", "wayland-sys", @@ -6408,7 +6996,7 @@ checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" dependencies = [ "nix 0.24.3", "once_cell", - "smallvec", + "smallvec 1.10.0", "wayland-sys", ] @@ -6479,7 +7067,7 @@ dependencies = [ "http", "schemars", "serde", - "tokio", + "tokio 1.29.1", "wasmparser 0.95.0", "wcgi", ] @@ -6501,10 +7089,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02ba1b6e7ad252e691a86d727aa422fbc5ed95e9bca4ec8547869e9b5779f8f3" dependencies = [ "anyhow", - "base64", + "base64 0.21.2", "byteorder", - "bytes", - "indexmap", + "bytes 1.4.0", + "indexmap 1.9.3", "leb128", "lexical-sort", "once_cell", @@ -6547,14 +7135,20 @@ checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" [[package]] name = "whoami" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" dependencies = [ "wasm-bindgen", "web-sys", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -6565,6 +7159,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -6577,7 +7177,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6592,7 +7192,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -6638,7 +7238,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets 0.48.1", ] [[package]] @@ -6658,9 +7258,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm 0.48.0", "windows_aarch64_msvc 0.48.0", @@ -6800,7 +7400,17 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi", + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", ] [[package]] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index f4c4233f935..ef62b833faf 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -42,7 +42,7 @@ wasmer-types = { version = "=4.0.0", path = "../types", features = ["enable-serd wasmer-registry = { version = "5.2.0", path = "../registry", features = ["build-package", "clap"] } wasmer-object = { version = "=4.0.0", path = "../object", optional = true } virtual-fs = { version = "0.6.0", path = "../virtual-fs", default-features = false, features = ["host-fs"] } -virtual-net = { version = "0.3.0", path = "../virtual-net" } +virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["tun-tap", "ws-net"] } # Wasmer-owned dependencies. webc = { workspace = true } diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index 1949e3da77a..8e3a0203a15 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -20,6 +20,15 @@ mio = { version = "0.8", optional = true } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } +tun-tap = { version = "0.1.3", optional = true } +tokio-tungstenite = { version = "0.19.0", optional = true } +base64 = "^0.13" +aes = { version = "^0.7", optional = true } +ctr = { version = "^0.8", optional = true } +sha3 = { version = "0.10.6", optional = true } +rand_chacha = { version = "^0.3", optional = true } +blake3 = { version = "1.3.3", optional = true } [features] host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "socket2", "mio" ] +ws-net = [ "tokio-tungstenite", "aes", "ctr", "sha3", "rand_chacha", "blake3" ] \ No newline at end of file diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index c3382cfdccd..f2ef827d805 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -135,6 +135,7 @@ host-vnet = ["virtual-net/host-net"] host-threads = [] host-reqwest = ["reqwest"] host-fs = ["virtual-fs/host-fs"] +ws-net = ["virtual-net/ws-net"] logging = ["tracing/log"] disable-all-logging = ["tracing/release_max_level_off", "tracing/max_level_off"] From 684c1522ea811b125eadcf120cdcf968c190df5e Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 1 Jul 2023 17:37:03 +1000 Subject: [PATCH 03/52] Added placeholders for VPN --- lib/virtual-net/src/client.rs | 1 + lib/virtual-net/src/tun.rs | 0 lib/wasi-web/.gitignore | 1 + 3 files changed, 2 insertions(+) create mode 100644 lib/virtual-net/src/client.rs create mode 100644 lib/virtual-net/src/tun.rs diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/lib/virtual-net/src/client.rs @@ -0,0 +1 @@ + diff --git a/lib/virtual-net/src/tun.rs b/lib/virtual-net/src/tun.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/wasi-web/.gitignore b/lib/wasi-web/.gitignore index bac5fc96dcd..a0142e077e0 100644 --- a/lib/wasi-web/.gitignore +++ b/lib/wasi-web/.gitignore @@ -6,3 +6,4 @@ dist target pkg wapm/*.wasm +vendor From 01ffc81cc1d6f2a9ca136dc1448f8597e3917ff9 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 2 Jul 2023 05:30:11 +1000 Subject: [PATCH 04/52] Added the metadata and client for remote networking --- Cargo.lock | 741 +++--------------- lib/cli/Cargo.toml | 2 +- lib/virtual-io/Cargo.toml | 1 + lib/virtual-io/src/lib.rs | 2 + .../task_manager => virtual-io/src}/waker.rs | 0 lib/virtual-net/Cargo.toml | 15 +- lib/virtual-net/src/client.rs | 695 ++++++++++++++++ lib/virtual-net/src/lib.rs | 24 +- lib/virtual-net/src/meta.rs | 300 +++++++ lib/wasix/src/runtime/task_manager/mod.rs | 3 +- 10 files changed, 1139 insertions(+), 644 deletions(-) rename lib/{wasix/src/runtime/task_manager => virtual-io/src}/waker.rs (100%) create mode 100644 lib/virtual-net/src/meta.rs diff --git a/Cargo.lock b/Cargo.lock index 55e45c1d910..9fdb41a57d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,14 +19,13 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aes" -version = "0.7.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" dependencies = [ "cfg-if 1.0.0", - "cipher 0.3.0", + "cipher", "cpufeatures", - "opaque-debug", ] [[package]] @@ -87,7 +86,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -226,7 +225,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.29.1", + "tokio", ] [[package]] @@ -257,7 +256,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -287,12 +286,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.2" @@ -436,16 +429,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.4.0" @@ -552,16 +535,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "winapi 0.3.9", -] - -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", + "winapi", ] [[package]] @@ -636,15 +610,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "colorchoice" version = "1.0.0" @@ -659,7 +624,7 @@ checksum = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" dependencies = [ "atty", "lazy_static", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -670,7 +635,7 @@ checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" dependencies = [ "atty", "lazy_static", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -721,14 +686,14 @@ dependencies = [ "lazy_static", "libc", "log", - "miow 0.3.7", + "miow", "regex", "rustfix", "serde", "serde_derive", "serde_json", "tester", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -846,7 +811,7 @@ dependencies = [ "hashbrown 0.12.3", "log", "regalloc2", - "smallvec 1.10.0", + "smallvec", "target-lexicon 0.12.8", ] @@ -876,7 +841,7 @@ dependencies = [ "hashbrown 0.12.3", "indexmap 1.9.3", "log", - "smallvec 1.10.0", + "smallvec", ] [[package]] @@ -894,7 +859,7 @@ dependencies = [ "cranelift-codegen", "hashbrown 0.12.3", "log", - "smallvec 1.10.0", + "smallvec", "target-lexicon 0.12.8", ] @@ -971,18 +936,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.16", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] @@ -992,23 +946,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.15", - "crossbeam-utils 0.8.16", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] @@ -1019,33 +958,11 @@ checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if 1.0.0", - "crossbeam-utils 0.8.16", + "crossbeam-utils", "memoffset 0.9.0", "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", -] - [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -1064,11 +981,11 @@ dependencies = [ "bitflags 1.3.2", "crossterm_winapi", "libc", - "mio 0.8.8", + "mio", "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1077,7 +994,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1123,11 +1040,11 @@ dependencies = [ [[package]] name = "ctr" -version = "0.8.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher 0.3.0", + "cipher", ] [[package]] @@ -1253,17 +1170,11 @@ checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if 1.0.0", "hashbrown 0.12.3", - "lock_api 0.4.10", + "lock_api", "once_cell", "parking_lot_core 0.9.8", ] -[[package]] -name = "data-encoding" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" - [[package]] name = "derivative" version = "2.2.0" @@ -1348,7 +1259,7 @@ checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1359,7 +1270,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1645,34 +1556,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags 1.3.2", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.28" @@ -1921,7 +1810,7 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "futures-core", "futures-sink", @@ -1929,7 +1818,7 @@ dependencies = [ "http", "indexmap 1.9.3", "slab", - "tokio 1.29.1", + "tokio", "tokio-util", "tracing", ] @@ -2050,7 +1939,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "itoa", ] @@ -2061,7 +1950,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.4.0", + "bytes", "http", "pin-project-lite", ] @@ -2100,7 +1989,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -2112,7 +2001,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio 1.29.1", + "tokio", "tower-service", "tracing", "want", @@ -2127,7 +2016,7 @@ dependencies = [ "http", "hyper", "rustls", - "tokio 1.29.1", + "tokio", "tokio-rustls", ] @@ -2137,10 +2026,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.4.0", + "bytes", "hyper", "native-tls", - "tokio 1.29.1", + "tokio", "tokio-native-tls", ] @@ -2369,7 +2258,7 @@ dependencies = [ "cfg-if 0.1.10", "libc", "redox_syscall 0.1.57", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2425,16 +2314,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2547,15 +2426,6 @@ dependencies = [ "semver 1.0.17", ] -[[package]] -name = "lock_api" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" -dependencies = [ - "scopeguard", -] - [[package]] name = "lock_api" version = "0.4.10" @@ -2631,12 +2501,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md5" version = "0.7.0" @@ -2667,15 +2531,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.5" @@ -2727,7 +2582,7 @@ checksum = "c66a1fdd7e946fe33fe9725012e25836bba3655769bee9ee347cce7de3f396df" dependencies = [ "cc", "dlib", - "futures 0.3.28", + "futures", "instant", "js-sys", "lazy_static", @@ -2741,7 +2596,7 @@ dependencies = [ "wayland-client", "wayland-cursor", "wayland-protocols", - "winapi 0.3.9", + "winapi", "x11-dl", ] @@ -2771,25 +2626,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.8.8" @@ -2802,36 +2638,13 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - [[package]] name = "miow" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2858,17 +2671,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "net2" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nix" version = "0.24.3" @@ -2929,7 +2731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3027,12 +2829,6 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl" version = "0.10.55" @@ -3095,7 +2891,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3104,17 +2900,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking_lot" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.6.3", - "rustc_version 0.2.3", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -3122,7 +2907,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api 0.4.10", + "lock_api", "parking_lot_core 0.8.6", ] @@ -3132,25 +2917,10 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api 0.4.10", + "lock_api", "parking_lot_core 0.9.8", ] -[[package]] -name = "parking_lot_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "rustc_version 0.2.3", - "smallvec 0.6.14", - "winapi 0.3.9", -] - [[package]] name = "parking_lot_core" version = "0.8.6" @@ -3161,8 +2931,8 @@ dependencies = [ "instant", "libc", "redox_syscall 0.2.16", - "smallvec 1.10.0", - "winapi 0.3.9", + "smallvec", + "winapi", ] [[package]] @@ -3590,8 +3360,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", - "crossbeam-deque 0.8.3", - "crossbeam-utils 0.8.16", + "crossbeam-deque", + "crossbeam-utils", "num_cpus", ] @@ -3645,7 +3415,7 @@ dependencies = [ "fxhash", "log", "slice-group-by", - "smallvec 1.10.0", + "smallvec", ] [[package]] @@ -3689,7 +3459,7 @@ dependencies = [ "bitflags 1.3.2", "libc", "mach", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3714,8 +3484,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "async-compression", - "base64 0.21.2", - "bytes 1.4.0", + "base64", + "bytes", "encoding_rs", "futures-core", "futures-util", @@ -3739,7 +3509,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio 1.29.1", + "tokio", "tokio-native-tls", "tokio-rustls", "tokio-util", @@ -3765,7 +3535,7 @@ dependencies = [ "spin 0.5.2", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3805,7 +3575,7 @@ checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" dependencies = [ "libc", "rtoolbox", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3815,7 +3585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3829,7 +3599,7 @@ dependencies = [ "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "smallvec 1.10.0", + "smallvec", ] [[package]] @@ -3922,7 +3692,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64", ] [[package]] @@ -3983,7 +3753,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher 0.4.4", + "cipher", ] [[package]] @@ -4030,12 +3800,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "scoped-tls" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" - [[package]] name = "scoped-tls" version = "1.0.1" @@ -4321,17 +4085,6 @@ dependencies = [ "sha1_smol", ] -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - [[package]] name = "sha1_smol" version = "1.0.0" @@ -4374,7 +4127,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cf61602ee61e2f83dd016b3e6387245291cf728ea071c378b35088125b4d995" dependencies = [ - "bytes 1.4.0", + "bytes", "memmap2 0.6.2", ] @@ -4410,7 +4163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio 0.8.8", + "mio", "signal-hook", ] @@ -4450,15 +4203,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.10.0" @@ -4472,7 +4216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4487,7 +4231,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api 0.4.10", + "lock_api", ] [[package]] @@ -4562,7 +4306,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1 0.6.1", + "sha1", "syn 1.0.109", ] @@ -4679,7 +4423,7 @@ checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ "dirs-next", "rustversion", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4689,7 +4433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4799,7 +4543,7 @@ dependencies = [ "stdweb", "time-macros 0.1.1", "version_check", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4901,7 +4645,7 @@ dependencies = [ "backtrace", "bytes", "libc", - "mio 0.8.8", + "mio", "num_cpus", "pin-project-lite", "signal-hook-registry", @@ -4910,78 +4654,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tokio-codec" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "tokio-io", -] - -[[package]] -name = "tokio-core" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87b1395334443abca552f63d4f61d0486f12377c2ba8b368e523f89e828cffd4" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "log", - "mio 0.6.23", - "scoped-tls 0.1.2", - "tokio 0.1.22", - "tokio-executor", - "tokio-io", - "tokio-reactor", - "tokio-timer", -] - -[[package]] -name = "tokio-current-thread" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" -dependencies = [ - "futures 0.1.31", - "tokio-executor", -] - -[[package]] -name = "tokio-executor" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", -] - -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.31", - "tokio-io", - "tokio-threadpool", -] - -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", -] - [[package]] name = "tokio-macros" version = "2.1.0" @@ -5000,26 +4672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.29.1", -] - -[[package]] -name = "tokio-reactor" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "mio 0.6.23", - "num_cpus", - "parking_lot 0.9.0", - "slab", - "tokio-executor", - "tokio-io", - "tokio-sync", + "tokio", ] [[package]] @@ -5029,105 +4682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", - "tokio 1.29.1", -] - -[[package]] -name = "tokio-sync" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" -dependencies = [ - "fnv", - "futures 0.1.31", -] - -[[package]] -name = "tokio-tcp" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "mio 0.6.23", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-threadpool" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" -dependencies = [ - "crossbeam-deque 0.7.4", - "crossbeam-queue", - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "num_cpus", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-timer" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c" -dependencies = [ - "futures-util", - "log", - "tokio 1.29.1", - "tungstenite", -] - -[[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", - "mio 0.6.23", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-uds" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "libc", - "log", - "mio 0.6.23", - "mio-uds", - "tokio-codec", - "tokio-io", - "tokio-reactor", + "tokio", ] [[package]] @@ -5136,11 +4691,11 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.29.1", + "tokio", "tracing", ] @@ -5197,7 +4752,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.29.1", + "tokio", "tower-layer", "tower-service", "tracing", @@ -5304,7 +4859,7 @@ dependencies = [ "serde", "serde_json", "sharded-slab", - "smallvec 1.10.0", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -5323,7 +4878,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec 1.10.0", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -5362,38 +4917,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "tun-tap" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27165dce5c30856f7e11754ff8d3bb6fc965fba6f25f31ecbddc7c23f99e1615" -dependencies = [ - "cc", - "futures 0.1.31", - "libc", - "mio 0.6.23", - "tokio-core", -] - -[[package]] -name = "tungstenite" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15fba1a6d6bb030745759a9a2a588bfe8490fc8b4751a277db3a0be1c9ebbf67" -dependencies = [ - "byteorder", - "bytes 1.4.0", - "data-encoding", - "http", - "httparse", - "log", - "rand", - "sha1 0.10.5", - "thiserror", - "url", - "utf-8", -] - [[package]] name = "typenum" version = "1.16.0" @@ -5523,12 +5046,6 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8parse" version = "0.2.1" @@ -5590,11 +5107,11 @@ version = "0.6.0" dependencies = [ "anyhow", "async-trait", - "bytes 1.4.0", + "bytes", "derivative", "filetime", "fs_extra", - "futures 0.3.28", + "futures", "getrandom", "indexmap 1.9.3", "lazy_static", @@ -5605,7 +5122,7 @@ dependencies = [ "slab", "tempfile", "thiserror", - "tokio 1.29.1", + "tokio", "tracing", "typetag", "webc", @@ -5616,9 +5133,10 @@ name = "virtual-io" version = "0.1.0" dependencies = [ "async-trait", - "bytes 1.4.0", + "bytes", "derivative", - "mio 0.8.8", + "futures", + "mio", "socket2", "thiserror", "tracing", @@ -5630,21 +5148,20 @@ version = "0.3.0" dependencies = [ "aes", "async-trait", - "base64 0.13.1", + "base64", "blake3", - "bytes 1.4.0", + "bytes", "ctr", "derivative", "libc", - "mio 0.8.8", + "mio", "rand_chacha", + "serde", "sha3", "socket2", "thiserror", - "tokio 1.29.1", - "tokio-tungstenite", + "tokio", "tracing", - "tun-tap", "virtual-io", ] @@ -5799,7 +5316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d816eaf89496a0eefecb79f43463b07205701165c17790fa6f0eab5ae72bc39" dependencies = [ "anyhow", - "base64 0.21.2", + "base64", "flate2", "indexmap 1.9.3", "json5", @@ -5942,7 +5459,7 @@ checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" dependencies = [ "console_error_panic_hook", "js-sys", - "scoped-tls 1.0.1", + "scoped-tls", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", @@ -6042,7 +5559,7 @@ name = "wasmer" version = "4.0.0" dependencies = [ "anyhow", - "bytes 1.4.0", + "bytes", "cfg-if 1.0.0", "derivative", "hashbrown 0.11.2", @@ -6071,7 +5588,7 @@ dependencies = [ "wasmparser 0.83.0", "wasmparser 0.95.0", "wat", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6082,7 +5599,7 @@ checksum = "96d3d713e203593b245f700f42f23ec632e6cf8c01be3ed2ce828b6948a261e4" dependencies = [ "anyhow", "cynic", - "futures 0.3.28", + "futures", "reqwest", "serde", "serde_json", @@ -6122,7 +5639,7 @@ dependencies = [ "libc", "paste", "thiserror", - "tokio 1.29.1", + "tokio", "typetag", "virtual-fs", "wasmer", @@ -6180,7 +5697,7 @@ dependencies = [ "assert_cmd 2.0.11", "async-trait", "atty", - "bytes 1.4.0", + "bytes", "bytesize", "cargo_metadata", "cfg-if 1.0.0", @@ -6215,7 +5732,7 @@ dependencies = [ "tempfile", "thiserror", "tldextract", - "tokio 1.29.1", + "tokio", "toml 0.5.11", "tracing", "tracing-subscriber 0.3.17", @@ -6261,13 +5778,13 @@ dependencies = [ "region", "serde", "serde_bytes", - "smallvec 1.10.0", + "smallvec", "thiserror", "wasmer-object", "wasmer-types", "wasmer-vm", "wasmparser 0.95.0", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6303,7 +5820,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec 1.10.0", + "smallvec", "target-lexicon 0.12.8", "tracing", "wasmer-compiler", @@ -6325,7 +5842,7 @@ dependencies = [ "regex", "rustc_version 0.4.0", "semver 1.0.17", - "smallvec 1.10.0", + "smallvec", "target-lexicon 0.12.8", "wasmer-compiler", "wasmer-types", @@ -6345,7 +5862,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec 1.10.0", + "smallvec", "target-lexicon 0.12.8", "wasmer-compiler", "wasmer-types", @@ -6362,7 +5879,7 @@ dependencies = [ "clap-verbosity-flag", "comfy-table", "dialoguer", - "futures 0.3.28", + "futures", "is-terminal", "log", "once_cell", @@ -6480,7 +5997,7 @@ dependencies = [ "derivative", "dirs", "flate2", - "futures 0.3.28", + "futures", "hex", "insta", "md5", @@ -6495,7 +6012,7 @@ dependencies = [ "tar", "target-lexicon 0.12.8", "tempfile", - "tokio 1.29.1", + "tokio", ] [[package]] @@ -6553,7 +6070,7 @@ dependencies = [ "thiserror", "time 0.3.22", "tldextract", - "tokio 1.29.1", + "tokio", "toml 0.5.11", "url", "wasmer-toml", @@ -6595,7 +6112,7 @@ dependencies = [ "thiserror", "time 0.3.22", "tldextract", - "tokio 1.29.1", + "tokio", "toml 0.5.11", "url", "wasmer-toml", @@ -6676,7 +6193,7 @@ dependencies = [ "thiserror", "tracing", "wasmer-types", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6686,13 +6203,13 @@ dependencies = [ "anyhow", "async-trait", "bincode", - "bytes 1.4.0", + "bytes", "cfg-if 1.0.0", "chrono", "cooked-waker", "dashmap", "derivative", - "futures 0.3.28", + "futures", "getrandom", "heapless", "hex", @@ -6720,7 +6237,7 @@ dependencies = [ "term_size", "termios", "thiserror", - "tokio 1.29.1", + "tokio", "tower", "tower-http", "tracing", @@ -6745,7 +6262,7 @@ dependencies = [ "wcgi-host", "webc", "weezl", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6814,11 +6331,11 @@ name = "wasmer-wast" version = "4.0.0" dependencies = [ "anyhow", - "futures 0.3.28", + "futures", "serde", "tempfile", "thiserror", - "tokio 1.29.1", + "tokio", "virtual-fs", "wasmer", "wasmer-wasix", @@ -6841,7 +6358,7 @@ dependencies = [ "tempfile", "test-generator", "test-log", - "tokio 1.29.1", + "tokio", "tracing", "tracing-subscriber 0.3.17", "wasi-test-generator", @@ -6949,7 +6466,7 @@ dependencies = [ "downcast-rs", "libc", "nix 0.24.3", - "scoped-tls 1.0.1", + "scoped-tls", "wayland-commons", "wayland-scanner", "wayland-sys", @@ -6963,7 +6480,7 @@ checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" dependencies = [ "nix 0.24.3", "once_cell", - "smallvec 1.10.0", + "smallvec", "wayland-sys", ] @@ -7034,7 +6551,7 @@ dependencies = [ "http", "schemars", "serde", - "tokio 1.29.1", + "tokio", "wasmparser 0.95.0", "wcgi", ] @@ -7056,7 +6573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02ba1b6e7ad252e691a86d727aa422fbc5ed95e9bca4ec8547869e9b5779f8f3" dependencies = [ "anyhow", - "base64 0.21.2", + "base64", "byteorder", "bytes", "indexmap 1.9.3", @@ -7110,12 +6627,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -7126,12 +6637,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -7144,7 +6649,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -7367,17 +6872,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 4e8ceede85b..0fcfdb00111 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -42,7 +42,7 @@ wasmer-types = { version = "=4.0.0", path = "../types", features = ["enable-serd wasmer-registry = { version = "5.2.0", path = "../registry", features = ["build-package", "clap"] } wasmer-object = { version = "=4.0.0", path = "../object", optional = true } virtual-fs = { version = "0.6.0", path = "../virtual-fs", default-features = false, features = ["host-fs"] } -virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["tun-tap", "ws-net"] } +virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["ws-net"] } # Wasmer-owned dependencies. webc = { workspace = true } diff --git a/lib/virtual-io/Cargo.toml b/lib/virtual-io/Cargo.toml index e6f60cb5d46..9aff58c8f37 100644 --- a/lib/virtual-io/Cargo.toml +++ b/lib/virtual-io/Cargo.toml @@ -17,6 +17,7 @@ tracing = "0.1" mio = { version = "0.8", features = [ "os-poll" ], optional = true } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } +futures = { version = "0.3" } [features] sys = [ "mio", "socket2" ] \ No newline at end of file diff --git a/lib/virtual-io/src/lib.rs b/lib/virtual-io/src/lib.rs index e1f82d10c79..9c2e0508c82 100644 --- a/lib/virtual-io/src/lib.rs +++ b/lib/virtual-io/src/lib.rs @@ -3,9 +3,11 @@ mod guard; mod interest; #[cfg(feature = "sys")] mod selector; +mod waker; #[cfg(feature = "sys")] pub use guard::*; pub use interest::*; #[cfg(feature = "sys")] pub use selector::*; +pub use waker::*; diff --git a/lib/wasix/src/runtime/task_manager/waker.rs b/lib/virtual-io/src/waker.rs similarity index 100% rename from lib/wasix/src/runtime/task_manager/waker.rs rename to lib/virtual-io/src/waker.rs diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index ae55a802969..ced709e23ca 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -20,15 +20,16 @@ mio = { version = "0.8", optional = true } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } -tun-tap = { version = "0.1.3", optional = true } -tokio-tungstenite = { version = "0.18.0", optional = true } -base64 = "^0.13" -aes = { version = "^0.7", optional = true } -ctr = { version = "^0.8", optional = true } -sha3 = { version = "0.10.6", optional = true } +#tun-tap = { version = "0.1.3", optional = true } +#tokio-tungstenite = { version = "0.18.0", optional = true } +base64 = "0.21" +aes = { version = "0.8", optional = true } +ctr = { version = "0.9.2", optional = true } +sha3 = { version = "0.10.8", optional = true } rand_chacha = { version = "^0.3", optional = true } blake3 = { version = "1.3.3", optional = true } +serde = { version = "1.0", default-features = false, features = ["derive"] } [features] host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "socket2", "mio" ] -ws-net = [ "tokio-tungstenite", "aes", "ctr", "sha3", "rand_chacha", "blake3" ] \ No newline at end of file +ws-net = [ "aes", "ctr", "sha3", "rand_chacha", "blake3" ] \ No newline at end of file diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 8b137891791..0998a6ed398 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -1 +1,696 @@ +use std::collections::HashMap; +use std::net::IpAddr; +use std::net::SocketAddr; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; +use derivative::Derivative; +use tokio::sync::mpsc; +use tokio::sync::Mutex; +use virtual_io::InlineWaker; + +use crate::meta::RequestType; +use crate::meta::ResponseType; +use crate::meta::SocketId; +use crate::meta::{InterfaceId, MessageRequest, MessageResponse}; +use crate::IpCidr; +use crate::IpRoute; +use crate::NetworkError; +use crate::StreamSecurity; +use crate::VirtualConnectedSocket; +use crate::VirtualConnectionlessSocket; +use crate::VirtualIcmpSocket; +use crate::VirtualIoSource; +use crate::VirtualNetworking; +use crate::VirtualRawSocket; +use crate::VirtualSocket; +use crate::VirtualTcpListener; +use crate::VirtualTcpSocket; +use crate::VirtualUdpSocket; + +use crate::Result; + +#[derive(Debug)] +enum RemoteTx { + Mpsc(mpsc::Sender), +} +impl RemoteTx { + async fn send(&self, req: MessageRequest) -> Result<()> { + match self { + RemoteTx::Mpsc(tx) => tx + .send(req) + .await + .map_err(|_| NetworkError::ConnectionAborted), + } + } +} + +#[derive(Debug)] +enum RemoteRx { + Mpsc(mpsc::Receiver), +} +impl RemoteRx { + async fn recv(&mut self) -> Option { + match self { + RemoteRx::Mpsc(rx) => rx.recv().await, + } + } +} + +#[derive(Derivative)] +#[derivative(Debug)] +struct RemoteCommon { + interface_id: InterfaceId, + tx: RemoteTx, + rx: Mutex, + request_seed: AtomicU64, + requests: Mutex>>, + socket_seed: AtomicU64, + handler_seed: AtomicU64, + #[derivative(Debug = "ignore")] + handlers: Mutex>>, +} + +impl RemoteCommon { + async fn io(&self, req_id: u64) -> ResponseType { + let req_rx = { + let (tx, rx) = mpsc::unbounded_channel(); + let mut guard = self.requests.lock().await; + guard.insert(req_id, tx); + rx + }; + + loop { + let mut rx_guard = tokio::select! { + rx = self.rx.lock() => rx, + res = req_rx => return res, + }; + tokio::select! { + msg = rx_guard.recv() => { + drop(rx_guard); + + let msg = match msg { + Some(msg) => msg, + None => return ResponseType::Err(NetworkError::ConnectionAborted) + }; + let mut requests = self.requests.lock().await; + if let Some(request) = requests.remove(&msg.req_id) { + request.send(msg.res); + } + }, + res = req_rx => return res, + } + } + } + + async fn io_iface(&self, req: RequestType) -> ResponseType { + let req_id = self.request_seed.fetch_add(1, Ordering::SeqCst); + if let Err(err) = self + .tx + .send(MessageRequest::Interface { + iface: self.interface_id, + req_id, + req, + }) + .await + { + return ResponseType::Err(err); + }; + self.io(req_id).await + } + + fn blocking_io_iface(&self, req: RequestType) -> ResponseType { + InlineWaker::block_on(self.io_iface(req)) + } +} + +#[derive(Debug)] +pub struct RemoteNetworking { + common: Arc, +} + +impl RemoteNetworking { + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_mpsc( + id: InterfaceId, + tx: mpsc::Sender, + rx: mpsc::Receiver, + ) -> Self { + let common = RemoteCommon { + interface_id: id, + tx: RemoteTx::Mpsc(tx), + rx: Mutex::new(RemoteRx::Mpsc(rx)), + request_seed: AtomicU64::new(1), + requests: Default::default(), + socket_seed: AtomicU64::new(1), + handler_seed: AtomicU64::new(1), + handlers: Default::default(), + }; + Self { + common: Arc::new(common), + } + } +} + +#[async_trait::async_trait] +impl VirtualNetworking for RemoteNetworking { + async fn bridge( + &self, + network: &str, + access_token: &str, + security: StreamSecurity, + ) -> Result<()> { + match self + .common + .io_iface(RequestType::Bridge { + network: network.to_string(), + access_token: access_token.to_string(), + security, + }) + .await + { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + async fn unbridge(&self) -> Result<()> { + match self.common.io_iface(RequestType::Unbridge).await { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + async fn dhcp_acquire(&self) -> Result> { + match self.common.io_iface(RequestType::DhcpAcquire).await { + ResponseType::Err(err) => Err(err), + ResponseType::IpAddressList(ips) => Ok(ips), + _ => Err(NetworkError::IOError), + } + } + + fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<()> { + match self + .common + .blocking_io_iface(RequestType::IpAdd { ip, prefix }) + { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn ip_remove(&self, ip: IpAddr) -> Result<()> { + match self.common.blocking_io_iface(RequestType::IpRemove(ip)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn ip_clear(&self) -> Result<()> { + match self.common.blocking_io_iface(RequestType::IpClear) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn ip_list(&self) -> Result> { + match self.common.blocking_io_iface(RequestType::Unbridge) { + ResponseType::Err(err) => Err(err), + ResponseType::CidrList(routes) => Ok(routes), + _ => Err(NetworkError::IOError), + } + } + + fn mac(&self) -> Result<[u8; 6]> { + match self.common.blocking_io_iface(RequestType::GetMac) { + ResponseType::Err(err) => Err(err), + ResponseType::Mac(mac) => Ok(mac), + _ => Err(NetworkError::IOError), + } + } + + fn gateway_set(&self, ip: IpAddr) -> Result<()> { + match self.common.blocking_io_iface(RequestType::GatewaySet(ip)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn route_add( + &self, + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, + ) -> Result<()> { + match self.common.blocking_io_iface(RequestType::RouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + }) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn route_remove(&self, cidr: IpAddr) -> Result<()> { + match self + .common + .blocking_io_iface(RequestType::RouteRemove(cidr)) + { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn route_clear(&self) -> Result<()> { + match self.common.blocking_io_iface(RequestType::RouteClear) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn route_list(&self) -> Result> { + match self.common.blocking_io_iface(RequestType::GetRouteList) { + ResponseType::Err(err) => Err(err), + ResponseType::RouteList(routes) => Ok(routes), + _ => Err(NetworkError::IOError), + } + } + + async fn bind_raw(&self) -> Result> { + let socket_id: SocketId = self + .common + .socket_seed + .fetch_add(1, Ordering::SeqCst) + .into(); + match self.common.io_iface(RequestType::BindRaw(socket_id)).await { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(Box::new(RemoteSocket { + socket_id, + common: self.common.clone(), + })), + _ => Err(NetworkError::IOError), + } + } + + async fn listen_tcp( + &self, + addr: SocketAddr, + only_v6: bool, + reuse_port: bool, + reuse_addr: bool, + ) -> Result> { + let socket_id: SocketId = self + .common + .socket_seed + .fetch_add(1, Ordering::SeqCst) + .into(); + match self + .common + .io_iface(RequestType::ListenTcp { + socket_id, + addr, + only_v6, + reuse_port, + reuse_addr, + }) + .await + { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(Box::new(RemoteSocket { + socket_id, + common: self.common.clone(), + })), + _ => Err(NetworkError::IOError), + } + } + + async fn bind_udp( + &self, + addr: SocketAddr, + reuse_port: bool, + reuse_addr: bool, + ) -> Result> { + let socket_id: SocketId = self + .common + .socket_seed + .fetch_add(1, Ordering::SeqCst) + .into(); + match self + .common + .io_iface(RequestType::BindUdp { + socket_id, + addr, + reuse_port, + reuse_addr, + }) + .await + { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(Box::new(RemoteSocket { + socket_id, + common: self.common.clone(), + })), + _ => Err(NetworkError::IOError), + } + } + + async fn bind_icmp(&self, addr: IpAddr) -> Result> { + let socket_id: SocketId = self + .common + .socket_seed + .fetch_add(1, Ordering::SeqCst) + .into(); + match self + .common + .io_iface(RequestType::BindIcmp { socket_id, addr }) + .await + { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(Box::new(RemoteSocket { + socket_id, + common: self.common.clone(), + })), + _ => Err(NetworkError::IOError), + } + } + + async fn connect_tcp( + &self, + addr: SocketAddr, + peer: SocketAddr, + ) -> Result> { + let socket_id: SocketId = self + .common + .socket_seed + .fetch_add(1, Ordering::SeqCst) + .into(); + match self + .common + .io_iface(RequestType::ConnectTcp { + socket_id, + addr, + peer, + }) + .await + { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(Box::new(RemoteSocket { + socket_id, + common: self.common.clone(), + })), + _ => Err(NetworkError::IOError), + } + } + + async fn resolve( + &self, + host: &str, + port: Option, + dns_server: Option, + ) -> Result> { + match self.common.io_iface(RequestType::RouteClear).await { + ResponseType::Err(err) => Err(err), + ResponseType::IpAddressList(ips) => Ok(ips), + _ => Err(NetworkError::IOError), + } + } +} + +#[derive(Debug)] +struct RemoteSocket { + socket_id: SocketId, + common: Arc, +} + +impl RemoteSocket { + async fn io_socket(&self, req: RequestType) -> ResponseType { + let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); + if let Err(_) = self + .common + .tx + .send(MessageRequest::Socket { + socket: self.socket_id, + req_id, + req, + }) + .await + { + return ResponseType::Err(NetworkError::ConnectionAborted); + }; + self.common.io(req_id).await + } + + fn blocking_io_socket(&self, req: RequestType) -> ResponseType { + InlineWaker::block_on(self.io_socket(req)) + } +} + +impl VirtualIoSource for RemoteSocket { + fn remove_handler(&mut self) { + self.blocking_io_socket(RequestType::RemoveHandler); + } +} + +impl VirtualSocket for RemoteSocket { + fn set_ttl(&mut self, ttl: u32) -> Result<()> { + match self.blocking_io_socket(RequestType::SetTtl(ttl)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } + } + + fn ttl(&self) -> Result { + match self.blocking_io_socket(RequestType::GetTtl) { + ResponseType::Err(err) => Err(err), + ResponseType::Ttl(ttl) => Ok(ttl), + _ => Err(NetworkError::IOError), + } + } + + fn addr_local(&self) -> Result { + match self.blocking_io_socket(RequestType::GetAddrLocal) { + ResponseType::Err(err) => Err(err), + ResponseType::SocketAddr(addr) => Ok(addr), + _ => Err(NetworkError::IOError), + } + } + + fn status(&self) -> Result { + match self.blocking_io_socket(RequestType::GetStatus) { + ResponseType::Err(err) => Err(err), + ResponseType::Status(status) => Ok(status), + _ => Err(NetworkError::IOError), + } + } + + fn set_handler( + &mut self, + handler: Box, + ) -> Result<()> { + todo!() + } +} + +impl VirtualTcpListener for RemoteSocket { + fn try_accept(&mut self) -> Result<(Box, SocketAddr)> { + todo!() + } + + fn set_handler( + &mut self, + handler: Box, + ) -> Result<()> { + todo!() + } + + fn addr_local(&self) -> Result { + todo!() + } + + fn set_ttl(&mut self, ttl: u8) -> Result<()> { + todo!() + } + + fn ttl(&self) -> Result { + todo!() + } +} + +impl VirtualRawSocket for RemoteSocket { + fn try_send(&mut self, data: &[u8]) -> Result { + todo!() + } + + fn try_flush(&mut self) -> Result<()> { + todo!() + } + + fn try_recv(&mut self, buf: &mut [std::mem::MaybeUninit]) -> Result { + todo!() + } + + fn set_promiscuous(&mut self, promiscuous: bool) -> Result<()> { + todo!() + } + + fn promiscuous(&self) -> Result { + todo!() + } +} + +impl VirtualConnectionlessSocket for RemoteSocket { + fn try_send_to(&mut self, data: &[u8], addr: SocketAddr) -> Result { + todo!() + } + + fn try_recv_from( + &mut self, + buf: &mut [std::mem::MaybeUninit], + ) -> Result<(usize, SocketAddr)> { + todo!() + } +} + +impl VirtualUdpSocket for RemoteSocket { + fn set_broadcast(&mut self, broadcast: bool) -> Result<()> { + todo!() + } + + fn broadcast(&self) -> Result { + todo!() + } + + fn set_multicast_loop_v4(&mut self, val: bool) -> Result<()> { + todo!() + } + + fn multicast_loop_v4(&self) -> Result { + todo!() + } + + fn set_multicast_loop_v6(&mut self, val: bool) -> Result<()> { + todo!() + } + + fn multicast_loop_v6(&self) -> Result { + todo!() + } + + fn set_multicast_ttl_v4(&mut self, ttl: u32) -> Result<()> { + todo!() + } + + fn multicast_ttl_v4(&self) -> Result { + todo!() + } + + fn join_multicast_v4( + &mut self, + multiaddr: std::net::Ipv4Addr, + iface: std::net::Ipv4Addr, + ) -> Result<()> { + todo!() + } + + fn leave_multicast_v4( + &mut self, + multiaddr: std::net::Ipv4Addr, + iface: std::net::Ipv4Addr, + ) -> Result<()> { + todo!() + } + + fn join_multicast_v6(&mut self, multiaddr: std::net::Ipv6Addr, iface: u32) -> Result<()> { + todo!() + } + + fn leave_multicast_v6(&mut self, multiaddr: std::net::Ipv6Addr, iface: u32) -> Result<()> { + todo!() + } + + fn addr_peer(&self) -> Result> { + todo!() + } +} + +impl VirtualIcmpSocket for RemoteSocket {} + +impl VirtualConnectedSocket for RemoteSocket { + fn set_linger(&mut self, linger: Option) -> Result<()> { + todo!() + } + + fn linger(&self) -> Result> { + todo!() + } + + fn try_send(&mut self, data: &[u8]) -> Result { + todo!() + } + + fn try_flush(&mut self) -> Result<()> { + todo!() + } + + fn close(&mut self) -> Result<()> { + todo!() + } + + fn try_recv(&mut self, buf: &mut [std::mem::MaybeUninit]) -> Result { + todo!() + } +} + +impl VirtualTcpSocket for RemoteSocket { + fn set_recv_buf_size(&mut self, size: usize) -> Result<()> { + todo!() + } + + fn recv_buf_size(&self) -> Result { + todo!() + } + + fn set_send_buf_size(&mut self, size: usize) -> Result<()> { + todo!() + } + + fn send_buf_size(&self) -> Result { + todo!() + } + + fn set_nodelay(&mut self, reuse: bool) -> Result<()> { + todo!() + } + + fn nodelay(&self) -> Result { + todo!() + } + + fn addr_peer(&self) -> Result { + todo!() + } + + fn shutdown(&mut self, how: std::net::Shutdown) -> Result<()> { + todo!() + } + + fn is_closed(&self) -> bool { + todo!() + } +} diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index e8fa29e1120..938a9e18204 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -2,22 +2,24 @@ pub mod client; #[cfg(feature = "host-net")] pub mod host; +pub mod meta; #[cfg(feature = "host-net")] pub mod tun; use std::fmt; use std::mem::MaybeUninit; -use std::net::IpAddr; -use std::net::Ipv4Addr; -use std::net::Ipv6Addr; -use std::net::Shutdown; -use std::net::SocketAddr; +pub use std::net::IpAddr; +pub use std::net::Ipv4Addr; +pub use std::net::Ipv6Addr; +pub use std::net::Shutdown; +pub use std::net::SocketAddr; use std::sync::Arc; -use std::time::Duration; +pub use std::time::Duration; use thiserror::Error; pub use bytes::Bytes; pub use bytes::BytesMut; +use serde::{Deserialize, Serialize}; pub use virtual_io::{handler_into_waker, InterestHandler}; #[cfg(feature = "host-net")] pub use virtual_io::{InterestGuard, InterestHandlerWaker, InterestType}; @@ -25,14 +27,14 @@ pub use virtual_io::{InterestGuard, InterestHandlerWaker, InterestType}; pub type Result = std::result::Result; /// Represents an IP address and its netmask -#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Deserialize)] pub struct IpCidr { pub ip: IpAddr, pub prefix: u8, } /// Represents a routing entry in the routing table of the interface -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct IpRoute { pub cidr: IpCidr, pub via_router: IpAddr, @@ -223,7 +225,7 @@ pub trait VirtualSocket: VirtualIoSource + fmt::Debug + Send + Sync + 'static { fn set_handler(&mut self, handler: Box) -> Result<()>; } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum SocketStatus { Opening, Opened, @@ -231,7 +233,7 @@ pub enum SocketStatus { Failed, } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum StreamSecurity { Unencrypted, AnyEncyption, @@ -411,7 +413,7 @@ pub struct UnsupportedVirtualNetworking {} #[async_trait::async_trait] impl VirtualNetworking for UnsupportedVirtualNetworking {} -#[derive(Error, Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Error, Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum NetworkError { /// The handle given was not usable #[error("invalid fd")] diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs new file mode 100644 index 00000000000..1291b617ce1 --- /dev/null +++ b/lib/virtual-net/src/meta.rs @@ -0,0 +1,300 @@ +use serde::{Deserialize, Serialize}; + +pub use super::Duration; +pub use super::IpAddr; +pub use super::IpCidr; +pub use super::IpRoute; +pub use super::Ipv4Addr; +pub use super::Ipv6Addr; +pub use super::NetworkError; +pub use super::SocketAddr; +pub use super::SocketStatus; +pub use super::StreamSecurity; + +/// Represents a socket ID +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct SocketId(u64); + +impl From for SocketId { + fn from(value: u64) -> Self { + Self(value) + } +} + +/// Represents a handler ID +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct HandlerId(u64); + +/// Possible values which can be passed to the [`TcpStream::shutdown`] method. +#[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub enum Shutdown { + /// The reading portion of the [`TcpStream`] should be shut down. + Read, + /// The writing portion of the [`TcpStream`] should be shut down. + Write, + /// Both the reading and the writing portions of the [`TcpStream`] should be shut down. + Both, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum RequestType { + /// Registers a waker for when a new connection has arrived. This uses + /// a stack machine which means more than one waker can be registered + SetHandler(HandlerId), + /// Removes a previously registered waker using a token + RemoveHandler, + /// Bridges this local network with a remote network, which is required in + /// order to make lower level networking calls (such as UDP/TCP) + Bridge { + network: String, + access_token: String, + security: StreamSecurity, + }, + /// Disconnects from the remote network essentially unbridging it + Unbridge, + /// Acquires an IP address on the network and configures the routing tables + DhcpAcquire, + /// Adds a static IP address to the interface with a netmask prefix + IpAdd { ip: IpAddr, prefix: u8 }, + /// Removes a static (or dynamic) IP address from the interface + IpRemove(IpAddr), + /// Clears all the assigned IP addresses for this interface + IpClear, + /// Lists all the IP addresses currently assigned to this interface + GetIpList, + /// Returns the hardware MAC address for this interface + GetMac, + /// Adds a default gateway to the routing table + GatewaySet(IpAddr), + /// Adds a specific route to the routing table + RouteAdd { + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, + }, + /// Removes a routing rule from the routing table + RouteRemove(IpAddr), + /// Clears the routing table for this interface + RouteClear, + /// Lists all the routes defined in the routing table for this interface + GetRouteList, + /// Creates a low level socket that can read and write Ethernet packets + /// directly to the interface + BindRaw(SocketId), + /// Lists for TCP connections on a specific IP and Port combination + /// Multiple servers (processes or threads) can bind to the same port if they each set + /// the reuse-port and-or reuse-addr flags + ListenTcp { + socket_id: SocketId, + addr: SocketAddr, + only_v6: bool, + reuse_port: bool, + reuse_addr: bool, + }, + /// Opens a UDP socket that listens on a specific IP and Port combination + /// Multiple servers (processes or threads) can bind to the same port if they each set + /// the reuse-port and-or reuse-addr flags + BindUdp { + socket_id: SocketId, + addr: SocketAddr, + reuse_port: bool, + reuse_addr: bool, + }, + /// Creates a socket that can be used to send and receive ICMP packets + /// from a paritcular IP address + BindIcmp { socket_id: SocketId, addr: IpAddr }, + /// Opens a TCP connection to a particular destination IP address and port + ConnectTcp { + socket_id: SocketId, + addr: SocketAddr, + peer: SocketAddr, + }, + /// Performs DNS resolution for a specific hostname + Resolve { + host: String, + port: Option, + dns_server: Option, + }, + /// Tries to send out a datagram or stream of bytes on this socket + TrySend(Vec), + /// Sends out a datagram or stream of bytes on this socket + /// to a specific address + TrySendTo { data: Vec, addr: SocketAddr }, + /// Tries to flush any data in the local buffers + TryFlush, + /// Tries to read a packet from the socket + TryRecv { max_bytes: u64 }, + /// Recv a packet from the socket + TryRecvFrom { max_bytes: u64 }, + /// Closes the socket + Close, + /// Tries to accept a new connection + TryAccept, + /// Returns the local address of this TCP listener + GetAddrLocal, + /// Returns the address (IP and Port) of the peer socket that this + /// is conencted to + GetAddrPeer, + /// Sets how many network hops the packets are permitted for new connections + SetTtl(u32), + /// Returns the maximum number of network hops before packets are dropped + GetTtl, + /// Returns the status/state of the socket + GetStatus, + /// Determines how long the socket will remain in a TIME_WAIT + /// after it disconnects (only the one that initiates the close will + /// be in a TIME_WAIT state thus the clients should always do this rather + /// than the server) + SetLinger(Option), + /// Returns how long the socket will remain in a TIME_WAIT + /// after it disconnects + GetLinger, + /// Tells the raw socket and its backing switch that all packets + /// should be received by this socket even if they are not + /// destined for this device + SetPromiscuous(bool), + /// Returns if the socket is running in promiscuous mode whereby it + /// will receive all packets even if they are not destined for the + /// local interface + GetPromiscuous, + /// Sets the receive buffer size which acts as a trottle for how + /// much data is buffered on this side of the pipe + SetRecvBufSize(u64), + /// Size of the receive buffer that holds all data that has not + /// yet been read + GetRecvBufSize, + /// Sets the size of the send buffer which will hold the bytes of + /// data while they are being sent over to the peer + SetSendBufSize(u64), + /// Size of the send buffer that holds all data that is currently + /// being transmitted. + GetSendBufSize, + /// When NO_DELAY is set the data that needs to be transmitted to + /// the peer is sent immediately rather than waiting for a bigger + /// batch of data, this reduces latency but increases encapsulation + /// overhead. + SetNoDelay(bool), + /// Indicates if the NO_DELAY flag is set which means that data + /// is immediately sent to the peer without waiting. This reduces + /// latency but increases encapsulation overhead. + GetNoDelay, + /// Shuts down either the READER or WRITER sides of the socket + /// connection. + Shutdown(Shutdown), + /// Return true if the socket is closed + IsClosed, + /// Sets a flag that means that the UDP socket is able + /// to receive and process broadcast packets. + SetBroadcast(bool), + /// Indicates if the SO_BROADCAST flag is set which means + /// that the UDP socket will receive and process broadcast + /// packets + GetBroadcast, + /// Sets a flag that indicates if multicast packets that + /// this socket is a member of will be looped back to + /// the sending socket. This applies to IPv4 addresses + SetMulticastLoopV4(bool), + /// Gets a flag that indicates if multicast packets that + /// this socket is a member of will be looped back to + /// the sending socket. This applies to IPv4 addresses + GetMulticastLoopV4, + /// Sets a flag that indicates if multicast packets that + /// this socket is a member of will be looped back to + /// the sending socket. This applies to IPv6 addresses + SetMulticastLoopV6(bool), + /// Gets a flag that indicates if multicast packets that + /// this socket is a member of will be looped back to + /// the sending socket. This applies to IPv6 addresses + GetMulticastLoopV6, + /// Sets the TTL for IPv4 multicast packets which is the + /// number of network hops before the packet is dropped + SetMulticastTtlV4(u32), + /// Gets the TTL for IPv4 multicast packets which is the + /// number of network hops before the packet is dropped + GetMulticastTtlV4, + /// Tells this interface that it will subscribe to a + /// particular multicast address. This applies to IPv4 addresses + JoinMulticastV4 { + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + }, + /// Tells this interface that it will unsubscribe to a + /// particular multicast address. This applies to IPv4 addresses + LeaveMulticastV4 { + multiaddr: Ipv4Addr, + iface: Ipv4Addr, + }, + /// Tells this interface that it will subscribe to a + /// particular multicast address. This applies to IPv6 addresses + JoinMulticastV6 { multiaddr: Ipv6Addr, iface: u32 }, + /// Tells this interface that it will unsubscribe to a + /// particular multicast address. This applies to IPv6 addresses + LeaveMulticastV6 { multiaddr: Ipv6Addr, iface: u32 }, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum ResponseType { + /// Nothing is returned (or noop) + None, + /// An error has occurred + Err(NetworkError), + /// Represents a duration of time + Duration(Duration), + /// Represents an amount (e.g. amount of bytes) + Amount(u64), + /// Represents a piece of data + Data(Vec), + /// Represents a peice of data and a socket address + DataWithAddr { data: Vec, addr: SocketAddr }, + /// Returns a flag of true or false + Flag(bool), + /// List of IP addresses + IpAddressList(Vec), + /// A single IP address + IpAddress(IpAddr), + /// List of socket addresses + SocketAddrList(Vec), + /// A single IP address + SocketAddr(SocketAddr), + /// Represents a MAC address + Mac([u8; 6]), + /// List of CIDR routes from a routing table + CidrList(Vec), + /// List of IP routes from a routing table + RouteList(Vec), + /// Reference to a socket + Socket(SocketId), + /// Reference to a socket with an address + SocketWithAddr { id: SocketId, addr: SocketAddr }, + /// The TTL of a packet + Ttl(u32), + /// The status of the socket + Status(SocketStatus), +} + +/// Represents an interface ID +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct InterfaceId(u64); + +/// Message sent by the client to the server +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum MessageRequest { + Interface { + iface: InterfaceId, + req: RequestType, + req_id: u64, + }, + Socket { + socket: SocketId, + req: RequestType, + req_id: u64, + }, +} + +/// Message sent by the server back to a client +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct MessageResponse { + pub req_id: u64, + pub res: ResponseType, +} diff --git a/lib/wasix/src/runtime/task_manager/mod.rs b/lib/wasix/src/runtime/task_manager/mod.rs index 02977367385..c69c43dab81 100644 --- a/lib/wasix/src/runtime/task_manager/mod.rs +++ b/lib/wasix/src/runtime/task_manager/mod.rs @@ -1,7 +1,6 @@ // TODO: should be behind a different , tokio specific feature flag. #[cfg(feature = "sys-thread")] pub mod tokio; -mod waker; use std::ops::Deref; use std::task::{Context, Poll}; @@ -17,7 +16,7 @@ use crate::os::task::thread::WasiThreadError; use crate::syscalls::AsyncifyFuture; use crate::{capture_snapshot, InstanceSnapshot, WasiEnv, WasiFunctionEnv, WasiThread}; -pub use waker::*; +pub use virtual_io::waker::*; #[derive(Debug)] pub enum SpawnMemoryType<'a> { From 7a55380f8fbe06d168516aacaae6d9cb89b67c1f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 2 Jul 2023 07:27:19 +1000 Subject: [PATCH 05/52] Shutdown does not need to be public --- lib/virtual-net/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index 938a9e18204..b7497582603 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -11,7 +11,7 @@ use std::mem::MaybeUninit; pub use std::net::IpAddr; pub use std::net::Ipv4Addr; pub use std::net::Ipv6Addr; -pub use std::net::Shutdown; +use std::net::Shutdown; pub use std::net::SocketAddr; use std::sync::Arc; pub use std::time::Duration; From a98deb5a8b9308cfa25e52f1e759fa6ecf6189ed Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 6 Jul 2023 21:35:01 +1000 Subject: [PATCH 06/52] Finished the client side of the VPN --- Cargo.lock | 2 + lib/virtual-io/Cargo.toml | 1 + lib/virtual-io/src/interest.rs | 3 +- lib/virtual-io/src/lib.rs | 2 +- lib/virtual-net/Cargo.toml | 1 + lib/virtual-net/src/client.rs | 694 +++++++++++++++++++++++++++------ lib/virtual-net/src/meta.rs | 64 +-- 7 files changed, 626 insertions(+), 141 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9fdb41a57d0..17e556fb54c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5137,6 +5137,7 @@ dependencies = [ "derivative", "futures", "mio", + "serde", "socket2", "thiserror", "tracing", @@ -5149,6 +5150,7 @@ dependencies = [ "aes", "async-trait", "base64", + "bincode", "blake3", "bytes", "ctr", diff --git a/lib/virtual-io/Cargo.toml b/lib/virtual-io/Cargo.toml index 9aff58c8f37..48cdf577209 100644 --- a/lib/virtual-io/Cargo.toml +++ b/lib/virtual-io/Cargo.toml @@ -18,6 +18,7 @@ mio = { version = "0.8", features = [ "os-poll" ], optional = true } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } futures = { version = "0.3" } +serde = { version = "1.0", default-features = false, features = ["derive"] } [features] sys = [ "mio", "socket2" ] \ No newline at end of file diff --git a/lib/virtual-io/src/interest.rs b/lib/virtual-io/src/interest.rs index bff6cb0bbbe..8da03be08b9 100644 --- a/lib/virtual-io/src/interest.rs +++ b/lib/virtual-io/src/interest.rs @@ -1,3 +1,4 @@ +use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, sync::{Arc, Mutex}, @@ -6,7 +7,7 @@ use std::{ use derivative::Derivative; -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Clone, Serialize, Deserialize, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum InterestType { Readable, Writable, diff --git a/lib/virtual-io/src/lib.rs b/lib/virtual-io/src/lib.rs index 9c2e0508c82..f5fa03bbc8e 100644 --- a/lib/virtual-io/src/lib.rs +++ b/lib/virtual-io/src/lib.rs @@ -3,7 +3,7 @@ mod guard; mod interest; #[cfg(feature = "sys")] mod selector; -mod waker; +pub mod waker; #[cfg(feature = "sys")] pub use guard::*; diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index ced709e23ca..1b3e3424291 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -28,6 +28,7 @@ ctr = { version = "0.9.2", optional = true } sha3 = { version = "0.10.8", optional = true } rand_chacha = { version = "^0.3", optional = true } blake3 = { version = "1.3.3", optional = true } +bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } [features] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 0998a6ed398..bbd0d1a6246 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -1,16 +1,32 @@ use std::collections::HashMap; +use std::future::Future; +use std::mem::MaybeUninit; use std::net::IpAddr; use std::net::SocketAddr; +use std::pin::Pin; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::sync::Arc; +use std::sync::Mutex; +use std::task::Context; +use std::task::Poll; use std::time::Duration; +use bytes::Buf; +use bytes::BytesMut; use derivative::Derivative; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; +use tokio::io::AsyncWriteExt; +use tokio::io::ReadBuf; use tokio::sync::mpsc; -use tokio::sync::Mutex; +use tokio::sync::mpsc::error::TryRecvError; +use tokio::sync::mpsc::error::TrySendError; use virtual_io::InlineWaker; +use virtual_io::InterestType; +use crate::host::io_err_into_net_error; +use crate::meta; use crate::meta::RequestType; use crate::meta::ResponseType; use crate::meta::SocketId; @@ -32,9 +48,9 @@ use crate::VirtualUdpSocket; use crate::Result; -#[derive(Debug)] enum RemoteTx { Mpsc(mpsc::Sender), + Stream(tokio::sync::Mutex>>), } impl RemoteTx { async fn send(&self, req: MessageRequest) -> Result<()> { @@ -43,18 +59,107 @@ impl RemoteTx { .send(req) .await .map_err(|_| NetworkError::ConnectionAborted), + RemoteTx::Stream(tx) => { + let mut tx = tx.lock().await; + let data = bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {}", err); + NetworkError::IOError + })?; + let data_len = data.len() as u64; + let data_len_buf = data_len.to_le_bytes(); + tx.write_all(&data_len_buf) + .await + .map_err(io_err_into_net_error)?; + tx.write_all(&data).await.map_err(io_err_into_net_error) + } + } + } + fn try_send(&self, req: MessageRequest) -> Result<()> { + match self { + RemoteTx::Mpsc(tx) => match tx.try_send(req) { + Ok(()) => Ok(()), + Err(TrySendError::Closed(_)) => Err(NetworkError::ConnectionAborted), + Err(TrySendError::Full(_)) => Err(NetworkError::WouldBlock), + }, + RemoteTx::Stream(_) => InlineWaker::block_on(self.send(req)), } } } -#[derive(Debug)] enum RemoteRx { Mpsc(mpsc::Receiver), + Stream { + rx: Pin>, + next: Option, + buf: BytesMut, + }, } impl RemoteRx { - async fn recv(&mut self) -> Option { - match self { - RemoteRx::Mpsc(rx) => rx.recv().await, + fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { + loop { + return match self { + RemoteRx::Mpsc(rx) => Pin::new(rx).poll_recv(cx), + RemoteRx::Stream { rx, next, buf } => { + match next.clone() { + Some(next) if (buf.len() as u64) >= next => { + let next = next as usize; + let msg = match bincode::deserialize(&buf[..next]) { + Ok(m) => m, + Err(err) => { + tracing::warn!("failed to deserialize message - {}", err); + return Poll::Ready(None); + } + }; + buf.advance(next); + return Poll::Ready(Some(msg)); + } + None if buf.len() >= 8 => { + let mut data_len_buf = [0u8; 8]; + data_len_buf.copy_from_slice(&buf[..8]); + buf.advance(8); + next.replace(u64::from_le_bytes(data_len_buf)); + continue; + } + _ => {} + } + + let mut chunk: [MaybeUninit; 4096] = + unsafe { MaybeUninit::uninit().assume_init() }; + let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..]; + let chunk_unsafe: &mut [u8] = unsafe { std::mem::transmute(chunk_unsafe) }; + + let mut read_buf = ReadBuf::new(chunk_unsafe); + match rx.as_mut().poll_read(cx, &mut read_buf) { + Poll::Ready(Ok(_)) => { + let filled = read_buf.filled(); + if filled.is_empty() { + return Poll::Ready(None); + } + buf.extend_from_slice(&filled); + continue; + } + Poll::Ready(Err(err)) => { + tracing::warn!("failed to read from channel - {}", err); + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + } + } + }; + } + } +} + +#[derive(Debug)] +struct RequestTx { + tx: mpsc::Sender, +} +impl RequestTx { + pub fn try_send(self, msg: ResponseType) -> Result<()> { + match self.tx.try_send(msg) { + Ok(()) => Ok(()), + Err(TrySendError::Closed(_)) => Err(NetworkError::ConnectionAborted), + Err(TrySendError::Full(_)) => Err(NetworkError::WouldBlock), } } } @@ -63,50 +168,28 @@ impl RemoteRx { #[derivative(Debug)] struct RemoteCommon { interface_id: InterfaceId, + #[derivative(Debug = "ignore")] tx: RemoteTx, + #[derivative(Debug = "ignore")] rx: Mutex, request_seed: AtomicU64, - requests: Mutex>>, + requests: Mutex>, socket_seed: AtomicU64, - handler_seed: AtomicU64, + recv_tx: Mutex>>>, + recv_with_addr_tx: Mutex, SocketAddr)>>>, #[derivative(Debug = "ignore")] - handlers: Mutex>>, + handlers: Mutex>>, } impl RemoteCommon { - async fn io(&self, req_id: u64) -> ResponseType { - let req_rx = { - let (tx, rx) = mpsc::unbounded_channel(); - let mut guard = self.requests.lock().await; - guard.insert(req_id, tx); - rx - }; - - loop { - let mut rx_guard = tokio::select! { - rx = self.rx.lock() => rx, - res = req_rx => return res, - }; - tokio::select! { - msg = rx_guard.recv() => { - drop(rx_guard); - - let msg = match msg { - Some(msg) => msg, - None => return ResponseType::Err(NetworkError::ConnectionAborted) - }; - let mut requests = self.requests.lock().await; - if let Some(request) = requests.remove(&msg.req_id) { - request.send(msg.res); - } - }, - res = req_rx => return res, - } - } - } - async fn io_iface(&self, req: RequestType) -> ResponseType { let req_id = self.request_seed.fetch_add(1, Ordering::SeqCst); + let mut req_rx = { + let (tx, rx) = mpsc::channel(1); + let mut guard = self.requests.lock().unwrap(); + guard.insert(req_id, RequestTx { tx }); + rx + }; if let Err(err) = self .tx .send(MessageRequest::Interface { @@ -118,7 +201,7 @@ impl RemoteCommon { { return ResponseType::Err(err); }; - self.io(req_id).await + req_rx.recv().await.unwrap() } fn blocking_io_iface(&self, req: RequestType) -> ResponseType { @@ -131,6 +214,109 @@ pub struct RemoteNetworking { common: Arc, } +#[derive(Derivative)] +#[derivative(Debug)] +pub struct RemoteNetworkingDriver { + #[derivative(Debug = "ignore")] + polling: Option + Send + Sync>>>, + common: Arc, +} + +impl Future for RemoteNetworkingDriver { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + loop { + if let Some(polling) = self.polling.as_mut() { + match polling.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(()) => {} + } + self.polling.take(); + } + let msg = { + let mut rx_guard = self.common.rx.lock().unwrap(); + rx_guard.poll(cx) + }; + return match msg { + Poll::Ready(Some(msg)) => { + match msg { + MessageResponse::Recv { socket_id, data } => { + let tx = { + let guard = self.common.recv_tx.lock().unwrap(); + match guard.get(&socket_id) { + Some(tx) => tx.clone(), + None => continue, + } + }; + let common = self.common.clone(); + self.polling.replace(Box::pin(async move { + tx.send(data).await.ok(); + + common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Readable)); + })); + } + MessageResponse::RecvWithAddr { + socket_id, + data, + addr, + } => { + let tx = { + let guard = self.common.recv_with_addr_tx.lock().unwrap(); + match guard.get(&socket_id) { + Some(tx) => tx.clone(), + None => continue, + } + }; + let common = self.common.clone(); + self.polling.replace(Box::pin(async move { + tx.send((data, addr)).await.ok(); + + common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Readable)); + })); + } + MessageResponse::Sent { socket_id, .. } => { + self.common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Writable)); + } + MessageResponse::Closed { socket_id } => { + self.common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Closed)); + } + MessageResponse::ResponseToRequest { req_id, res } => { + let mut requests = self.common.requests.lock().unwrap(); + if let Some(request) = requests.remove(&req_id) { + request.try_send(res).ok(); + } + } + } + continue; + } + Poll::Ready(None) => Poll::Ready(()), + Poll::Pending => Poll::Pending, + }; + } + } +} + impl RemoteNetworking { /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels @@ -138,7 +324,7 @@ impl RemoteNetworking { id: InterfaceId, tx: mpsc::Sender, rx: mpsc::Receiver, - ) -> Self { + ) -> (Self, RemoteNetworkingDriver) { let common = RemoteCommon { interface_id: id, tx: RemoteTx::Mpsc(tx), @@ -146,11 +332,67 @@ impl RemoteNetworking { request_seed: AtomicU64::new(1), requests: Default::default(), socket_seed: AtomicU64::new(1), - handler_seed: AtomicU64::new(1), + recv_tx: Default::default(), + recv_with_addr_tx: Default::default(), handlers: Default::default(), }; - Self { - common: Arc::new(common), + let common = Arc::new(common); + + let driver = RemoteNetworkingDriver { + polling: None, + common: common.clone(), + }; + let networking = Self { common }; + + (networking, driver) + } + + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_stream( + id: InterfaceId, + tx: Pin>, + rx: Pin>, + ) -> (Self, RemoteNetworkingDriver) { + let common = RemoteCommon { + interface_id: id, + tx: RemoteTx::Stream(tokio::sync::Mutex::new(tx)), + rx: Mutex::new(RemoteRx::Stream { + rx, + next: None, + buf: BytesMut::new(), + }), + request_seed: AtomicU64::new(1), + requests: Default::default(), + socket_seed: AtomicU64::new(1), + recv_tx: Default::default(), + recv_with_addr_tx: Default::default(), + handlers: Default::default(), + }; + let common = Arc::new(common); + + let driver = RemoteNetworkingDriver { + polling: None, + common: common.clone(), + }; + let networking = Self { common }; + + (networking, driver) + } + + fn new_socket(&self, id: SocketId) -> RemoteSocket { + let (tx, rx_recv) = tokio::sync::mpsc::channel(100); + self.common.recv_tx.lock().unwrap().insert(id, tx); + + let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); + self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); + + RemoteSocket { + socket_id: id, + common: self.common.clone(), + rx_buffer: BytesMut::new(), + rx_recv, + rx_recv_with_addr, } } } @@ -299,10 +541,7 @@ impl VirtualNetworking for RemoteNetworking { .into(); match self.common.io_iface(RequestType::BindRaw(socket_id)).await { ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(Box::new(RemoteSocket { - socket_id, - common: self.common.clone(), - })), + ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), _ => Err(NetworkError::IOError), } } @@ -331,10 +570,7 @@ impl VirtualNetworking for RemoteNetworking { .await { ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(Box::new(RemoteSocket { - socket_id, - common: self.common.clone(), - })), + ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), _ => Err(NetworkError::IOError), } } @@ -361,10 +597,7 @@ impl VirtualNetworking for RemoteNetworking { .await { ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(Box::new(RemoteSocket { - socket_id, - common: self.common.clone(), - })), + ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), _ => Err(NetworkError::IOError), } } @@ -381,10 +614,7 @@ impl VirtualNetworking for RemoteNetworking { .await { ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(Box::new(RemoteSocket { - socket_id, - common: self.common.clone(), - })), + ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), _ => Err(NetworkError::IOError), } } @@ -409,10 +639,7 @@ impl VirtualNetworking for RemoteNetworking { .await { ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(Box::new(RemoteSocket { - socket_id, - common: self.common.clone(), - })), + ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), _ => Err(NetworkError::IOError), } } @@ -423,7 +650,15 @@ impl VirtualNetworking for RemoteNetworking { port: Option, dns_server: Option, ) -> Result> { - match self.common.io_iface(RequestType::RouteClear).await { + match self + .common + .io_iface(RequestType::Resolve { + host: host.to_string(), + port, + dns_server, + }) + .await + { ResponseType::Err(err) => Err(err), ResponseType::IpAddressList(ips) => Ok(ips), _ => Err(NetworkError::IOError), @@ -435,12 +670,31 @@ impl VirtualNetworking for RemoteNetworking { struct RemoteSocket { socket_id: SocketId, common: Arc, + rx_buffer: BytesMut, + rx_recv: mpsc::Receiver>, + rx_recv_with_addr: mpsc::Receiver<(Vec, SocketAddr)>, +} +impl Drop for RemoteSocket { + fn drop(&mut self) { + self.common.recv_tx.lock().unwrap().remove(&self.socket_id); + self.common + .recv_with_addr_tx + .lock() + .unwrap() + .remove(&self.socket_id); + } } impl RemoteSocket { async fn io_socket(&self, req: RequestType) -> ResponseType { let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); - if let Err(_) = self + let mut req_rx = { + let (tx, rx) = mpsc::channel(1); + let mut guard = self.common.requests.lock().unwrap(); + guard.insert(req_id, RequestTx { tx }); + rx + }; + if let Err(err) = self .common .tx .send(MessageRequest::Socket { @@ -450,9 +704,9 @@ impl RemoteSocket { }) .await { - return ResponseType::Err(NetworkError::ConnectionAborted); + return ResponseType::Err(err); }; - self.common.io(req_id).await + req_rx.recv().await.unwrap() } fn blocking_io_socket(&self, req: RequestType) -> ResponseType { @@ -462,7 +716,7 @@ impl RemoteSocket { impl VirtualIoSource for RemoteSocket { fn remove_handler(&mut self) { - self.blocking_io_socket(RequestType::RemoveHandler); + self.common.handlers.lock().unwrap().remove(&self.socket_id); } } @@ -503,101 +757,220 @@ impl VirtualSocket for RemoteSocket { &mut self, handler: Box, ) -> Result<()> { - todo!() + self.common + .handlers + .lock() + .unwrap() + .insert(self.socket_id, handler); + Ok(()) } } impl VirtualTcpListener for RemoteSocket { fn try_accept(&mut self) -> Result<(Box, SocketAddr)> { - todo!() + match self.blocking_io_socket(RequestType::TryAccept) { + ResponseType::Err(err) => Err(err), + ResponseType::SocketWithAddr { id, addr } => { + let (tx, rx_recv) = tokio::sync::mpsc::channel(100); + self.common.recv_tx.lock().unwrap().insert(id, tx); + + let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); + self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); + + let socket = RemoteSocket { + socket_id: id, + common: self.common.clone(), + rx_buffer: BytesMut::new(), + rx_recv, + rx_recv_with_addr, + }; + Ok((Box::new(socket), addr)) + } + _ => Err(NetworkError::IOError), + } } fn set_handler( &mut self, handler: Box, ) -> Result<()> { - todo!() + VirtualSocket::set_handler(self, handler) } fn addr_local(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetAddrLocal) { + ResponseType::Err(err) => Err(err), + ResponseType::SocketAddr(addr) => Ok(addr), + _ => Err(NetworkError::IOError), + } } fn set_ttl(&mut self, ttl: u8) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetTtl(ttl as u32)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn ttl(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetTtl) { + ResponseType::Err(err) => Err(err), + ResponseType::Ttl(val) => Ok(val.try_into().map_err(|_| NetworkError::InvalidData)?), + _ => Err(NetworkError::IOError), + } } } impl VirtualRawSocket for RemoteSocket { fn try_send(&mut self, data: &[u8]) -> Result { - todo!() + let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); + self.common + .tx + .try_send(MessageRequest::Send { + socket: self.socket_id, + data: data.to_vec(), + req_id, + }) + .map(|_| data.len()) } fn try_flush(&mut self) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::Flush) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn try_recv(&mut self, buf: &mut [std::mem::MaybeUninit]) -> Result { - todo!() + loop { + if self.rx_buffer.len() > 0 { + let amt = self.rx_buffer.len().min(buf.len()); + let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; + buf.copy_from_slice(&self.rx_buffer[..amt]); + self.rx_buffer.advance(amt); + return Ok(amt); + } + match self.rx_recv.try_recv() { + Ok(data) => self.rx_buffer.extend_from_slice(&data), + Err(TryRecvError::Disconnected) => return Err(NetworkError::ConnectionAborted), + Err(TryRecvError::Empty) => return Err(NetworkError::WouldBlock), + } + } } fn set_promiscuous(&mut self, promiscuous: bool) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetPromiscuous(promiscuous)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn promiscuous(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetPromiscuous) { + ResponseType::Err(err) => Err(err), + ResponseType::Flag(val) => Ok(val), + _ => Err(NetworkError::IOError), + } } } impl VirtualConnectionlessSocket for RemoteSocket { fn try_send_to(&mut self, data: &[u8], addr: SocketAddr) -> Result { - todo!() + let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); + self.common + .tx + .try_send(MessageRequest::SendTo { + socket: self.socket_id, + data: data.to_vec(), + addr, + req_id, + }) + .map(|_| data.len()) } fn try_recv_from( &mut self, buf: &mut [std::mem::MaybeUninit], ) -> Result<(usize, SocketAddr)> { - todo!() + match self.rx_recv_with_addr.try_recv() { + Ok((data, addr)) => { + let amt = buf.len().min(data.len()); + let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; + buf.copy_from_slice(&data[..amt]); + Ok((amt, addr)) + } + Err(TryRecvError::Disconnected) => Err(NetworkError::ConnectionAborted), + Err(TryRecvError::Empty) => Err(NetworkError::WouldBlock), + } } } impl VirtualUdpSocket for RemoteSocket { fn set_broadcast(&mut self, broadcast: bool) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetBroadcast(broadcast)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn broadcast(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetBroadcast) { + ResponseType::Err(err) => Err(err), + ResponseType::Flag(val) => Ok(val), + _ => Err(NetworkError::IOError), + } } fn set_multicast_loop_v4(&mut self, val: bool) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetMulticastLoopV4(val)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn multicast_loop_v4(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetMulticastLoopV4) { + ResponseType::Err(err) => Err(err), + ResponseType::Flag(val) => Ok(val), + _ => Err(NetworkError::IOError), + } } fn set_multicast_loop_v6(&mut self, val: bool) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetMulticastLoopV6(val)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn multicast_loop_v6(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetMulticastLoopV6) { + ResponseType::Err(err) => Err(err), + ResponseType::Flag(val) => Ok(val), + _ => Err(NetworkError::IOError), + } } fn set_multicast_ttl_v4(&mut self, ttl: u32) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetMulticastTtlV4(ttl)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn multicast_ttl_v4(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetMulticastTtlV4) { + ResponseType::Err(err) => Err(err), + ResponseType::Ttl(ttl) => Ok(ttl), + _ => Err(NetworkError::IOError), + } } fn join_multicast_v4( @@ -605,7 +978,11 @@ impl VirtualUdpSocket for RemoteSocket { multiaddr: std::net::Ipv4Addr, iface: std::net::Ipv4Addr, ) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::JoinMulticastV4 { multiaddr, iface }) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn leave_multicast_v4( @@ -613,19 +990,36 @@ impl VirtualUdpSocket for RemoteSocket { multiaddr: std::net::Ipv4Addr, iface: std::net::Ipv4Addr, ) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::LeaveMulticastV4 { multiaddr, iface }) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn join_multicast_v6(&mut self, multiaddr: std::net::Ipv6Addr, iface: u32) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::JoinMulticastV6 { multiaddr, iface }) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn leave_multicast_v6(&mut self, multiaddr: std::net::Ipv6Addr, iface: u32) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::LeaveMulticastV6 { multiaddr, iface }) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn addr_peer(&self) -> Result> { - todo!() + match self.blocking_io_socket(RequestType::GetAddrPeer) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(None), + ResponseType::SocketAddr(addr) => Ok(Some(addr)), + _ => Err(NetworkError::IOError), + } } } @@ -633,64 +1027,142 @@ impl VirtualIcmpSocket for RemoteSocket {} impl VirtualConnectedSocket for RemoteSocket { fn set_linger(&mut self, linger: Option) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetLinger(linger)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn linger(&self) -> Result> { - todo!() + match self.blocking_io_socket(RequestType::GetLinger) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(None), + ResponseType::Duration(val) => Ok(Some(val)), + _ => Err(NetworkError::IOError), + } } fn try_send(&mut self, data: &[u8]) -> Result { - todo!() + let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); + self.common + .tx + .try_send(MessageRequest::Send { + socket: self.socket_id, + data: data.to_vec(), + req_id, + }) + .map(|_| data.len()) } fn try_flush(&mut self) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::Flush) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn close(&mut self) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::Close) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn try_recv(&mut self, buf: &mut [std::mem::MaybeUninit]) -> Result { - todo!() + loop { + if self.rx_buffer.len() > 0 { + let amt = self.rx_buffer.len().min(buf.len()); + let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; + buf.copy_from_slice(&self.rx_buffer[..amt]); + self.rx_buffer.advance(amt); + return Ok(amt); + } + match self.rx_recv.try_recv() { + Ok(data) => self.rx_buffer.extend_from_slice(&data), + Err(TryRecvError::Disconnected) => return Err(NetworkError::ConnectionAborted), + Err(TryRecvError::Empty) => return Err(NetworkError::WouldBlock), + } + } } } impl VirtualTcpSocket for RemoteSocket { fn set_recv_buf_size(&mut self, size: usize) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetRecvBufSize(size as u64)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn recv_buf_size(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetRecvBufSize) { + ResponseType::Err(err) => Err(err), + ResponseType::Amount(amt) => Ok(amt.try_into().map_err(|_| NetworkError::IOError)?), + _ => Err(NetworkError::IOError), + } } fn set_send_buf_size(&mut self, size: usize) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetSendBufSize(size as u64)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn send_buf_size(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetSendBufSize) { + ResponseType::Err(err) => Err(err), + ResponseType::Amount(val) => Ok(val.try_into().map_err(|_| NetworkError::IOError)?), + _ => Err(NetworkError::IOError), + } } fn set_nodelay(&mut self, reuse: bool) -> Result<()> { - todo!() + match self.blocking_io_socket(RequestType::SetNoDelay(reuse)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn nodelay(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetNoDelay) { + ResponseType::Err(err) => Err(err), + ResponseType::Flag(val) => Ok(val), + _ => Err(NetworkError::IOError), + } } fn addr_peer(&self) -> Result { - todo!() + match self.blocking_io_socket(RequestType::GetAddrPeer) { + ResponseType::Err(err) => Err(err), + ResponseType::SocketAddr(addr) => Ok(addr), + _ => Err(NetworkError::IOError), + } } fn shutdown(&mut self, how: std::net::Shutdown) -> Result<()> { - todo!() + let shutdown = match how { + std::net::Shutdown::Read => meta::Shutdown::Read, + std::net::Shutdown::Write => meta::Shutdown::Write, + std::net::Shutdown::Both => meta::Shutdown::Both, + }; + match self.blocking_io_socket(RequestType::Shutdown(shutdown)) { + ResponseType::Err(err) => Err(err), + ResponseType::None => Ok(()), + _ => Err(NetworkError::IOError), + } } fn is_closed(&self) -> bool { - todo!() + match self.blocking_io_socket(RequestType::IsClosed) { + ResponseType::Flag(val) => val, + _ => false, + } } } diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index 1291b617ce1..d6288716ff7 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -12,7 +12,7 @@ pub use super::SocketStatus; pub use super::StreamSecurity; /// Represents a socket ID -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct SocketId(u64); impl From for SocketId { @@ -21,10 +21,6 @@ impl From for SocketId { } } -/// Represents a handler ID -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] -pub struct HandlerId(u64); - /// Possible values which can be passed to the [`TcpStream::shutdown`] method. #[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] pub enum Shutdown { @@ -38,11 +34,6 @@ pub enum Shutdown { #[derive(Clone, Debug, Serialize, Deserialize)] pub enum RequestType { - /// Registers a waker for when a new connection has arrived. This uses - /// a stack machine which means more than one waker can be registered - SetHandler(HandlerId), - /// Removes a previously registered waker using a token - RemoveHandler, /// Bridges this local network with a remote network, which is required in /// order to make lower level networking calls (such as UDP/TCP) Bridge { @@ -50,6 +41,8 @@ pub enum RequestType { access_token: String, security: StreamSecurity, }, + /// Flushes all the data by ensuring a full round trip is completed + Flush, /// Disconnects from the remote network essentially unbridging it Unbridge, /// Acquires an IP address on the network and configures the routing tables @@ -116,17 +109,6 @@ pub enum RequestType { port: Option, dns_server: Option, }, - /// Tries to send out a datagram or stream of bytes on this socket - TrySend(Vec), - /// Sends out a datagram or stream of bytes on this socket - /// to a specific address - TrySendTo { data: Vec, addr: SocketAddr }, - /// Tries to flush any data in the local buffers - TryFlush, - /// Tries to read a packet from the socket - TryRecv { max_bytes: u64 }, - /// Recv a packet from the socket - TryRecvFrom { max_bytes: u64 }, /// Closes the socket Close, /// Tries to accept a new connection @@ -243,10 +225,6 @@ pub enum ResponseType { Duration(Duration), /// Represents an amount (e.g. amount of bytes) Amount(u64), - /// Represents a piece of data - Data(Vec), - /// Represents a peice of data and a socket address - DataWithAddr { data: Vec, addr: SocketAddr }, /// Returns a flag of true or false Flag(bool), /// List of IP addresses @@ -290,11 +268,41 @@ pub enum MessageRequest { req: RequestType, req_id: u64, }, + Send { + socket: SocketId, + data: Vec, + req_id: u64, + }, + SendTo { + socket: SocketId, + data: Vec, + addr: SocketAddr, + req_id: u64, + }, } /// Message sent by the server back to a client #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct MessageResponse { - pub req_id: u64, - pub res: ResponseType, +pub enum MessageResponse { + ResponseToRequest { + req_id: u64, + res: ResponseType, + }, + Recv { + socket_id: SocketId, + data: Vec, + }, + RecvWithAddr { + socket_id: SocketId, + data: Vec, + addr: SocketAddr, + }, + Sent { + socket_id: SocketId, + req_id: u64, + amount: u64, + }, + Closed { + socket_id: SocketId, + }, } From 0ffa79bacd7216dd8a201072a5725982b6dd99d3 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 10 Jul 2023 13:00:56 +1000 Subject: [PATCH 07/52] Completed the server side of the remote networking --- Cargo.lock | 2 + lib/virtual-net/Cargo.toml | 2 + lib/virtual-net/src/client.rs | 738 ++++++++++++------ lib/virtual-net/src/lib.rs | 2 + lib/virtual-net/src/meta.rs | 12 +- lib/virtual-net/src/server.rs | 1316 +++++++++++++++++++++++++++++++++ 6 files changed, 1840 insertions(+), 232 deletions(-) create mode 100644 lib/virtual-net/src/server.rs diff --git a/Cargo.lock b/Cargo.lock index 17e556fb54c..3056d2df481 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5155,8 +5155,10 @@ dependencies = [ "bytes", "ctr", "derivative", + "futures-util", "libc", "mio", + "pin-project-lite", "rand_chacha", "serde", "sha3", diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index 1b3e3424291..c88ac522d5a 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -30,6 +30,8 @@ rand_chacha = { version = "^0.3", optional = true } blake3 = { version = "1.3.3", optional = true } bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } +pin-project-lite = "0.2.9" +futures-util = { version = "0.3" } [features] host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "socket2", "mio" ] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index bbd0d1a6246..fb3d385998a 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -3,6 +3,7 @@ use std::future::Future; use std::mem::MaybeUninit; use std::net::IpAddr; use std::net::SocketAddr; +use std::ops::DerefMut; use std::pin::Pin; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; @@ -10,11 +11,18 @@ use std::sync::Arc; use std::sync::Mutex; use std::task::Context; use std::task::Poll; +use std::task::RawWaker; +use std::task::RawWakerVTable; +use std::task::Waker; use std::time::Duration; use bytes::Buf; use bytes::BytesMut; use derivative::Derivative; +use futures_util::future::BoxFuture; +use futures_util::stream::FuturesOrdered; +use futures_util::StreamExt; +use serde::Serialize; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; use tokio::io::AsyncWriteExt; @@ -22,6 +30,7 @@ use tokio::io::ReadBuf; use tokio::sync::mpsc; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::mpsc::error::TrySendError; +use tokio::sync::oneshot; use virtual_io::InlineWaker; use virtual_io::InterestType; @@ -30,7 +39,7 @@ use crate::meta; use crate::meta::RequestType; use crate::meta::ResponseType; use crate::meta::SocketId; -use crate::meta::{InterfaceId, MessageRequest, MessageResponse}; +use crate::meta::{MessageRequest, MessageResponse}; use crate::IpCidr; use crate::IpRoute; use crate::NetworkError; @@ -48,18 +57,299 @@ use crate::VirtualUdpSocket; use crate::Result; -enum RemoteTx { - Mpsc(mpsc::Sender), - Stream(tokio::sync::Mutex>>), +#[derive(Debug, Clone)] +pub struct RemoteNetworking { + common: Arc, +} + +impl RemoteNetworking { + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_mpsc( + tx: mpsc::Sender, + rx: mpsc::Receiver, + ) -> (Self, RemoteNetworkingDriver) { + let (_, rx_work) = mpsc::unbounded_channel(); + + let common = RemoteCommon { + tx: RemoteTx::Mpsc(tx), + rx: Mutex::new(RemoteRx::Mpsc(rx)), + request_seed: AtomicU64::new(1), + requests: Default::default(), + socket_seed: AtomicU64::new(1), + recv_tx: Default::default(), + recv_with_addr_tx: Default::default(), + handlers: Default::default(), + stall: Default::default(), + }; + let common = Arc::new(common); + + let driver = RemoteNetworkingDriver { + more_work: rx_work, + tasks: Default::default(), + common: common.clone(), + }; + let networking = Self { common }; + + (networking, driver) + } + + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_stream( + tx: Pin>, + rx: Pin>, + ) -> (Self, RemoteNetworkingDriver) { + Self::new_from_stream_internal(tx, rx, false) + } + + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + /// + /// This version will run the async read and write operations + /// only the driver (this is needed for mixed runtimes) + pub fn new_from_stream_via_driver( + tx: Pin>, + rx: Pin>, + ) -> (Self, RemoteNetworkingDriver) { + Self::new_from_stream_internal(tx, rx, true) + } + + fn new_from_stream_internal( + tx: Pin>, + rx: Pin>, + via_driver: bool, + ) -> (Self, RemoteNetworkingDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + + let common = RemoteCommon { + tx: if via_driver { + RemoteTx::StreamViaDriver { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + } + } else { + RemoteTx::Stream { + tx: tokio::sync::Mutex::new(tx), + } + }, + rx: Mutex::new(RemoteRx::Stream { + rx, + next: None, + buf: BytesMut::new(), + }), + request_seed: AtomicU64::new(1), + requests: Default::default(), + socket_seed: AtomicU64::new(1), + recv_tx: Default::default(), + recv_with_addr_tx: Default::default(), + handlers: Default::default(), + stall: Default::default(), + }; + let common = Arc::new(common); + + let driver = RemoteNetworkingDriver { + more_work: rx_work, + tasks: Default::default(), + common: common.clone(), + }; + let networking = Self { common }; + + (networking, driver) + } + + fn new_socket(&self, id: SocketId) -> RemoteSocket { + let (tx, rx_recv) = tokio::sync::mpsc::channel(100); + self.common.recv_tx.lock().unwrap().insert(id, tx); + + let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); + self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); + + RemoteSocket { + socket_id: id, + common: self.common.clone(), + rx_buffer: BytesMut::new(), + rx_recv, + rx_recv_with_addr, + tx_waker: TxWaker::new(&self.common).as_waker(), + } + } +} + +pin_project_lite::pin_project! { + pub struct RemoteNetworkingDriver { + common: Arc, + more_work: mpsc::UnboundedReceiver>, + #[pin] + tasks: FuturesOrdered>, + } +} + +impl Future for RemoteNetworkingDriver { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // This guard will be held while the pipeline is not currently + // stalled by some back pressure. It is only acquired when there + // is background tasks being processed + let mut not_stalled_guard = None; + + // We loop until the waker is registered with the receiving stream + // and all the background tasks + loop { + // Background tasks are sent to this driver in certain circumstances + while let Poll::Ready(Some(work)) = Pin::new(&mut self.more_work).poll_recv(cx) { + self.tasks.push_back(work); + } + + // Background work basically stalls the stream until its all processed + // which makes the back pressure system work properly + match self.tasks.poll_next_unpin(cx) { + Poll::Ready(Some(_)) => continue, + Poll::Ready(None) => { + not_stalled_guard.take(); + } + Poll::Pending if not_stalled_guard.is_none() => { + if let Ok(guard) = self.common.stall.clone().try_lock_owned() { + not_stalled_guard.replace(guard); + } else { + return Poll::Pending; + } + } + Poll::Pending => {} + }; + + // We grab the next message sent by the server to us + let msg = { + let mut rx_guard = self.common.rx.lock().unwrap(); + rx_guard.poll(cx) + }; + return match msg { + Poll::Ready(Some(msg)) => { + match msg { + MessageResponse::Recv { socket_id, data } => { + let tx = { + let guard = self.common.recv_tx.lock().unwrap(); + match guard.get(&socket_id) { + Some(tx) => tx.clone(), + None => continue, + } + }; + let common = self.common.clone(); + self.tasks.push_back(Box::pin(async move { + tx.send(data).await.ok(); + + common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Readable)); + })); + } + MessageResponse::RecvWithAddr { + socket_id, + data, + addr, + } => { + let tx = { + let guard = self.common.recv_with_addr_tx.lock().unwrap(); + match guard.get(&socket_id) { + Some(tx) => tx.clone(), + None => continue, + } + }; + let common = self.common.clone(); + self.tasks.push_back(Box::pin(async move { + tx.send((data, addr)).await.ok(); + + common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Readable)); + })); + } + MessageResponse::Sent { socket_id, .. } => { + self.common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Writable)); + } + MessageResponse::SendError { + socket_id, error, .. + } => match &error { + NetworkError::ConnectionAborted + | NetworkError::ConnectionReset + | NetworkError::BrokenPipe => { + self.common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Closed)); + } + _ => { + self.common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Writable)); + } + }, + MessageResponse::Closed { socket_id } => { + self.common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Closed)); + } + MessageResponse::ResponseToRequest { req_id, res } => { + let mut requests = self.common.requests.lock().unwrap(); + if let Some(request) = requests.remove(&req_id) { + request.try_send(res).ok(); + } + } + } + continue; + } + Poll::Ready(None) => Poll::Ready(()), + Poll::Pending => Poll::Pending, + }; + } + } +} + +pub(crate) type RemoteTxStream = Pin>; +pub(crate) enum RemoteTx +where + T: Serialize, +{ + Mpsc(mpsc::Sender), + Stream { + tx: tokio::sync::Mutex, + }, + StreamViaDriver { + tx: Arc>, + work: mpsc::UnboundedSender>, + }, } -impl RemoteTx { - async fn send(&self, req: MessageRequest) -> Result<()> { +impl RemoteTx +where + T: Serialize + Send + Sync + 'static, +{ + pub(crate) async fn send(&self, req: T) -> Result<()> { match self { RemoteTx::Mpsc(tx) => tx .send(req) .await .map_err(|_| NetworkError::ConnectionAborted), - RemoteTx::Stream(tx) => { + RemoteTx::Stream { tx, .. } => { let mut tx = tx.lock().await; let data = bincode::serialize(&req).map_err(|err| { tracing::warn!("failed to serialize message - {}", err); @@ -72,30 +362,189 @@ impl RemoteTx { .map_err(io_err_into_net_error)?; tx.write_all(&data).await.map_err(io_err_into_net_error) } + RemoteTx::StreamViaDriver { tx, work, .. } => { + let (tx_done, rx_done) = oneshot::channel(); + let tx = tx.clone(); + work.send(Box::pin(async move { + let job = async { + let mut tx = tx.lock().await; + let data = bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {}", err); + NetworkError::IOError + })?; + let data_len = data.len() as u64; + let data_len_buf = data_len.to_le_bytes(); + tx.write_all(&data_len_buf) + .await + .map_err(io_err_into_net_error)?; + tx.write_all(&data).await.map_err(io_err_into_net_error) + }; + tx_done.send(job.await).ok(); + })) + .map_err(|_| NetworkError::ConnectionAborted)?; + + rx_done + .await + .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) + } } } - fn try_send(&self, req: MessageRequest) -> Result<()> { + pub(crate) fn try_send(&self, interest_cx: &mut Context<'_>, req: T) -> Result<()> { match self { RemoteTx::Mpsc(tx) => match tx.try_send(req) { Ok(()) => Ok(()), Err(TrySendError::Closed(_)) => Err(NetworkError::ConnectionAborted), Err(TrySendError::Full(_)) => Err(NetworkError::WouldBlock), }, - RemoteTx::Stream(_) => InlineWaker::block_on(self.send(req)), + RemoteTx::Stream { tx } => { + let data = bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {}", err); + NetworkError::IOError + })?; + let data_len = data.len() as u64; + let data_len_buf = data_len.to_le_bytes(); + + let mut tx = InlineWaker::block_on(tx.lock()); + let data_len_buf_left = + match Pin::new(tx.deref_mut()).poll_write(interest_cx, &data_len_buf) { + Poll::Ready(Ok(0)) => { + return Err(NetworkError::ConnectionAborted); + } + Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => 0, + Poll::Ready(Ok(amt)) => data_len_buf.len() - amt, + Poll::Ready(Err(err)) => { + return Err(io_err_into_net_error(err)); + } + Poll::Pending => { + return Err(NetworkError::WouldBlock); + } + }; + + InlineWaker::block_on(Box::pin(async move { + if data_len_buf_left > 0 { + let offset = data_len_buf.len() - data_len_buf_left; + tx.write_all(&data_len_buf[offset..]) + .await + .map_err(io_err_into_net_error)?; + } + tx.write_all(&data).await.map_err(io_err_into_net_error) + })) + } + RemoteTx::StreamViaDriver { tx, work } => { + let data = bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {}", err); + NetworkError::IOError + })?; + let data_len = data.len() as u64; + let data_len_buf = data_len.to_le_bytes(); + + let interest_waker = interest_cx.waker().clone(); + + let (tx_done, rx_done) = std::sync::mpsc::channel(); + let tx = tx.clone(); + work.send(Box::pin(async move { + let job = async { + let mut tx = tx.lock().await; + + let mut interest_cx = Context::from_waker(&interest_waker); + let data_len_buf_left = match Pin::new(tx.deref_mut()) + .poll_write(&mut interest_cx, &data_len_buf) + { + Poll::Ready(Ok(0)) => { + return Err(NetworkError::ConnectionAborted); + } + Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => 0, + Poll::Ready(Ok(amt)) => data_len_buf.len() - amt, + Poll::Ready(Err(err)) => { + return Err(io_err_into_net_error(err)); + } + Poll::Pending => { + return Err(NetworkError::WouldBlock); + } + }; + + if data_len_buf_left > 0 { + let offset = data_len_buf.len() - data_len_buf_left; + tx.write_all(&data_len_buf[offset..]) + .await + .map_err(io_err_into_net_error)?; + } + tx.write_all(&data).await.map_err(io_err_into_net_error) + }; + let ret = job.await; + tx_done.send(ret).ok(); + })) + .map_err(|_| NetworkError::ConnectionAborted)?; + + rx_done + .recv() + .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) + } + } + } +} + +#[derive(Debug)] +struct TxWaker { + common: Arc, +} +impl TxWaker { + pub fn new(common: &Arc) -> Arc { + Arc::new(Self { + common: common.clone(), + }) + } + + fn wake_now(&self) { + let mut guard = self.common.handlers.lock().unwrap(); + for (_, handler) in guard.iter_mut() { + handler.interest(InterestType::Writable); } } + + pub fn as_waker(self: &Arc) -> Waker { + let s: *const Self = Arc::into_raw(Arc::clone(self)); + let raw_waker = RawWaker::new(s as *const (), &VTABLE); + unsafe { Waker::from_raw(raw_waker) } + } +} + +fn tx_waker_wake(s: &TxWaker) { + let waker_arc = unsafe { Arc::from_raw(s) }; + waker_arc.wake_now(); +} + +fn tx_waker_clone(s: &TxWaker) -> RawWaker { + let arc = unsafe { Arc::from_raw(s) }; + std::mem::forget(arc.clone()); + RawWaker::new(Arc::into_raw(arc) as *const (), &VTABLE) } -enum RemoteRx { - Mpsc(mpsc::Receiver), +const VTABLE: RawWakerVTable = unsafe { + RawWakerVTable::new( + |s| tx_waker_clone(&*(s as *const TxWaker)), // clone + |s| tx_waker_wake(&*(s as *const TxWaker)), // wake + |s| (*(s as *const TxWaker)).wake_now(), // wake by ref (don't decrease refcount) + |s| drop(Arc::from_raw(s as *const TxWaker)), // decrease refcount + ) +}; + +pub(crate) enum RemoteRx +where + T: serde::de::DeserializeOwned, +{ + Mpsc(mpsc::Receiver), Stream { rx: Pin>, next: Option, buf: BytesMut, }, } -impl RemoteRx { - fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { +impl RemoteRx +where + T: serde::de::DeserializeOwned, +{ + pub(crate) fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { loop { return match self { RemoteRx::Mpsc(rx) => Pin::new(rx).poll_recv(cx), @@ -111,12 +560,18 @@ impl RemoteRx { } }; buf.advance(next); + if buf.is_empty() { + buf.clear(); + } return Poll::Ready(Some(msg)); } None if buf.len() >= 8 => { let mut data_len_buf = [0u8; 8]; data_len_buf.copy_from_slice(&buf[..8]); buf.advance(8); + if buf.is_empty() { + buf.clear(); + } next.replace(u64::from_le_bytes(data_len_buf)); continue; } @@ -167,11 +622,10 @@ impl RequestTx { #[derive(Derivative)] #[derivative(Debug)] struct RemoteCommon { - interface_id: InterfaceId, #[derivative(Debug = "ignore")] - tx: RemoteTx, + tx: RemoteTx, #[derivative(Debug = "ignore")] - rx: Mutex, + rx: Mutex>, request_seed: AtomicU64, requests: Mutex>, socket_seed: AtomicU64, @@ -179,6 +633,10 @@ struct RemoteCommon { recv_with_addr_tx: Mutex, SocketAddr)>>>, #[derivative(Debug = "ignore")] handlers: Mutex>>, + + // The stall guard will prevent reads while its held and there are background tasks running + // (the idea behind this is to create back pressure so that the task list infinitely grow) + stall: Arc>, } impl RemoteCommon { @@ -192,11 +650,7 @@ impl RemoteCommon { }; if let Err(err) = self .tx - .send(MessageRequest::Interface { - iface: self.interface_id, - req_id, - req, - }) + .send(MessageRequest::Interface { req_id, req }) .await { return ResponseType::Err(err); @@ -209,194 +663,6 @@ impl RemoteCommon { } } -#[derive(Debug)] -pub struct RemoteNetworking { - common: Arc, -} - -#[derive(Derivative)] -#[derivative(Debug)] -pub struct RemoteNetworkingDriver { - #[derivative(Debug = "ignore")] - polling: Option + Send + Sync>>>, - common: Arc, -} - -impl Future for RemoteNetworkingDriver { - type Output = (); - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - loop { - if let Some(polling) = self.polling.as_mut() { - match polling.as_mut().poll(cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(()) => {} - } - self.polling.take(); - } - let msg = { - let mut rx_guard = self.common.rx.lock().unwrap(); - rx_guard.poll(cx) - }; - return match msg { - Poll::Ready(Some(msg)) => { - match msg { - MessageResponse::Recv { socket_id, data } => { - let tx = { - let guard = self.common.recv_tx.lock().unwrap(); - match guard.get(&socket_id) { - Some(tx) => tx.clone(), - None => continue, - } - }; - let common = self.common.clone(); - self.polling.replace(Box::pin(async move { - tx.send(data).await.ok(); - - common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Readable)); - })); - } - MessageResponse::RecvWithAddr { - socket_id, - data, - addr, - } => { - let tx = { - let guard = self.common.recv_with_addr_tx.lock().unwrap(); - match guard.get(&socket_id) { - Some(tx) => tx.clone(), - None => continue, - } - }; - let common = self.common.clone(); - self.polling.replace(Box::pin(async move { - tx.send((data, addr)).await.ok(); - - common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Readable)); - })); - } - MessageResponse::Sent { socket_id, .. } => { - self.common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Writable)); - } - MessageResponse::Closed { socket_id } => { - self.common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Closed)); - } - MessageResponse::ResponseToRequest { req_id, res } => { - let mut requests = self.common.requests.lock().unwrap(); - if let Some(request) = requests.remove(&req_id) { - request.try_send(res).ok(); - } - } - } - continue; - } - Poll::Ready(None) => Poll::Ready(()), - Poll::Pending => Poll::Pending, - }; - } - } -} - -impl RemoteNetworking { - /// Creates a new interface on the remote location using - /// a unique interface ID and a pair of channels - pub fn new_from_mpsc( - id: InterfaceId, - tx: mpsc::Sender, - rx: mpsc::Receiver, - ) -> (Self, RemoteNetworkingDriver) { - let common = RemoteCommon { - interface_id: id, - tx: RemoteTx::Mpsc(tx), - rx: Mutex::new(RemoteRx::Mpsc(rx)), - request_seed: AtomicU64::new(1), - requests: Default::default(), - socket_seed: AtomicU64::new(1), - recv_tx: Default::default(), - recv_with_addr_tx: Default::default(), - handlers: Default::default(), - }; - let common = Arc::new(common); - - let driver = RemoteNetworkingDriver { - polling: None, - common: common.clone(), - }; - let networking = Self { common }; - - (networking, driver) - } - - /// Creates a new interface on the remote location using - /// a unique interface ID and a pair of channels - pub fn new_from_stream( - id: InterfaceId, - tx: Pin>, - rx: Pin>, - ) -> (Self, RemoteNetworkingDriver) { - let common = RemoteCommon { - interface_id: id, - tx: RemoteTx::Stream(tokio::sync::Mutex::new(tx)), - rx: Mutex::new(RemoteRx::Stream { - rx, - next: None, - buf: BytesMut::new(), - }), - request_seed: AtomicU64::new(1), - requests: Default::default(), - socket_seed: AtomicU64::new(1), - recv_tx: Default::default(), - recv_with_addr_tx: Default::default(), - handlers: Default::default(), - }; - let common = Arc::new(common); - - let driver = RemoteNetworkingDriver { - polling: None, - common: common.clone(), - }; - let networking = Self { common }; - - (networking, driver) - } - - fn new_socket(&self, id: SocketId) -> RemoteSocket { - let (tx, rx_recv) = tokio::sync::mpsc::channel(100); - self.common.recv_tx.lock().unwrap().insert(id, tx); - - let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); - self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); - - RemoteSocket { - socket_id: id, - common: self.common.clone(), - rx_buffer: BytesMut::new(), - rx_recv, - rx_recv_with_addr, - } - } -} - #[async_trait::async_trait] impl VirtualNetworking for RemoteNetworking { async fn bridge( @@ -673,6 +939,7 @@ struct RemoteSocket { rx_buffer: BytesMut, rx_recv: mpsc::Receiver>, rx_recv_with_addr: mpsc::Receiver<(Vec, SocketAddr)>, + tx_waker: Waker, } impl Drop for RemoteSocket { fn drop(&mut self) { @@ -768,7 +1035,12 @@ impl VirtualSocket for RemoteSocket { impl VirtualTcpListener for RemoteSocket { fn try_accept(&mut self) -> Result<(Box, SocketAddr)> { - match self.blocking_io_socket(RequestType::TryAccept) { + let child_id: SocketId = self + .common + .socket_seed + .fetch_add(1, Ordering::SeqCst) + .into(); + match self.blocking_io_socket(RequestType::TryAccept(child_id)) { ResponseType::Err(err) => Err(err), ResponseType::SocketWithAddr { id, addr } => { let (tx, rx_recv) = tokio::sync::mpsc::channel(100); @@ -783,6 +1055,7 @@ impl VirtualTcpListener for RemoteSocket { rx_buffer: BytesMut::new(), rx_recv, rx_recv_with_addr, + tx_waker: TxWaker::new(&self.common).as_waker(), }; Ok((Box::new(socket), addr)) } @@ -825,13 +1098,18 @@ impl VirtualTcpListener for RemoteSocket { impl VirtualRawSocket for RemoteSocket { fn try_send(&mut self, data: &[u8]) -> Result { let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); + + let mut cx = Context::from_waker(&self.tx_waker); self.common .tx - .try_send(MessageRequest::Send { - socket: self.socket_id, - data: data.to_vec(), - req_id, - }) + .try_send( + &mut cx, + MessageRequest::Send { + socket: self.socket_id, + data: data.to_vec(), + req_id, + }, + ) .map(|_| data.len()) } @@ -880,14 +1158,18 @@ impl VirtualRawSocket for RemoteSocket { impl VirtualConnectionlessSocket for RemoteSocket { fn try_send_to(&mut self, data: &[u8], addr: SocketAddr) -> Result { let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); + let mut cx = Context::from_waker(&self.tx_waker); self.common .tx - .try_send(MessageRequest::SendTo { - socket: self.socket_id, - data: data.to_vec(), - addr, - req_id, - }) + .try_send( + &mut cx, + MessageRequest::SendTo { + socket: self.socket_id, + data: data.to_vec(), + addr, + req_id, + }, + ) .map(|_| data.len()) } @@ -1045,13 +1327,17 @@ impl VirtualConnectedSocket for RemoteSocket { fn try_send(&mut self, data: &[u8]) -> Result { let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); + let mut cx = Context::from_waker(&self.tx_waker); self.common .tx - .try_send(MessageRequest::Send { - socket: self.socket_id, - data: data.to_vec(), - req_id, - }) + .try_send( + &mut cx, + MessageRequest::Send { + socket: self.socket_id, + data: data.to_vec(), + req_id, + }, + ) .map(|_| data.len()) } diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index b7497582603..504d71eb982 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -3,6 +3,8 @@ pub mod client; #[cfg(feature = "host-net")] pub mod host; pub mod meta; +#[cfg(any(feature = "ws-net", feature = "host-net"))] +pub mod server; #[cfg(feature = "host-net")] pub mod tun; diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index d6288716ff7..d32fd2d1b0b 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -112,7 +112,7 @@ pub enum RequestType { /// Closes the socket Close, /// Tries to accept a new connection - TryAccept, + TryAccept(SocketId), /// Returns the local address of this TCP listener GetAddrLocal, /// Returns the address (IP and Port) of the peer socket that this @@ -251,15 +251,10 @@ pub enum ResponseType { Status(SocketStatus), } -/// Represents an interface ID -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] -pub struct InterfaceId(u64); - /// Message sent by the client to the server #[derive(Clone, Debug, Serialize, Deserialize)] pub enum MessageRequest { Interface { - iface: InterfaceId, req: RequestType, req_id: u64, }, @@ -302,6 +297,11 @@ pub enum MessageResponse { req_id: u64, amount: u64, }, + SendError { + socket_id: SocketId, + req_id: u64, + error: NetworkError, + }, Closed { socket_id: SocketId, }, diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs new file mode 100644 index 00000000000..5f28f5a7a93 --- /dev/null +++ b/lib/virtual-net/src/server.rs @@ -0,0 +1,1316 @@ +use crate::meta::ResponseType; +use crate::{ + client::{RemoteRx, RemoteTx}, + meta::{MessageRequest, MessageResponse, RequestType, SocketId}, + VirtualNetworking, VirtualRawSocket, VirtualTcpListener, VirtualTcpSocket, VirtualUdpSocket, +}; +use crate::{NetworkError, VirtualIcmpSocket}; +use bytes::BytesMut; +use derivative::Derivative; +use futures_util::stream::FuturesOrdered; +use futures_util::{future::BoxFuture, StreamExt}; +use std::collections::HashSet; +use std::mem::MaybeUninit; +use std::task::Waker; +use std::{ + collections::HashMap, + future::Future, + net::SocketAddr, + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll}, +}; +use tokio::sync::OwnedMutexGuard; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::mpsc, +}; +use virtual_io::InterestHandler; + +type BackgroundTask = Option>; + +#[derive(Debug, Clone)] +pub struct RemoteNetworkingAdapter { + #[allow(dead_code)] + common: Arc, +} + +impl RemoteNetworkingAdapter { + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_mpsc( + tx: mpsc::Sender, + rx: mpsc::Receiver, + inner: Box, + ) -> (Self, RemoteNetworkingAdapterDriver) { + let (_, rx_work) = mpsc::unbounded_channel(); + + let common = RemoteAdapterCommon { + tx: RemoteTx::Mpsc(tx), + rx: Mutex::new(RemoteRx::Mpsc(rx)), + sockets: Default::default(), + handler: Default::default(), + stall_rx: Default::default(), + }; + let common = Arc::new(common); + + let driver = RemoteNetworkingAdapterDriver { + more_work: rx_work, + tasks: Default::default(), + common: common.clone(), + inner: Arc::new(tokio::sync::Mutex::new(inner)), + }; + let networking = Self { common }; + + (networking, driver) + } + + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_stream( + tx: Pin>, + rx: Pin>, + inner: Box, + ) -> (Self, RemoteNetworkingAdapterDriver) { + Self::new_from_stream_internal(tx, rx, inner, false) + } + + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_stream_via_driver( + tx: Pin>, + rx: Pin>, + inner: Box, + ) -> (Self, RemoteNetworkingAdapterDriver) { + Self::new_from_stream_internal(tx, rx, inner, true) + } + + fn new_from_stream_internal( + tx: Pin>, + rx: Pin>, + inner: Box, + via_driver: bool, + ) -> (Self, RemoteNetworkingAdapterDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + + let handler = RemoteAdapterHandler::default(); + let common = RemoteAdapterCommon { + tx: if via_driver { + RemoteTx::StreamViaDriver { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + } + } else { + RemoteTx::Stream { + tx: tokio::sync::Mutex::new(tx), + } + }, + rx: Mutex::new(RemoteRx::Stream { + rx, + next: None, + buf: BytesMut::new(), + }), + sockets: Default::default(), + handler, + stall_rx: Default::default(), + }; + let common = Arc::new(common); + + let driver = RemoteNetworkingAdapterDriver { + more_work: rx_work, + tasks: Default::default(), + common: common.clone(), + inner: Arc::new(tokio::sync::Mutex::new(inner)), + }; + let networking = Self { common }; + + (networking, driver) + } +} + +pin_project_lite::pin_project! { + pub struct RemoteNetworkingAdapterDriver { + common: Arc, + more_work: mpsc::UnboundedReceiver>, + #[pin] + tasks: FuturesOrdered>, + inner: Arc>>, + } +} + +impl Future for RemoteNetworkingAdapterDriver { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // We register the waker into the interest of the sockets so + // that it is woken when something is ready to read or write + let readable = { + let mut guard = self.common.handler.state.lock().unwrap(); + if guard.driver_wakers.iter().any(|w| w.will_wake(cx.waker())) == false { + guard.driver_wakers.push(cx.waker().clone()); + } + guard.readable.drain().collect() + }; + let readable: Vec<_> = readable; + + { + // When a socket is marked as readable then we should drain all the data + // from it and start sending it to the client + let common = self.common.clone(); + let mut guard = common.sockets.lock().unwrap(); + for socket_id in readable { + if let Some(task) = guard + .get_mut(&socket_id) + .map(|s| s.drain_reads(&common, socket_id)) + .unwrap_or(None) + { + self.tasks.push_back(task); + } + } + } + + // This guard will be held while the pipeline is not currently + // stalled by some back pressure. It is only acquired when there + // is background tasks being processed + let mut not_stalled_guard = None; + + // We loop until the waker is registered with the receiving stream + // and all the background tasks + loop { + // Background tasks are sent to this driver in certain circumstances + while let Poll::Ready(Some(work)) = Pin::new(&mut self.more_work).poll_recv(cx) { + self.tasks.push_back(work); + } + + // Background work basically stalls the stream until its all processed + // which creates back pressure on the client so that they don't overload + // the system + match self.tasks.poll_next_unpin(cx) { + Poll::Ready(Some(_)) => continue, + Poll::Ready(None) => { + not_stalled_guard.take(); + } + Poll::Pending if not_stalled_guard.is_none() => { + if let Ok(guard) = self.common.stall_rx.clone().try_lock_owned() { + not_stalled_guard.replace(guard); + } else { + return Poll::Pending; + } + } + Poll::Pending => {} + }; + + // We grab the next message sent by the client to us + let msg = { + let mut rx_guard = self.common.rx.lock().unwrap(); + rx_guard.poll(cx) + }; + return match msg { + Poll::Ready(Some(msg)) => { + if let Some(task) = self.process(msg) { + // With some messages we process there are background tasks that need to + // be further driver to completion by the driver + self.tasks.push_back(task) + }; + continue; + } + Poll::Ready(None) => Poll::Ready(()), + Poll::Pending => Poll::Pending, + }; + } + } +} + +impl RemoteNetworkingAdapterDriver { + fn process(&mut self, msg: MessageRequest) -> BackgroundTask { + match msg { + MessageRequest::Send { + socket, + data, + req_id, + } => self.process_send(socket, data, req_id), + MessageRequest::SendTo { + socket, + data, + addr, + req_id, + } => self.process_send_to(socket, data, addr, req_id), + MessageRequest::Interface { req, req_id } => self.process_interface(req, req_id), + MessageRequest::Socket { + socket, + req, + req_id, + } => self.process_socket(socket, req, req_id), + } + } + + fn process_send(&mut self, socket_id: SocketId, data: Vec, req_id: u64) -> BackgroundTask { + let mut guard = self.common.sockets.lock().unwrap(); + guard + .get_mut(&socket_id) + .map(|s| s.send(&self.common, socket_id, data, req_id)) + .unwrap_or(None) + } + + fn process_send_to( + &mut self, + socket_id: SocketId, + data: Vec, + addr: SocketAddr, + req_id: u64, + ) -> BackgroundTask { + let mut guard = self.common.sockets.lock().unwrap(); + guard + .get_mut(&socket_id) + .map(|s| s.send_to(&self.common, socket_id, data, addr, req_id)) + .unwrap_or(None) + } + + fn process_async(future: F) -> BackgroundTask + where + F: Future + Send + 'static, + { + Some(Box::pin(async move { + let background_task = future.await; + if let Some(background_task) = background_task { + background_task.await; + } + })) + } + + fn process_async_inner(&self, work: F, transmute: T, req_id: u64) -> BackgroundTask + where + F: FnOnce(OwnedMutexGuard>) -> Fut + + Send + + 'static, + Fut: Future + Send + 'static, + T: FnOnce(Fut::Output) -> ResponseType + Send + 'static, + { + let inner = self.inner.clone(); + let common = self.common.clone(); + Self::process_async(async move { + let inner = inner.lock_owned().await; + let future = work(inner); + let ret = future.await; + common.send(MessageResponse::ResponseToRequest { + req_id, + res: transmute(ret), + }) + }) + } + + fn process_async_noop(&self, work: F, req_id: u64) -> BackgroundTask + where + F: FnOnce(OwnedMutexGuard>) -> Fut + + Send + + 'static, + Fut: Future> + Send + 'static, + { + self.process_async_inner( + work, + move |ret| match ret { + Ok(()) => ResponseType::None, + Err(err) => ResponseType::Err(err), + }, + req_id, + ) + } + + fn process_async_socket( + &self, + work: F, + socket_id: SocketId, + req_id: u64, + ) -> BackgroundTask + where + F: FnOnce(OwnedMutexGuard>) -> Fut + + Send + + 'static, + Fut: Future> + Send + 'static, + { + let common = self.common.clone(); + self.process_async_inner( + work, + move |ret| match ret { + Ok(mut socket) => { + let handler = Box::new(common.handler.clone().for_socket(socket_id)); + + let err = match &mut socket { + RemoteAdapterSocket::TcpListener(s) => s.set_handler(handler), + RemoteAdapterSocket::TcpSocket(s) => s.set_handler(handler), + RemoteAdapterSocket::UdpSocket(s) => s.set_handler(handler), + RemoteAdapterSocket::IcmpSocket(s) => s.set_handler(handler), + RemoteAdapterSocket::RawSocket(s) => s.set_handler(handler), + }; + if let Err(err) = err { + return ResponseType::Err(err); + } + + let mut guard = common.sockets.lock().unwrap(); + guard.insert(socket_id, socket); + + ResponseType::Socket(socket_id) + } + Err(err) => ResponseType::Err(err), + }, + req_id, + ) + } + + fn process_inner( + &self, + work: F, + transmute: T, + socket_id: SocketId, + req_id: u64, + ) -> BackgroundTask + where + F: FnOnce(&mut RemoteAdapterSocket) -> R + Send + 'static, + T: FnOnce(R) -> ResponseType + Send + 'static, + { + let ret = { + let mut guard = self.common.sockets.lock().unwrap(); + let socket = match guard.get_mut(&socket_id) { + Some(s) => s, + None => { + return self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::InvalidFd), + }) + } + }; + work(socket) + }; + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: transmute(ret), + }) + } + + fn process_inner_noop(&self, work: F, socket_id: SocketId, req_id: u64) -> BackgroundTask + where + F: FnOnce(&mut RemoteAdapterSocket) -> Result<(), NetworkError> + Send + 'static, + { + self.process_inner( + work, + move |ret| match ret { + Ok(()) => ResponseType::None, + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ) + } + + fn process_inner_socket( + &self, + work: F, + socket_id: SocketId, + child_id: SocketId, + req_id: u64, + ) -> BackgroundTask + where + F: FnOnce( + &mut RemoteAdapterSocket, + ) -> Result<(RemoteAdapterSocket, SocketAddr), NetworkError> + + Send + + 'static, + { + let common = self.common.clone(); + self.process_inner( + work, + move |ret| match ret { + Ok((mut socket, addr)) => { + let handler = Box::new(common.handler.clone().for_socket(child_id)); + + let err = match &mut socket { + RemoteAdapterSocket::TcpListener(s) => s.set_handler(handler), + RemoteAdapterSocket::TcpSocket(s) => s.set_handler(handler), + RemoteAdapterSocket::UdpSocket(s) => s.set_handler(handler), + RemoteAdapterSocket::IcmpSocket(s) => s.set_handler(handler), + RemoteAdapterSocket::RawSocket(s) => s.set_handler(handler), + }; + if let Err(err) = err { + return ResponseType::Err(err); + } + + let mut guard = common.sockets.lock().unwrap(); + guard.insert(child_id, socket); + + ResponseType::SocketWithAddr { id: child_id, addr } + } + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ) + } + + fn process_interface(&mut self, req: RequestType, req_id: u64) -> BackgroundTask { + match req { + RequestType::Bridge { + network, + access_token, + security, + } => self.process_async_noop( + move |inner| async move { inner.bridge(&network, &access_token, security).await }, + req_id, + ), + RequestType::Unbridge => { + self.process_async_noop(move |inner| async move { inner.unbridge().await }, req_id) + } + RequestType::DhcpAcquire => self.process_async_inner( + move |inner| async move { inner.dhcp_acquire().await }, + |ret| match ret { + Ok(ips) => ResponseType::IpAddressList(ips), + Err(err) => ResponseType::Err(err), + }, + req_id, + ), + RequestType::IpAdd { ip, prefix } => self.process_async_noop( + move |inner: OwnedMutexGuard>| async move { + inner.ip_add(ip, prefix) + }, + req_id, + ), + RequestType::IpRemove(ip) => self.process_async_noop( + move |inner: OwnedMutexGuard>| async move { + inner.ip_remove(ip) + }, + req_id, + ), + RequestType::IpClear => self.process_async_noop( + move |inner: OwnedMutexGuard>| async move { + inner.ip_clear() + }, + req_id, + ), + RequestType::GetIpList => self.process_async_inner( + move |inner: OwnedMutexGuard>| async move { + inner.ip_list() + }, + |ret| match ret { + Ok(cidr) => ResponseType::CidrList(cidr), + Err(err) => ResponseType::Err(err), + }, + req_id, + ), + RequestType::GetMac => self.process_async_inner( + move |inner: OwnedMutexGuard>| async move { + inner.mac() + }, + |ret| match ret { + Ok(mac) => ResponseType::Mac(mac), + Err(err) => ResponseType::Err(err), + }, + req_id, + ), + RequestType::GatewaySet(ip) => self.process_async_noop( + move |inner: OwnedMutexGuard>| async move { + inner.gateway_set(ip) + }, + req_id, + ), + RequestType::RouteAdd { + cidr, + via_router, + preferred_until, + expires_at, + } => self.process_async_noop( + move |inner: OwnedMutexGuard>| async move { + inner.route_add(cidr, via_router, preferred_until, expires_at) + }, + req_id, + ), + RequestType::RouteRemove(ip) => self.process_async_noop( + move |inner: OwnedMutexGuard>| async move { + inner.route_remove(ip) + }, + req_id, + ), + RequestType::RouteClear => self.process_async_noop( + move |inner: OwnedMutexGuard>| async move { + inner.route_clear() + }, + req_id, + ), + RequestType::GetRouteList => self.process_async_inner( + move |inner: OwnedMutexGuard>| async move { + inner.route_list() + }, + |ret| match ret { + Ok(routes) => ResponseType::RouteList(routes), + Err(err) => ResponseType::Err(err), + }, + req_id, + ), + RequestType::BindRaw(socket_id) => self.process_async_socket( + move |inner: OwnedMutexGuard>| async move { + Ok(RemoteAdapterSocket::RawSocket(inner.bind_raw().await?)) + }, + socket_id, + req_id, + ), + RequestType::ListenTcp { + socket_id, + addr, + only_v6, + reuse_port, + reuse_addr, + } => self.process_async_socket( + move |inner: OwnedMutexGuard>| async move { + Ok(RemoteAdapterSocket::TcpListener( + inner + .listen_tcp(addr, only_v6, reuse_port, reuse_addr) + .await?, + )) + }, + socket_id, + req_id, + ), + RequestType::BindUdp { + socket_id, + addr, + reuse_port, + reuse_addr, + } => self.process_async_socket( + move |inner: OwnedMutexGuard>| async move { + Ok(RemoteAdapterSocket::UdpSocket( + inner.bind_udp(addr, reuse_port, reuse_addr).await?, + )) + }, + socket_id, + req_id, + ), + RequestType::BindIcmp { socket_id, addr } => self.process_async_socket( + move |inner: OwnedMutexGuard>| async move { + Ok(RemoteAdapterSocket::IcmpSocket( + inner.bind_icmp(addr).await?, + )) + }, + socket_id, + req_id, + ), + RequestType::ConnectTcp { + socket_id, + addr, + peer, + } => self.process_async_socket( + move |inner: OwnedMutexGuard>| async move { + Ok(RemoteAdapterSocket::TcpSocket( + inner.connect_tcp(addr, peer).await?, + )) + }, + socket_id, + req_id, + ), + RequestType::Resolve { + host, + port, + dns_server, + } => self.process_async_inner( + move |inner: OwnedMutexGuard>| async move { + inner.resolve(&host, port, dns_server).await + }, + |ret| match ret { + Ok(ips) => ResponseType::IpAddressList(ips), + Err(err) => ResponseType::Err(err), + }, + req_id, + ), + _ => self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::Unsupported), + }), + } + } + + fn process_socket( + &mut self, + socket_id: SocketId, + req: RequestType, + req_id: u64, + ) -> BackgroundTask { + match req { + RequestType::Flush => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.try_flush(), + RemoteAdapterSocket::RawSocket(s) => s.try_flush(), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::Close => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.close(), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::TryAccept(child_id) => self.process_inner_socket( + move |socket| match socket { + RemoteAdapterSocket::TcpListener(s) => match s.try_accept() { + Ok((socket, addr)) => Ok((RemoteAdapterSocket::TcpSocket(socket), addr)), + Err(err) => Err(err), + }, + _ => Err(NetworkError::Unsupported), + }, + socket_id, + child_id, + req_id, + ), + RequestType::GetAddrLocal => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.addr_local(), + RemoteAdapterSocket::TcpListener(s) => s.addr_local(), + RemoteAdapterSocket::UdpSocket(s) => s.addr_local(), + RemoteAdapterSocket::IcmpSocket(s) => s.addr_local(), + RemoteAdapterSocket::RawSocket(s) => s.addr_local(), + }, + |ret| match ret { + Ok(addr) => ResponseType::SocketAddr(addr), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::GetAddrPeer => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.addr_peer().map(Some), + RemoteAdapterSocket::TcpListener(_) => Err(NetworkError::Unsupported), + RemoteAdapterSocket::UdpSocket(s) => s.addr_peer(), + RemoteAdapterSocket::IcmpSocket(_) => Err(NetworkError::Unsupported), + RemoteAdapterSocket::RawSocket(_) => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(Some(addr)) => ResponseType::SocketAddr(addr), + Ok(None) => ResponseType::None, + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetTtl(ttl) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.set_ttl(ttl), + RemoteAdapterSocket::TcpListener(s) => { + s.set_ttl(ttl.try_into().unwrap_or_default()) + } + RemoteAdapterSocket::UdpSocket(s) => s.set_ttl(ttl), + RemoteAdapterSocket::IcmpSocket(s) => s.set_ttl(ttl), + RemoteAdapterSocket::RawSocket(s) => s.set_ttl(ttl), + }, + socket_id, + req_id, + ), + RequestType::GetTtl => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.ttl(), + RemoteAdapterSocket::TcpListener(s) => s.ttl().map(|t| t as u32), + RemoteAdapterSocket::UdpSocket(s) => s.ttl(), + RemoteAdapterSocket::IcmpSocket(s) => s.ttl(), + RemoteAdapterSocket::RawSocket(s) => s.ttl(), + }, + |ret| match ret { + Ok(ttl) => ResponseType::Ttl(ttl), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::GetStatus => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.status(), + RemoteAdapterSocket::TcpListener(_) => Err(NetworkError::Unsupported), + RemoteAdapterSocket::UdpSocket(s) => s.status(), + RemoteAdapterSocket::IcmpSocket(s) => s.status(), + RemoteAdapterSocket::RawSocket(s) => s.status(), + }, + |ret| match ret { + Ok(status) => ResponseType::Status(status), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetLinger(linger) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.set_linger(linger), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetLinger => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.linger(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(Some(time)) => ResponseType::Duration(time), + Ok(None) => ResponseType::None, + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetPromiscuous(promiscuous) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::RawSocket(s) => s.set_promiscuous(promiscuous), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetPromiscuous => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::RawSocket(s) => s.promiscuous(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(flag) => ResponseType::Flag(flag), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetRecvBufSize(size) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => { + s.set_recv_buf_size(size.try_into().unwrap_or_default()) + } + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetRecvBufSize => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.recv_buf_size(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(amt) => ResponseType::Amount(amt as u64), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetSendBufSize(size) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => { + s.set_send_buf_size(size.try_into().unwrap_or_default()) + } + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetSendBufSize => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.send_buf_size(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(amt) => ResponseType::Amount(amt as u64), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetNoDelay(reuse) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.set_nodelay(reuse), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetNoDelay => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.nodelay(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(flag) => ResponseType::Flag(flag), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::Shutdown(shutdown) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => s.shutdown(match shutdown { + crate::meta::Shutdown::Read => std::net::Shutdown::Read, + crate::meta::Shutdown::Write => std::net::Shutdown::Write, + crate::meta::Shutdown::Both => std::net::Shutdown::Both, + }), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::IsClosed => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::TcpSocket(s) => Ok(s.is_closed()), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(flag) => ResponseType::Flag(flag), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetBroadcast(broadcast) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.set_broadcast(broadcast), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetBroadcast => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.broadcast(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(flag) => ResponseType::Flag(flag), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetMulticastLoopV4(val) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.set_multicast_loop_v4(val), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetMulticastLoopV4 => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.multicast_loop_v4(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(flag) => ResponseType::Flag(flag), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetMulticastLoopV6(val) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.set_multicast_loop_v6(val), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetMulticastLoopV6 => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.multicast_loop_v6(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(flag) => ResponseType::Flag(flag), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::SetMulticastTtlV4(ttl) => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.set_multicast_ttl_v4(ttl), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::GetMulticastTtlV4 => self.process_inner( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.multicast_ttl_v4(), + _ => Err(NetworkError::Unsupported), + }, + |ret| match ret { + Ok(ttl) => ResponseType::Ttl(ttl), + Err(err) => ResponseType::Err(err), + }, + socket_id, + req_id, + ), + RequestType::JoinMulticastV4 { multiaddr, iface } => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.join_multicast_v4(multiaddr, iface), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::LeaveMulticastV4 { multiaddr, iface } => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.leave_multicast_v4(multiaddr, iface), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::JoinMulticastV6 { multiaddr, iface } => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.join_multicast_v6(multiaddr, iface), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + RequestType::LeaveMulticastV6 { multiaddr, iface } => self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::UdpSocket(s) => s.leave_multicast_v6(multiaddr, iface), + _ => Err(NetworkError::Unsupported), + }, + socket_id, + req_id, + ), + _ => self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::Unsupported), + }), + } + } +} + +enum RemoteAdapterSocket { + TcpListener(Box), + TcpSocket(Box), + UdpSocket(Box), + RawSocket(Box), + IcmpSocket(Box), +} + +impl RemoteAdapterSocket { + pub fn send( + &mut self, + common: &Arc, + socket_id: SocketId, + data: Vec, + req_id: u64, + ) -> BackgroundTask { + match self { + Self::TcpSocket(this) => match this.try_send(&data) { + Ok(amount) => common.send(MessageResponse::Sent { + socket_id, + req_id, + amount: amount as u64, + }), + Err(NetworkError::WouldBlock) => { + let common = common.clone(); + Some(Box::pin(async move { + // We will stall the receiver so that back pressure is sent back to the + // sender and they don't overwhelm us with transmitting data. + let _stall_rx = common.stall_rx.clone().lock_owned().await; + + // We use a poller here that uses the handler to wake itself up + struct Poller { + common: Arc, + socket_id: SocketId, + data: Vec, + req_id: u64, + } + impl Future for Poller { + type Output = BackgroundTask; + fn poll( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll { + // We make sure the waker is registered with the interest driver which will + // wake up this poller when there is writeability + let mut guard = self.common.handler.state.lock().unwrap(); + if guard.driver_wakers.iter().any(|w| w.will_wake(cx.waker())) + == false + { + guard.driver_wakers.push(cx.waker().clone()); + } + drop(guard); + + let mut guard = self.common.sockets.lock().unwrap(); + if let Some(socket) = guard.get_mut(&self.socket_id) { + if let RemoteAdapterSocket::TcpSocket(socket) = socket { + match socket.try_send(&self.data) { + Ok(amount) => { + return Poll::Ready(self.common.send( + MessageResponse::Sent { + socket_id: self.socket_id, + req_id: self.req_id, + amount: amount as u64, + }, + )) + } + Err(NetworkError::WouldBlock) => return Poll::Pending, + Err(error) => { + return Poll::Ready(self.common.send( + MessageResponse::SendError { + socket_id: self.socket_id, + req_id: self.req_id, + error, + }, + )) + } + } + } + } + Poll::Ready(None) + } + } + + // Run the poller until this message is sent, or the socket fails + let background_task = Poller { + common, + socket_id, + data, + req_id, + } + .await; + + // There might be more work left to finish off the send operation + if let Some(background_task) = background_task { + background_task.await; + } + })) + } + Err(error) => common.send(MessageResponse::SendError { + socket_id, + req_id, + error, + }), + }, + Self::RawSocket(this) => { + // when the RAW socket is overloaded we just silently drop the packet + // rather than buffering it and retrying later - Ethernet packets are + // not lossless. In reality most socket drivers under this remote socket + // will always succeed on `try_send` with RawSockets as they are always + // processed. + this.try_send(&data).ok(); + None + } + _ => common.send(MessageResponse::SendError { + socket_id, + req_id, + error: NetworkError::Unsupported, + }), + } + } + pub fn send_to( + &mut self, + common: &Arc, + socket_id: SocketId, + data: Vec, + addr: SocketAddr, + req_id: u64, + ) -> BackgroundTask { + match self { + Self::UdpSocket(this) => { + // when the UDP socket is overloaded we just silently drop the packet + // rather than buffering it and retrying later + this.try_send_to(&data, addr).ok(); + None + } + + Self::IcmpSocket(this) => { + // when the ICMP socket is overloaded we just silently drop the packet + // rather than buffering it and retrying later + this.try_send_to(&data, addr).ok(); + None + } + _ => common.send(MessageResponse::SendError { + socket_id, + req_id, + error: NetworkError::Unsupported, + }), + } + } + pub fn drain_reads( + &mut self, + common: &Arc, + socket_id: SocketId, + ) -> BackgroundTask { + // We loop reading the socket until all the pending reads are either + // being processed in a background task or they are empty + let mut ret: FuturesOrdered> = Default::default(); + loop { + break match self { + Self::TcpSocket(this) => { + let mut chunk: [MaybeUninit; 4096] = + unsafe { MaybeUninit::uninit().assume_init() }; + match this.try_recv(&mut chunk) { + Ok(0) => {} + Ok(amt) => { + let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..amt]; + let chunk_unsafe: &mut [u8] = + unsafe { std::mem::transmute(chunk_unsafe) }; + if let Some(task) = common.send(MessageResponse::Recv { + socket_id, + data: chunk_unsafe.to_vec(), + }) { + ret.push_back(task); + } + continue; + } + Err(_) => {} + } + } + Self::UdpSocket(this) => { + let mut chunk: [MaybeUninit; 4096] = + unsafe { MaybeUninit::uninit().assume_init() }; + match this.try_recv_from(&mut chunk) { + Ok((0, _)) => {} + Ok((amt, addr)) => { + let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..amt]; + let chunk_unsafe: &mut [u8] = + unsafe { std::mem::transmute(chunk_unsafe) }; + if let Some(task) = common.send(MessageResponse::RecvWithAddr { + socket_id, + data: chunk_unsafe.to_vec(), + addr, + }) { + ret.push_back(task); + } + } + Err(_) => {} + } + } + Self::IcmpSocket(this) => { + let mut chunk: [MaybeUninit; 4096] = + unsafe { MaybeUninit::uninit().assume_init() }; + match this.try_recv_from(&mut chunk) { + Ok((0, _)) => {} + Ok((amt, addr)) => { + let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..amt]; + let chunk_unsafe: &mut [u8] = + unsafe { std::mem::transmute(chunk_unsafe) }; + if let Some(task) = common.send(MessageResponse::RecvWithAddr { + socket_id, + data: chunk_unsafe.to_vec(), + addr, + }) { + ret.push_back(task); + } + } + Err(_) => {} + } + } + Self::RawSocket(this) => { + let mut chunk: [MaybeUninit; 4096] = + unsafe { MaybeUninit::uninit().assume_init() }; + match this.try_recv(&mut chunk) { + Ok(0) => {} + Ok(amt) => { + let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..amt]; + let chunk_unsafe: &mut [u8] = + unsafe { std::mem::transmute(chunk_unsafe) }; + if let Some(task) = common.send(MessageResponse::Recv { + socket_id, + data: chunk_unsafe.to_vec(), + }) { + ret.push_back(task); + } + } + Err(_) => {} + } + } + _ => {} + }; + } + + if ret.is_empty() { + // There is nothing to process so we are done + None + } else { + Some(Box::pin(async move { + // Processes all the background tasks until completion + let mut stream = ret; + loop { + let (next, s) = stream.into_future().await; + if next.is_none() { + break; + } + stream = s; + } + })) + } + } +} + +#[derive(Debug, Default)] +struct RemoteAdapterHandlerState { + readable: HashSet, + driver_wakers: Vec, +} + +#[derive(Debug, Clone)] +struct RemoteAdapterHandler { + socket_id: Option, + state: Arc>, +} +impl Default for RemoteAdapterHandler { + fn default() -> Self { + Self { + socket_id: None, + state: Default::default(), + } + } +} +impl RemoteAdapterHandler { + pub fn for_socket(self, id: SocketId) -> Self { + Self { + socket_id: Some(id), + state: self.state, + } + } +} +impl InterestHandler for RemoteAdapterHandler { + fn interest(&mut self, interest: virtual_io::InterestType) { + let mut guard = self.state.lock().unwrap(); + guard.driver_wakers.drain(..).for_each(|w| w.wake()); + let socket_id = match self.socket_id.clone() { + Some(s) => s, + None => return, + }; + match interest { + virtual_io::InterestType::Readable => { + guard.readable.insert(socket_id); + } + _ => {} + } + } +} + +#[derive(Derivative)] +#[derivative(Debug)] +struct RemoteAdapterCommon { + #[derivative(Debug = "ignore")] + tx: RemoteTx, + #[derivative(Debug = "ignore")] + rx: Mutex>, + #[derivative(Debug = "ignore")] + sockets: Mutex>, + handler: RemoteAdapterHandler, + + // The stall guard will prevent reads while its held and there are background tasks running + // (the idea behind this is to create back pressure so that the task list infinitely grow) + stall_rx: Arc>, +} +impl RemoteAdapterCommon { + fn send(self: &Arc, req: MessageResponse) -> BackgroundTask { + let this = self.clone(); + Some(Box::pin(async move { + if let Err(err) = this.tx.send(req).await { + tracing::debug!("failed to send message - {}", err); + } + })) + } +} From cfcd7b732b407b9ef595639aa5e068a01b5b21a3 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 12 Jul 2023 15:20:23 +1000 Subject: [PATCH 08/52] Changed the feature flags --- Cargo.lock | 44 -------------------------------------- lib/cli/Cargo.toml | 2 +- lib/virtual-net/Cargo.toml | 7 +----- lib/virtual-net/src/lib.rs | 4 ++-- lib/wasix/Cargo.toml | 2 +- 5 files changed, 5 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3056d2df481..5da4f4158a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,17 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" -dependencies = [ - "cfg-if 1.0.0", - "cipher", - "cpufeatures", -] - [[package]] name = "ahash" version = "0.7.6" @@ -1038,15 +1027,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher", -] - [[package]] name = "cty" version = "0.2.2" @@ -2305,15 +2285,6 @@ dependencies = [ "serde", ] -[[package]] -name = "keccak" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f6d5ed8676d904364de097082f4e7d240b571b67989ced0240f08b7f966f940" -dependencies = [ - "cpufeatures", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -4102,16 +4073,6 @@ dependencies = [ "digest", ] -[[package]] -name = "sha3" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" -dependencies = [ - "digest", - "keccak", -] - [[package]] name = "sharded-slab" version = "0.1.4" @@ -5147,21 +5108,16 @@ dependencies = [ name = "virtual-net" version = "0.3.0" dependencies = [ - "aes", "async-trait", "base64", "bincode", - "blake3", "bytes", - "ctr", "derivative", "futures-util", "libc", "mio", "pin-project-lite", - "rand_chacha", "serde", - "sha3", "socket2", "thiserror", "tokio", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 0fcfdb00111..65d68eed4a9 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -42,7 +42,7 @@ wasmer-types = { version = "=4.0.0", path = "../types", features = ["enable-serd wasmer-registry = { version = "5.2.0", path = "../registry", features = ["build-package", "clap"] } wasmer-object = { version = "=4.0.0", path = "../object", optional = true } virtual-fs = { version = "0.6.0", path = "../virtual-fs", default-features = false, features = ["host-fs"] } -virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["ws-net"] } +virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["remote"] } # Wasmer-owned dependencies. webc = { workspace = true } diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index c88ac522d5a..8742c407068 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -23,11 +23,6 @@ virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = fa #tun-tap = { version = "0.1.3", optional = true } #tokio-tungstenite = { version = "0.18.0", optional = true } base64 = "0.21" -aes = { version = "0.8", optional = true } -ctr = { version = "0.9.2", optional = true } -sha3 = { version = "0.10.8", optional = true } -rand_chacha = { version = "^0.3", optional = true } -blake3 = { version = "1.3.3", optional = true } bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } pin-project-lite = "0.2.9" @@ -35,4 +30,4 @@ futures-util = { version = "0.3" } [features] host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "socket2", "mio" ] -ws-net = [ "aes", "ctr", "sha3", "rand_chacha", "blake3" ] \ No newline at end of file +remote = [ ] diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index 504d71eb982..83aca8bd5ed 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -1,9 +1,9 @@ -#[cfg(any(feature = "ws-net", feature = "host-net"))] +#[cfg(any(feature = "remote", feature = "host-net"))] pub mod client; #[cfg(feature = "host-net")] pub mod host; pub mod meta; -#[cfg(any(feature = "ws-net", feature = "host-net"))] +#[cfg(any(feature = "remote", feature = "host-net"))] pub mod server; #[cfg(feature = "host-net")] pub mod tun; diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index cf63c3e2f30..853f149b502 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -133,7 +133,7 @@ host-vnet = ["virtual-net/host-net"] host-threads = [] host-reqwest = ["reqwest"] host-fs = ["virtual-fs/host-fs"] -ws-net = ["virtual-net/ws-net"] +remote-vnet = ["virtual-net/remote"] logging = ["tracing/log"] disable-all-logging = ["tracing/release_max_level_off", "tracing/max_level_off"] From b8eab947b4f021e955e30121f085953913e2ecae Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 17 Jul 2023 09:02:05 +1000 Subject: [PATCH 09/52] Updated cargo.lock --- lib/wasi-web/Cargo.lock | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 711fb10c50b..c22ef8bac97 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -2093,6 +2093,8 @@ dependencies = [ "async-trait", "bytes", "derivative", + "futures", + "serde", "thiserror", "tracing", ] @@ -2102,8 +2104,13 @@ name = "virtual-net" version = "0.3.0" dependencies = [ "async-trait", + "base64 0.21.2", + "bincode", "bytes", "derivative", + "futures-util", + "pin-project-lite", + "serde", "thiserror", "tracing", "virtual-io", From 00450347bd73a5144105d7da45ff986e833daf93 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 17 Jul 2023 09:10:20 +1000 Subject: [PATCH 10/52] Fixed the tokio dependencies on remote --- lib/virtual-net/Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index a1b49911af7..2e7fe21bd23 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -30,8 +30,9 @@ futures-util = { version = "0.3" } [features] host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "socket2", "mio" ] -remote = [ ] +remote = [ "tokio/io-util", "tokio/sync" ] +tun-tap = [ "remote", "host-net" ] [package.metadata.docs.rs] -features = ["host-net"] +features = ["host-net", "remote", "tun-tap"] rustc-args = ["--cfg", "docsrs"] From a1752704f8c2d08fb4f09898796c68838a6cc2cb Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 17 Jul 2023 11:32:44 +1000 Subject: [PATCH 11/52] Added the TunTap adapter --- Cargo.lock | 838 +++++++++++++++++++++++++++------- lib/cli/Cargo.toml | 2 +- lib/virtual-net/Cargo.toml | 15 +- lib/virtual-net/src/client.rs | 4 +- lib/virtual-net/src/host.rs | 54 +-- lib/virtual-net/src/lib.rs | 66 ++- lib/virtual-net/src/server.rs | 8 +- lib/virtual-net/src/tun.rs | 208 +++++++++ 8 files changed, 973 insertions(+), 222 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e4bdddef8f..305a15c151d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", "version_check", ] @@ -75,7 +75,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -214,7 +214,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -245,7 +245,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -262,7 +262,7 @@ checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", "cc", - "cfg-if", + "cfg-if 1.0.0", "libc", "miniz_oxide", "object 0.31.1", @@ -275,6 +275,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.2" @@ -332,7 +338,7 @@ dependencies = [ "arrayref", "arrayvec 0.7.4", "cc", - "cfg-if", + "cfg-if 1.0.0", "constant_time_eq", "digest", ] @@ -417,6 +423,16 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.4.0" @@ -500,6 +516,12 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -517,7 +539,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -592,6 +614,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -606,7 +637,7 @@ checksum = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" dependencies = [ "atty", "lazy_static", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -668,14 +699,14 @@ dependencies = [ "lazy_static", "libc", "log", - "miow", + "miow 0.3.7", "regex", "rustfix", "serde", "serde_derive", "serde_json", "tester", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -697,7 +728,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen", ] @@ -742,7 +773,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9847f90f32a50b0dcbd68bc23ff242798b13080b97b0569f6ed96a45ce4cf2cd" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.0", "libc", "scopeguard", "windows-sys 0.33.0", @@ -793,7 +824,7 @@ dependencies = [ "hashbrown 0.12.3", "log", "regalloc2", - "smallvec", + "smallvec 1.11.0", "target-lexicon 0.12.9", ] @@ -823,7 +854,7 @@ dependencies = [ "hashbrown 0.12.3", "indexmap 1.9.3", "log", - "smallvec", + "smallvec 1.11.0", ] [[package]] @@ -841,7 +872,7 @@ dependencies = [ "cranelift-codegen", "hashbrown 0.12.3", "log", - "smallvec", + "smallvec 1.11.0", "target-lexicon 0.12.9", ] @@ -866,7 +897,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -917,8 +948,19 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] @@ -927,9 +969,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-epoch 0.9.15", + "crossbeam-utils 0.8.16", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", ] [[package]] @@ -939,19 +996,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", - "cfg-if", - "crossbeam-utils", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.16", "memoffset 0.9.0", "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -963,11 +1042,11 @@ dependencies = [ "bitflags 1.3.2", "crossterm_winapi", "libc", - "mio", + "mio 0.8.8", "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -976,7 +1055,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1141,9 +1220,9 @@ version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown 0.14.0", - "lock_api", + "lock_api 0.4.10", "once_cell", "parking_lot_core 0.9.8", ] @@ -1220,7 +1299,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "dirs-sys-next", ] @@ -1232,7 +1311,7 @@ checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1243,7 +1322,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -1335,7 +1414,7 @@ version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -1462,7 +1541,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.2.16", "windows-sys 0.48.0", @@ -1529,12 +1608,34 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags 1.3.2", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.28" @@ -1658,7 +1759,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "libc", "wasi", @@ -1783,7 +1884,7 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "futures-core", "futures-sink", @@ -1791,7 +1892,7 @@ dependencies = [ "http", "indexmap 1.9.3", "slab", - "tokio", + "tokio 1.29.1", "tokio-util", "tracing", ] @@ -1912,7 +2013,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "itoa", ] @@ -1923,7 +2024,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes", + "bytes 1.4.0", "http", "pin-project-lite", ] @@ -1962,7 +2063,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -1974,7 +2075,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio", + "tokio 1.29.1", "tower-service", "tracing", "want", @@ -1990,7 +2091,7 @@ dependencies = [ "http", "hyper", "rustls", - "tokio", + "tokio 1.29.1", "tokio-rustls", ] @@ -2000,10 +2101,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes", + "bytes 1.4.0", "hyper", "native-tls", - "tokio", + "tokio 1.29.1", "tokio-native-tls", ] @@ -2179,7 +2280,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -2206,6 +2307,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" version = "2.8.0" @@ -2267,6 +2377,16 @@ dependencies = [ "serde", ] +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2287,7 +2407,7 @@ checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec 0.5.2", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "ryu", "static_assertions", ] @@ -2324,7 +2444,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d580318f95776505201b28cf98eb1fa5e4be3b689633ba6a3e6cd880ff22d8cb" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "windows-sys 0.48.0", ] @@ -2379,6 +2499,15 @@ dependencies = [ "semver 1.0.17", ] +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "lock_api" version = "0.4.10" @@ -2454,6 +2583,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + [[package]] name = "md5" version = "0.7.0" @@ -2484,6 +2619,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -2535,7 +2679,7 @@ checksum = "c66a1fdd7e946fe33fe9725012e25836bba3655769bee9ee347cce7de3f396df" dependencies = [ "cc", "dlib", - "futures", + "futures 0.3.28", "instant", "js-sys", "lazy_static", @@ -2549,7 +2693,7 @@ dependencies = [ "wayland-client", "wayland-cursor", "wayland-protocols", - "winapi", + "winapi 0.3.9", "x11-dl", ] @@ -2579,6 +2723,25 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow 0.2.2", + "net2", + "slab", + "winapi 0.2.8", +] + [[package]] name = "mio" version = "0.8.8" @@ -2591,13 +2754,36 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio 0.6.23", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + [[package]] name = "miow" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2624,6 +2810,17 @@ dependencies = [ "tempfile", ] +[[package]] +name = "net2" +version = "0.2.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + [[package]] name = "nix" version = "0.24.3" @@ -2631,7 +2828,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "libc", "memoffset 0.6.5", ] @@ -2644,7 +2841,7 @@ checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "libc", "memoffset 0.6.5", "pin-utils", @@ -2693,7 +2890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2799,7 +2996,7 @@ checksum = "6c62dcb6174f9cb326eac248f07e955d5d559c272730b6c03e396b443b562788" dependencies = [ "bstr 1.6.0", "normpath", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -2809,7 +3006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ "bitflags 1.3.2", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -2864,6 +3061,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.3", + "rustc_version 0.2.3", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2871,7 +3079,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api", + "lock_api 0.4.10", "parking_lot_core 0.8.6", ] @@ -2881,22 +3089,37 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api", + "lock_api 0.4.10", "parking_lot_core 0.9.8", ] +[[package]] +name = "parking_lot_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version 0.2.3", + "smallvec 0.6.14", + "winapi 0.3.9", +] + [[package]] name = "parking_lot_core" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall 0.2.16", - "smallvec", - "winapi", + "smallvec 1.11.0", + "winapi 0.3.9", ] [[package]] @@ -2905,10 +3128,10 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall 0.3.5", - "smallvec", + "smallvec 1.11.0", "windows-targets 0.48.1", ] @@ -3322,11 +3545,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", + "crossbeam-deque 0.8.3", + "crossbeam-utils 0.8.16", "num_cpus", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -3371,7 +3600,7 @@ dependencies = [ "fxhash", "log", "slice-group-by", - "smallvec", + "smallvec 1.11.0", ] [[package]] @@ -3427,7 +3656,7 @@ dependencies = [ "bitflags 1.3.2", "libc", "mach", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3452,8 +3681,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "async-compression", - "base64", - "bytes", + "base64 0.21.2", + "bytes 1.4.0", "encoding_rs", "futures-core", "futures-util", @@ -3477,7 +3706,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio", + "tokio 1.29.1", "tokio-native-tls", "tokio-rustls", "tokio-util", @@ -3503,7 +3732,7 @@ dependencies = [ "spin 0.5.2", "untrusted", "web-sys", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3543,7 +3772,7 @@ checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" dependencies = [ "libc", "rtoolbox", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3553,7 +3782,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -3567,7 +3796,7 @@ dependencies = [ "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "smallvec", + "smallvec 1.11.0", ] [[package]] @@ -3660,7 +3889,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64", + "base64 0.21.2", ] [[package]] @@ -3768,6 +3997,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3819,7 +4054,7 @@ version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3586be2cf6c0a8099a79a12b4084357aa9b3e0b0d7980e3b67aaf7a9d55f9f0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "version-compare", ] @@ -4054,6 +4289,17 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.0" @@ -4066,7 +4312,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", ] @@ -4086,7 +4332,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cf61602ee61e2f83dd016b3e6387245291cf728ea071c378b35088125b4d995" dependencies = [ - "bytes", + "bytes 1.4.0", "memmap2 0.6.2", ] @@ -4113,7 +4359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio", + "mio 0.8.8", "signal-hook", ] @@ -4153,6 +4399,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + [[package]] name = "smallvec" version = "1.11.0" @@ -4166,7 +4421,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4181,7 +4436,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api", + "lock_api 0.4.10", ] [[package]] @@ -4256,7 +4511,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1", + "sha1 0.6.1", "syn 1.0.109", ] @@ -4358,7 +4613,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", "rustix 0.37.23", @@ -4373,7 +4628,7 @@ checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ "dirs-next", "rustversion", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4383,7 +4638,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" dependencies = [ "libc", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4435,7 +4690,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e8bf7e0eb2dd7b4228cc1b6821fc5114cd6841ae59f652a85488c016091e5f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "getopts", "libc", "num_cpus", @@ -4477,7 +4732,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "once_cell", ] @@ -4493,7 +4748,7 @@ dependencies = [ "stdweb", "time-macros 0.1.1", "version_check", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -4585,6 +4840,30 @@ dependencies = [ "url", ] +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "mio 0.6.23", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + [[package]] name = "tokio" version = "1.29.1" @@ -4593,9 +4872,9 @@ checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", "backtrace", - "bytes", + "bytes 1.4.0", "libc", - "mio", + "mio 0.8.8", "num_cpus", "pin-project-lite", "socket2", @@ -4603,6 +4882,78 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "tokio-io", +] + +[[package]] +name = "tokio-core" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87b1395334443abca552f63d4f61d0486f12377c2ba8b368e523f89e828cffd4" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "log", + "mio 0.6.23", + "scoped-tls 0.1.2", + "tokio 0.1.22", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-timer", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.31", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.31", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -4621,7 +4972,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio", + "tokio 1.29.1", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "mio 0.6.23", + "num_cpus", + "parking_lot 0.9.0", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", ] [[package]] @@ -4631,7 +5001,105 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", - "tokio", + "tokio 1.29.1", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.31", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "mio 0.6.23", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque 0.7.4", + "crossbeam-queue", + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures 0.1.31", + "slab", + "tokio-executor", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" +dependencies = [ + "futures-util", + "log", + "tokio 1.29.1", + "tungstenite", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", + "mio 0.6.23", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "iovec", + "libc", + "log", + "mio 0.6.23", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", ] [[package]] @@ -4640,11 +5108,11 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-core", "futures-sink", "pin-project-lite", - "tokio", + "tokio 1.29.1", "tracing", ] @@ -4701,7 +5169,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio", + "tokio 1.29.1", "tower-layer", "tower-service", "tracing", @@ -4714,7 +5182,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ "bitflags 2.3.3", - "bytes", + "bytes 1.4.0", "futures-core", "futures-util", "http", @@ -4745,7 +5213,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -4808,7 +5276,7 @@ dependencies = [ "serde", "serde_json", "sharded-slab", - "smallvec", + "smallvec 1.11.0", "thread_local", "tracing", "tracing-core", @@ -4827,7 +5295,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec", + "smallvec 1.11.0", "thread_local", "tracing", "tracing-core", @@ -4866,6 +5334,38 @@ dependencies = [ "termcolor", ] +[[package]] +name = "tun-tap" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27165dce5c30856f7e11754ff8d3bb6fc965fba6f25f31ecbddc7c23f99e1615" +dependencies = [ + "cc", + "futures 0.1.31", + "libc", + "mio 0.6.23", + "tokio-core", +] + +[[package]] +name = "tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" +dependencies = [ + "base64 0.13.1", + "byteorder", + "bytes 1.4.0", + "http", + "httparse", + "log", + "rand", + "sha1 0.10.5", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.16.0" @@ -4995,6 +5495,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8parse" version = "0.2.1" @@ -5056,11 +5562,11 @@ version = "0.6.0" dependencies = [ "anyhow", "async-trait", - "bytes", + "bytes 1.4.0", "derivative", "filetime", "fs_extra", - "futures", + "futures 0.3.28", "getrandom", "indexmap 1.9.3", "lazy_static", @@ -5071,7 +5577,7 @@ dependencies = [ "slab", "tempfile", "thiserror", - "tokio", + "tokio 1.29.1", "tracing", "typetag", "webc", @@ -5082,10 +5588,10 @@ name = "virtual-io" version = "0.1.0" dependencies = [ "async-trait", - "bytes", + "bytes 1.4.0", "derivative", - "futures", - "mio", + "futures 0.3.28", + "mio 0.8.8", "serde", "socket2", "thiserror", @@ -5096,20 +5602,24 @@ dependencies = [ name = "virtual-net" version = "0.3.0" dependencies = [ + "anyhow", "async-trait", - "base64", + "base64 0.21.2", "bincode", - "bytes", + "bytes 1.4.0", "derivative", + "futures 0.3.28", "futures-util", "libc", - "mio", + "mio 0.8.8", "pin-project-lite", "serde", "socket2", "thiserror", - "tokio", + "tokio 1.29.1", + "tokio-tungstenite", "tracing", + "tun-tap", "virtual-io", ] @@ -5264,7 +5774,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d816eaf89496a0eefecb79f43463b07205701165c17790fa6f0eab5ae72bc39" dependencies = [ "anyhow", - "base64", + "base64 0.21.2", "flate2", "indexmap 1.9.3", "json5", @@ -5316,7 +5826,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -5364,7 +5874,7 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -5407,7 +5917,7 @@ checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" dependencies = [ "console_error_panic_hook", "js-sys", - "scoped-tls", + "scoped-tls 1.0.1", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", @@ -5507,8 +6017,8 @@ name = "wasmer" version = "4.0.0" dependencies = [ "anyhow", - "bytes", - "cfg-if", + "bytes 1.4.0", + "cfg-if 1.0.0", "derivative", "hashbrown 0.11.2", "indexmap 1.9.3", @@ -5536,7 +6046,7 @@ dependencies = [ "wasmparser 0.83.0", "wasmparser 0.95.0", "wat", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5547,7 +6057,7 @@ checksum = "e5f22cf70f93c4690d4e82b325634ad973cd3a02d5a6b6565d442b452ea729bf" dependencies = [ "anyhow", "cynic", - "futures", + "futures 0.3.28", "pin-project-lite", "reqwest", "serde", @@ -5580,7 +6090,7 @@ name = "wasmer-c-api" version = "4.0.0" dependencies = [ "cbindgen", - "cfg-if", + "cfg-if 1.0.0", "enumset", "field-offset", "inline-c", @@ -5588,7 +6098,7 @@ dependencies = [ "libc", "paste", "thiserror", - "tokio", + "tokio 1.29.1", "typetag", "virtual-fs", "wasmer", @@ -5646,10 +6156,10 @@ dependencies = [ "assert_cmd 2.0.11", "async-trait", "atty", - "bytes", + "bytes 1.4.0", "bytesize", "cargo_metadata", - "cfg-if", + "cfg-if 1.0.0", "chrono", "clap 4.3.11", "colored 2.0.4", @@ -5683,7 +6193,7 @@ dependencies = [ "tempfile", "thiserror", "tldextract", - "tokio", + "tokio 1.29.1", "toml 0.5.11", "tracing", "tracing-subscriber 0.3.17", @@ -5718,7 +6228,7 @@ name = "wasmer-compiler" version = "4.0.0" dependencies = [ "backtrace", - "cfg-if", + "cfg-if 1.0.0", "enum-iterator", "enumset", "hashbrown 0.11.2", @@ -5729,13 +6239,13 @@ dependencies = [ "region", "serde", "serde_bytes", - "smallvec", + "smallvec 1.11.0", "thiserror", "wasmer-object", "wasmer-types", "wasmer-vm", "wasmparser 0.95.0", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -5744,7 +6254,7 @@ version = "4.0.0" dependencies = [ "anyhow", "bytesize", - "cfg-if", + "cfg-if 1.0.0", "clap 4.3.11", "colored 2.0.4", "distance", @@ -5771,7 +6281,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec", + "smallvec 1.11.0", "target-lexicon 0.12.9", "tracing", "wasmer-compiler", @@ -5793,7 +6303,7 @@ dependencies = [ "regex", "rustc_version 0.4.0", "semver 1.0.17", - "smallvec", + "smallvec 1.11.0", "target-lexicon 0.12.9", "wasmer-compiler", "wasmer-types", @@ -5813,7 +6323,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec", + "smallvec 1.11.0", "target-lexicon 0.12.9", "wasmer-compiler", "wasmer-types", @@ -5830,7 +6340,7 @@ dependencies = [ "clap-verbosity-flag", "comfy-table", "dialoguer", - "futures", + "futures 0.3.28", "is-terminal", "log", "once_cell", @@ -5841,7 +6351,7 @@ dependencies = [ "serde_json", "serde_yaml 0.8.26", "time 0.3.23", - "tokio", + "tokio 1.29.1", "toml 0.7.6", "tracing", "tracing-subscriber 0.3.17", @@ -5948,7 +6458,7 @@ dependencies = [ "derivative", "dirs", "flate2", - "futures", + "futures 0.3.28", "hex", "insta", "md5", @@ -5963,7 +6473,7 @@ dependencies = [ "tar", "target-lexicon 0.12.9", "tempfile", - "tokio", + "tokio 1.29.1", ] [[package]] @@ -6021,7 +6531,7 @@ dependencies = [ "thiserror", "time 0.3.23", "tldextract", - "tokio", + "tokio 1.29.1", "toml 0.5.11", "url", "wasmer-toml", @@ -6063,7 +6573,7 @@ dependencies = [ "thiserror", "time 0.3.23", "tldextract", - "tokio", + "tokio 1.29.1", "toml 0.5.11", "url", "wasmer-toml", @@ -6126,7 +6636,7 @@ version = "4.0.0" dependencies = [ "backtrace", "cc", - "cfg-if", + "cfg-if 1.0.0", "corosensei", "dashmap", "derivative", @@ -6144,7 +6654,7 @@ dependencies = [ "thiserror", "tracing", "wasmer-types", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6154,13 +6664,13 @@ dependencies = [ "anyhow", "async-trait", "bincode", - "bytes", - "cfg-if", + "bytes 1.4.0", + "cfg-if 1.0.0", "chrono", "cooked-waker", "dashmap", "derivative", - "futures", + "futures 0.3.28", "getrandom", "heapless", "hex", @@ -6187,7 +6697,7 @@ dependencies = [ "term_size", "termios", "thiserror", - "tokio", + "tokio 1.29.1", "tower", "tower-http", "tracing", @@ -6212,7 +6722,7 @@ dependencies = [ "wcgi-host", "webc", "weezl", - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6237,7 +6747,7 @@ dependencies = [ "anyhow", "bitflags 1.3.2", "byteorder", - "cfg-if", + "cfg-if 1.0.0", "num_enum", "pretty_assertions", "serde", @@ -6282,11 +6792,11 @@ name = "wasmer-wast" version = "4.0.0" dependencies = [ "anyhow", - "futures", + "futures 0.3.28", "serde", "tempfile", "thiserror", - "tokio", + "tokio 1.29.1", "virtual-fs", "wasmer", "wasmer-wasix", @@ -6299,7 +6809,7 @@ version = "4.0.0" dependencies = [ "anyhow", "build-deps", - "cfg-if", + "cfg-if 1.0.0", "compiler-test-derive", "criterion", "glob", @@ -6309,7 +6819,7 @@ dependencies = [ "tempfile", "test-generator", "test-log", - "tokio", + "tokio 1.29.1", "tracing", "tracing-subscriber 0.3.17", "wasi-test-generator", @@ -6417,7 +6927,7 @@ dependencies = [ "downcast-rs", "libc", "nix 0.24.3", - "scoped-tls", + "scoped-tls 1.0.1", "wayland-commons", "wayland-scanner", "wayland-sys", @@ -6431,7 +6941,7 @@ checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" dependencies = [ "nix 0.24.3", "once_cell", - "smallvec", + "smallvec 1.11.0", "wayland-sys", ] @@ -6502,7 +7012,7 @@ dependencies = [ "http", "schemars", "serde", - "tokio", + "tokio 1.29.1", "wasmparser 0.95.0", "wcgi", ] @@ -6524,9 +7034,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02ba1b6e7ad252e691a86d727aa422fbc5ed95e9bca4ec8547869e9b5779f8f3" dependencies = [ "anyhow", - "base64", + "base64 0.21.2", "byteorder", - "bytes", + "bytes 1.4.0", "indexmap 1.9.3", "leb128", "lexical-sort", @@ -6578,6 +7088,12 @@ dependencies = [ "web-sys", ] +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + [[package]] name = "winapi" version = "0.3.9" @@ -6588,6 +7104,12 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -6600,7 +7122,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi", + "winapi 0.3.9", ] [[package]] @@ -6808,7 +7330,17 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi", + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", ] [[package]] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index ec29ffccd03..971f7fea9ff 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -60,7 +60,7 @@ wasmer-object = { version = "=4.0.0", path = "../object", optional = true } virtual-fs = { version = "0.6.0", path = "../virtual-fs", default-features = false, features = [ "host-fs", ] } -virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["remote"] } +virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["tun-tap"] } # Wasmer-owned dependencies. webc = { workspace = true } diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index 2e7fe21bd23..ddf3f80e12c 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -14,24 +14,27 @@ thiserror = "1" bytes = "1.1" async-trait = { version = "^0.1" } tracing = "0.1" -tokio = { version = "1", features = [ "net", "rt" ], default_features = false, optional = true } +tokio = { version = "1", default_features = false, optional = true } libc = { version = "0.2.139", optional = true } mio = { version = "0.8", optional = true } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } -#tun-tap = { version = "0.1.3", optional = true } -#tokio-tungstenite = { version = "0.18.0", optional = true } +tun-tap = { version = "0.1.3", features = [ "tokio" ], optional = true } +tokio-tungstenite = { version = "0.18.0", optional = true } base64 = "0.21" bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } pin-project-lite = "0.2.9" futures-util = { version = "0.3" } +anyhow = { version = "1.0", optional = true } +futures = { version = "0.3", optional = true } [features] -host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "socket2", "mio" ] -remote = [ "tokio/io-util", "tokio/sync" ] -tun-tap = [ "remote", "host-net" ] +default = [ "host-net", "remote", "tun-tap" ] +host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] +remote = [ "tokio", "libc", "tokio/io-util", "tokio/sync" ] +tun-tap = [ "remote", "host-net", "dep:tun-tap", "tokio-tungstenite", "anyhow", "futures" ] [package.metadata.docs.rs] features = ["host-net", "remote", "tun-tap"] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index fb3d385998a..5fa5dc98df3 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -34,7 +34,7 @@ use tokio::sync::oneshot; use virtual_io::InlineWaker; use virtual_io::InterestType; -use crate::host::io_err_into_net_error; +use crate::io_err_into_net_error; use crate::meta; use crate::meta::RequestType; use crate::meta::ResponseType; @@ -578,7 +578,7 @@ where _ => {} } - let mut chunk: [MaybeUninit; 4096] = + let mut chunk: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..]; let chunk_unsafe: &mut [u8] = unsafe { std::mem::transmute(chunk_unsafe) }; diff --git a/lib/virtual-net/src/host.rs b/lib/virtual-net/src/host.rs index ab82d34d6e7..16df4f209e5 100644 --- a/lib/virtual-net/src/host.rs +++ b/lib/virtual-net/src/host.rs @@ -1,5 +1,5 @@ #![allow(unused_variables)] -use crate::VirtualIoSource; +use crate::{io_err_into_net_error, VirtualIoSource}; #[allow(unused_imports)] use crate::{ IpCidr, IpRoute, NetworkError, Result, SocketStatus, StreamSecurity, VirtualConnectedSocket, @@ -482,55 +482,3 @@ impl VirtualIoSource for LocalUdpSocket { } } } - -pub fn io_err_into_net_error(net_error: std::io::Error) -> NetworkError { - use std::io::ErrorKind; - match net_error.kind() { - ErrorKind::BrokenPipe => NetworkError::BrokenPipe, - ErrorKind::AlreadyExists => NetworkError::AlreadyExists, - ErrorKind::AddrInUse => NetworkError::AddressInUse, - ErrorKind::AddrNotAvailable => NetworkError::AddressNotAvailable, - ErrorKind::ConnectionAborted => NetworkError::ConnectionAborted, - ErrorKind::ConnectionRefused => NetworkError::ConnectionRefused, - ErrorKind::ConnectionReset => NetworkError::ConnectionReset, - ErrorKind::Interrupted => NetworkError::Interrupted, - ErrorKind::InvalidData => NetworkError::InvalidData, - ErrorKind::InvalidInput => NetworkError::InvalidInput, - ErrorKind::NotConnected => NetworkError::NotConnected, - ErrorKind::PermissionDenied => NetworkError::PermissionDenied, - ErrorKind::TimedOut => NetworkError::TimedOut, - ErrorKind::UnexpectedEof => NetworkError::UnexpectedEof, - ErrorKind::WouldBlock => NetworkError::WouldBlock, - ErrorKind::WriteZero => NetworkError::WriteZero, - ErrorKind::Unsupported => NetworkError::Unsupported, - - #[cfg(target_family = "unix")] - _ => { - if let Some(code) = net_error.raw_os_error() { - match code { - libc::EPERM => NetworkError::PermissionDenied, - libc::EBADF => NetworkError::InvalidFd, - libc::ECHILD => NetworkError::InvalidFd, - libc::EMFILE => NetworkError::TooManyOpenFiles, - libc::EINTR => NetworkError::Interrupted, - libc::EIO => NetworkError::IOError, - libc::ENXIO => NetworkError::IOError, - libc::EAGAIN => NetworkError::WouldBlock, - libc::ENOMEM => NetworkError::InsufficientMemory, - libc::EACCES => NetworkError::PermissionDenied, - libc::ENODEV => NetworkError::NoDevice, - libc::EINVAL => NetworkError::InvalidInput, - libc::EPIPE => NetworkError::BrokenPipe, - err => { - tracing::trace!("unknown os error {}", err); - NetworkError::UnknownError - } - } - } else { - NetworkError::UnknownError - } - } - #[cfg(not(target_family = "unix"))] - _ => NetworkError::UnknownError, - } -} diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index 8a8adbcf0e7..d1333df7318 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -1,15 +1,20 @@ #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] -#[cfg(any(feature = "remote", feature = "host-net"))] +#[cfg(any(feature = "remote"))] pub mod client; #[cfg(feature = "host-net")] pub mod host; pub mod meta; -#[cfg(any(feature = "remote", feature = "host-net"))] +#[cfg(any(feature = "remote"))] pub mod server; -#[cfg(feature = "host-net")] +#[cfg(target_os = "linux")] +#[cfg(feature = "tun-tap")] pub mod tun; +#[cfg(any(feature = "remote"))] +pub use client::{RemoteNetworking, RemoteNetworkingDriver}; +#[cfg(any(feature = "remote"))] +pub use server::{RemoteNetworkingAdapter, RemoteNetworkingAdapterDriver}; use std::fmt; use std::mem::MaybeUninit; pub use std::net::IpAddr; @@ -20,6 +25,9 @@ pub use std::net::SocketAddr; use std::sync::Arc; pub use std::time::Duration; use thiserror::Error; +#[cfg(target_os = "linux")] +#[cfg(feature = "tun-tap")] +pub use tun::{TunTapDriver, TunTapSocket}; pub use bytes::Bytes; pub use bytes::BytesMut; @@ -493,3 +501,55 @@ pub enum NetworkError { #[error("unknown error found")] UnknownError, } + +pub fn io_err_into_net_error(net_error: std::io::Error) -> NetworkError { + use std::io::ErrorKind; + match net_error.kind() { + ErrorKind::BrokenPipe => NetworkError::BrokenPipe, + ErrorKind::AlreadyExists => NetworkError::AlreadyExists, + ErrorKind::AddrInUse => NetworkError::AddressInUse, + ErrorKind::AddrNotAvailable => NetworkError::AddressNotAvailable, + ErrorKind::ConnectionAborted => NetworkError::ConnectionAborted, + ErrorKind::ConnectionRefused => NetworkError::ConnectionRefused, + ErrorKind::ConnectionReset => NetworkError::ConnectionReset, + ErrorKind::Interrupted => NetworkError::Interrupted, + ErrorKind::InvalidData => NetworkError::InvalidData, + ErrorKind::InvalidInput => NetworkError::InvalidInput, + ErrorKind::NotConnected => NetworkError::NotConnected, + ErrorKind::PermissionDenied => NetworkError::PermissionDenied, + ErrorKind::TimedOut => NetworkError::TimedOut, + ErrorKind::UnexpectedEof => NetworkError::UnexpectedEof, + ErrorKind::WouldBlock => NetworkError::WouldBlock, + ErrorKind::WriteZero => NetworkError::WriteZero, + ErrorKind::Unsupported => NetworkError::Unsupported, + + #[cfg(all(target_family = "unix", feature = "libc"))] + _ => { + if let Some(code) = net_error.raw_os_error() { + match code { + libc::EPERM => NetworkError::PermissionDenied, + libc::EBADF => NetworkError::InvalidFd, + libc::ECHILD => NetworkError::InvalidFd, + libc::EMFILE => NetworkError::TooManyOpenFiles, + libc::EINTR => NetworkError::Interrupted, + libc::EIO => NetworkError::IOError, + libc::ENXIO => NetworkError::IOError, + libc::EAGAIN => NetworkError::WouldBlock, + libc::ENOMEM => NetworkError::InsufficientMemory, + libc::EACCES => NetworkError::PermissionDenied, + libc::ENODEV => NetworkError::NoDevice, + libc::EINVAL => NetworkError::InvalidInput, + libc::EPIPE => NetworkError::BrokenPipe, + err => { + tracing::trace!("unknown os error {}", err); + NetworkError::UnknownError + } + } + } else { + NetworkError::UnknownError + } + } + #[cfg(not(all(target_family = "unix", feature = "libc")))] + _ => NetworkError::UnknownError, + } +} diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index 5f28f5a7a93..416593be814 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -1144,7 +1144,7 @@ impl RemoteAdapterSocket { loop { break match self { Self::TcpSocket(this) => { - let mut chunk: [MaybeUninit; 4096] = + let mut chunk: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; match this.try_recv(&mut chunk) { Ok(0) => {} @@ -1164,7 +1164,7 @@ impl RemoteAdapterSocket { } } Self::UdpSocket(this) => { - let mut chunk: [MaybeUninit; 4096] = + let mut chunk: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; match this.try_recv_from(&mut chunk) { Ok((0, _)) => {} @@ -1184,7 +1184,7 @@ impl RemoteAdapterSocket { } } Self::IcmpSocket(this) => { - let mut chunk: [MaybeUninit; 4096] = + let mut chunk: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; match this.try_recv_from(&mut chunk) { Ok((0, _)) => {} @@ -1204,7 +1204,7 @@ impl RemoteAdapterSocket { } } Self::RawSocket(this) => { - let mut chunk: [MaybeUninit; 4096] = + let mut chunk: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; match this.try_recv(&mut chunk) { Ok(0) => {} diff --git a/lib/virtual-net/src/tun.rs b/lib/virtual-net/src/tun.rs index e69de29bb2d..ca5be97a731 100644 --- a/lib/virtual-net/src/tun.rs +++ b/lib/virtual-net/src/tun.rs @@ -0,0 +1,208 @@ +use std::collections::VecDeque; +use std::io::{self, ErrorKind}; +use std::mem::MaybeUninit; +use std::os::fd::{AsRawFd, RawFd}; +use std::pin::Pin; +use std::process::Command; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll, Waker}; + +use bytes::Bytes; +use futures_util::Future; +use mio::unix::SourceFd; +use mio::{event, Interest, Registry, Token}; +use tun_tap::{Iface, Mode}; +use virtual_io::{InterestGuard, InterestHandler, Selector}; + +use crate::{io_err_into_net_error, NetworkError, VirtualRawSocket}; + +fn cmd(cmd: &str, args: &[&str]) -> anyhow::Result<()> { + let ecode = Command::new(cmd).args(args).spawn()?.wait()?; + if ecode.success() { + Ok(()) + } else { + Err(anyhow::format_err!( + "failed to allocate IP address (code={:?})", + ecode.code() + )) + } +} + +pub struct TunTapSocket {} +impl TunTapSocket { + pub fn create( + selector: Arc, + mut client: Box, + ) -> anyhow::Result { + let iface = Iface::new("testtun%d", Mode::Tun)?; + cmd("ip", &["link", "set", "up", "dev", iface.name()])?; + + // Set non-blocking and wrap the MIO + set_non_blocking(&iface)?; + let handler = TunTapHandler::default(); + let mut source = MioWrapper { + fd: iface.as_raw_fd(), + }; + + // Register interest in the read and write events + let interest = InterestGuard::new( + &selector, + Box::new(handler.clone()), + &mut source, + mio::Interest::READABLE.add(mio::Interest::WRITABLE), + ) + .map_err(io_err_into_net_error)?; + + // Set the client handler + client.set_handler(Box::new(handler.clone()))?; + + let driver = TunTapDriver { + iface, + handler, + send_queue: Default::default(), + _interest: interest, + client, + }; + + Ok(driver) + } +} + +fn set_non_blocking(iface: &Iface) -> anyhow::Result<()> { + let fd = iface.as_raw_fd(); + let mut nonblock: libc::c_int = 1; + let result = unsafe { libc::ioctl(fd, libc::FIONBIO, &mut nonblock) }; + if result == -1 { + Err(std::io::Error::last_os_error().into()) + } else { + Ok(()) + } +} + +#[derive(Debug, Default)] +struct TunTapHandlerInner { + wakers: Vec, +} + +#[derive(Debug, Clone, Default)] +struct TunTapHandler { + inner: Arc>, +} +impl InterestHandler for TunTapHandler { + fn interest(&mut self, _interest: virtual_io::InterestType) { + let mut guard = self.inner.lock().unwrap(); + guard.wakers.drain(..).for_each(Waker::wake); + } +} + +pub struct TunTapDriver { + iface: Iface, + handler: TunTapHandler, + send_queue: VecDeque, + _interest: InterestGuard, + client: Box, +} + +impl Future for TunTapDriver { + type Output = io::Result<()>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // Add the waker before we drain all the the events + // we need to read and send + let inner = self.handler.inner.clone(); + let mut guard = inner.lock().unwrap(); + if guard.wakers.iter().any(|w| w.will_wake(cx.waker())) == false { + guard.wakers.push(cx.waker().clone()); + } + + // First we drain the packet + while let Some(packet) = self.send_queue.pop_front() { + if self.client.try_send(&packet).is_ok() == false { + self.send_queue.push_front(packet); + break; + } + } + + // Now we drain all the packets from the interface (but only if we have room) + if self.send_queue.is_empty() { + loop { + let mut chunk: [MaybeUninit; 10240] = + unsafe { MaybeUninit::uninit().assume_init() }; + let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..]; + let chunk_unsafe: &mut [u8] = unsafe { std::mem::transmute(chunk_unsafe) }; + + match self.iface.recv(chunk_unsafe) { + Ok(0) => return Poll::Ready(Err(ErrorKind::ConnectionReset.into())), + Ok(data) => { + // Send the packet down the client + let packet = &chunk_unsafe[..data]; + if self.client.try_send(packet).is_ok() == false { + self.send_queue.push_back(Bytes::copy_from_slice(packet)); + break; + } + continue; + } + Err(err) if err.kind() == ErrorKind::WouldBlock => { + break; + } + Err(err) => return Poll::Ready(Err(err)), + }; + } + } + + // Drain the client into the interfacee + loop { + let mut chunk: [MaybeUninit; 10240] = + unsafe { MaybeUninit::uninit().assume_init() }; + let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..]; + + match self.client.try_recv(chunk_unsafe) { + Ok(0) => return Poll::Ready(Err(ErrorKind::ConnectionReset.into())), + Ok(data) => { + // Send the packet down the client + let chunk_unsafe: &mut [u8] = unsafe { std::mem::transmute(chunk_unsafe) }; + let packet = &chunk_unsafe[..data]; + self.iface.send(packet).ok(); + continue; + } + Err(NetworkError::WouldBlock) => { + break; + } + Err(err) => { + tracing::error!("packet recv error - {}", err); + return Poll::Ready(Err(ErrorKind::Other.into())); + } + } + } + + // Wait for some more interest or something to send + Poll::Pending + } +} + +struct MioWrapper { + fd: RawFd, +} + +impl event::Source for MioWrapper { + fn register( + &mut self, + registry: &Registry, + token: Token, + interests: Interest, + ) -> io::Result<()> { + SourceFd(&self.fd).register(registry, token, interests) + } + + fn reregister( + &mut self, + registry: &Registry, + token: Token, + interests: Interest, + ) -> io::Result<()> { + SourceFd(&self.fd).reregister(registry, token, interests) + } + + fn deregister(&mut self, registry: &Registry) -> io::Result<()> { + SourceFd(&self.fd).deregister(registry) + } +} From 0bf4bfcb6242f8f23ec491f48f371422279ccbe9 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 17 Jul 2023 20:32:59 +1000 Subject: [PATCH 12/52] Checking in the code for now --- lib/virtual-net/src/host.rs | 2 +- lib/virtual-net/src/meta.rs | 8 ++++++++ lib/virtual-net/src/tun.rs | 23 ++++++++++++++++++----- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/lib/virtual-net/src/host.rs b/lib/virtual-net/src/host.rs index 16df4f209e5..2ad8f28472e 100644 --- a/lib/virtual-net/src/host.rs +++ b/lib/virtual-net/src/host.rs @@ -1,5 +1,5 @@ #![allow(unused_variables)] -use crate::{io_err_into_net_error, VirtualIoSource}; +use crate::{VirtualIoSource, io_err_into_net_error}; #[allow(unused_imports)] use crate::{ IpCidr, IpRoute, NetworkError, Result, SocketStatus, StreamSecurity, VirtualConnectedSocket, diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index d32fd2d1b0b..8ef8481921d 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -32,6 +32,14 @@ pub enum Shutdown { Both, } +#[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub struct Hello { + /// The network that we will connect to + pub network_hash: [u8; 16], + /// The encryption key that authorizes the connection + pub encrypt_key: [u8; 32], +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub enum RequestType { /// Bridges this local network with a remote network, which is required in diff --git a/lib/virtual-net/src/tun.rs b/lib/virtual-net/src/tun.rs index ca5be97a731..e78ff75ff1a 100644 --- a/lib/virtual-net/src/tun.rs +++ b/lib/virtual-net/src/tun.rs @@ -8,13 +8,15 @@ use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; use bytes::Bytes; -use futures_util::Future; +use futures_util::{Future, SinkExt, StreamExt}; use mio::unix::SourceFd; use mio::{event, Interest, Registry, Token}; +use tokio_tungstenite::tungstenite::Message; use tun_tap::{Iface, Mode}; use virtual_io::{InterestGuard, InterestHandler, Selector}; -use crate::{io_err_into_net_error, NetworkError, VirtualRawSocket}; +use crate::meta::Hello; +use crate::{io_err_into_net_error, NetworkError, RemoteNetworking, VirtualRawSocket}; fn cmd(cmd: &str, args: &[&str]) -> anyhow::Result<()> { let ecode = Command::new(cmd).args(args).spawn()?.wait()?; @@ -30,11 +32,22 @@ fn cmd(cmd: &str, args: &[&str]) -> anyhow::Result<()> { pub struct TunTapSocket {} impl TunTapSocket { - pub fn create( + pub async fn create( selector: Arc, - mut client: Box, + url: &str, + hello: Hello, ) -> anyhow::Result { - let iface = Iface::new("testtun%d", Mode::Tun)?; + // Create the remote client + let (stream, res) = tokio_tungstenite::connect_async(url).await?; + let (tx, rx) = stream.split(); + tx.send(Message::Binary(bincode::serialize(&hello)?.to_vec())) + .await?; + + // Now pass it on to a remote networking adapter + let (remote, remote_driver) = + RemoteNetworking::new_from_stream_via_driver(Box::pin(tx), Box::pin(rx)); + + let iface = Iface::new("wasmer%d", Mode::Tun)?; cmd("ip", &["link", "set", "up", "dev", iface.name()])?; // Set non-blocking and wrap the MIO From a9942a137d4c9ff870f6118671430bbbe3ee16e3 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 17 Jul 2023 22:12:17 +1000 Subject: [PATCH 13/52] Tuntap device now has a plumbed in remote networking --- lib/virtual-net/src/tun.rs | 93 ++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/lib/virtual-net/src/tun.rs b/lib/virtual-net/src/tun.rs index e78ff75ff1a..691b18c182b 100644 --- a/lib/virtual-net/src/tun.rs +++ b/lib/virtual-net/src/tun.rs @@ -1,6 +1,7 @@ use std::collections::VecDeque; use std::io::{self, ErrorKind}; use std::mem::MaybeUninit; +use std::ops::DerefMut; use std::os::fd::{AsRawFd, RawFd}; use std::pin::Pin; use std::process::Command; @@ -8,15 +9,20 @@ use std::sync::{Arc, Mutex}; use std::task::{Context, Poll, Waker}; use bytes::Bytes; -use futures_util::{Future, SinkExt, StreamExt}; +use futures_util::Future; use mio::unix::SourceFd; use mio::{event, Interest, Registry, Token}; -use tokio_tungstenite::tungstenite::Message; +use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; use tun_tap::{Iface, Mode}; use virtual_io::{InterestGuard, InterestHandler, Selector}; use crate::meta::Hello; -use crate::{io_err_into_net_error, NetworkError, RemoteNetworking, VirtualRawSocket}; +use crate::{ + io_err_into_net_error, NetworkError, RemoteNetworking, RemoteNetworkingDriver, + VirtualNetworking, VirtualRawSocket, +}; fn cmd(cmd: &str, args: &[&str]) -> anyhow::Result<()> { let ecode = Command::new(cmd).args(args).spawn()?.wait()?; @@ -30,6 +36,65 @@ fn cmd(cmd: &str, args: &[&str]) -> anyhow::Result<()> { } } +#[derive(Debug, Clone)] +struct TungstensiteAdapter { + stream: Arc>>>, +} +impl TungstensiteAdapter { + fn new(stream: WebSocketStream>) -> Self { + Self { + stream: Arc::new(Mutex::new(stream)), + } + } + fn split(self) -> (Self, Self) { + (self.clone(), self.clone()) + } +} +impl AsyncRead for TungstensiteAdapter { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + let mut guard = self.stream.lock().unwrap(); + Pin::new(guard.deref_mut().get_mut()).poll_read(cx, buf) + } +} +impl AsyncWrite for TungstensiteAdapter { + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[io::IoSlice<'_>], + ) -> Poll> { + let mut guard = self.stream.lock().unwrap(); + Pin::new(guard.deref_mut().get_mut()).poll_write_vectored(cx, bufs) + } + + fn is_write_vectored(&self) -> bool { + let mut guard = self.stream.lock().unwrap(); + Pin::new(guard.deref_mut().get_mut()).is_write_vectored() + } + + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let mut guard = self.stream.lock().unwrap(); + Pin::new(guard.deref_mut().get_mut()).poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut guard = self.stream.lock().unwrap(); + Pin::new(guard.deref_mut().get_mut()).poll_flush(cx) + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut guard = self.stream.lock().unwrap(); + Pin::new(guard.deref_mut().get_mut()).poll_shutdown(cx) + } +} + pub struct TunTapSocket {} impl TunTapSocket { pub async fn create( @@ -38,14 +103,16 @@ impl TunTapSocket { hello: Hello, ) -> anyhow::Result { // Create the remote client - let (stream, res) = tokio_tungstenite::connect_async(url).await?; - let (tx, rx) = stream.split(); - tx.send(Message::Binary(bincode::serialize(&hello)?.to_vec())) - .await?; + let (stream, _res) = tokio_tungstenite::connect_async(url).await?; + let (mut tx, rx) = TungstensiteAdapter::new(stream).split(); + + // Send the hello message + tx.write_all(&bincode::serialize(&hello)?).await?; // Now pass it on to a remote networking adapter let (remote, remote_driver) = RemoteNetworking::new_from_stream_via_driver(Box::pin(tx), Box::pin(rx)); + let mut client = remote.bind_raw().await?; let iface = Iface::new("wasmer%d", Mode::Tun)?; cmd("ip", &["link", "set", "up", "dev", iface.name()])?; @@ -75,6 +142,7 @@ impl TunTapSocket { send_queue: Default::default(), _interest: interest, client, + remote_driver, }; Ok(driver) @@ -113,12 +181,21 @@ pub struct TunTapDriver { handler: TunTapHandler, send_queue: VecDeque, _interest: InterestGuard, - client: Box, + client: Box, + remote_driver: RemoteNetworkingDriver, } impl Future for TunTapDriver { type Output = io::Result<()>; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // Run the remote networking + loop { + match Pin::new(&mut self.remote_driver).poll(cx) { + Poll::Ready(()) => return Poll::Ready(Ok(())), + Poll::Pending => break, + } + } + // Add the waker before we drain all the the events // we need to read and send let inner = self.handler.inner.clone(); From 86802a81c6a04f2a9b902c2694e001875a792573 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 17 Jul 2023 22:38:30 +1000 Subject: [PATCH 14/52] Added a connect command --- lib/cli/src/commands.rs | 4 +- lib/cli/src/commands/connect.rs | 520 ++++++++++++++++++++++++++++++++ 2 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 lib/cli/src/commands/connect.rs diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 9f2365837cd..3d78fa5a899 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -6,6 +6,8 @@ mod cache; #[cfg(feature = "compiler")] mod compile; mod config; +mod connect; +mod connect; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] mod create_exe; #[cfg(feature = "static-artifact-create")] @@ -32,7 +34,7 @@ pub use create_exe::*; #[cfg(feature = "wast")] pub use wast::*; pub use { - add::*, cache::*, config::*, init::*, inspect::*, login::*, publish::*, run::Run, + add::*, cache::*, config::*, connect::*, init::*, inspect::*, login::*, publish::*, run::Run, self_update::*, validate::*, whoami::*, }; #[cfg(feature = "static-artifact-create")] diff --git a/lib/cli/src/commands/connect.rs b/lib/cli/src/commands/connect.rs new file mode 100644 index 00000000000..27258c0ea77 --- /dev/null +++ b/lib/cli/src/commands/connect.rs @@ -0,0 +1,520 @@ +use std::{net::TcpListener, path::PathBuf, str::FromStr, time::Duration}; + +use anyhow::Ok; + +use clap::Parser; +#[cfg(not(test))] +use dialoguer::{console::style, Input}; +use reqwest::Method; + +use serde::Deserialize; +use wasmer_registry::{ + types::NewNonceOutput, + wasmer_env::{Registry, WasmerEnv, WASMER_DIR}, + RegistryClient, +}; + +use hyper::{ + service::{make_service_fn, service_fn}, + Body, Request, Response, Server, StatusCode, +}; + +const WASMER_CLI: &str = "wasmer-cli"; + +/// Payload from the frontend after the user has authenticated. +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum TokenStatus { + /// Signifying that the token is cancelled + Cancelled, + /// Signifying that the token is authorized + Authorized, +} + +/// Payload from the frontend after the user has authenticated. +/// +/// This has the token that we need to set in the WASMER_TOML file. +#[derive(Clone, Debug, Deserialize)] +pub struct ValidatedNonceOutput { + /// Token Received from the frontend + pub token: Option, + /// Status of the token , whether it is authorized or cancelled + pub status: TokenStatus, +} + +/// Enum for the boolean like prompt options +#[derive(Debug, Clone, PartialEq)] +pub enum BoolPromptOptions { + /// Signifying a yes/true - using `y/Y` + Yes, + /// Signifying a No/false - using `n/N` + No, +} + +impl FromStr for BoolPromptOptions { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "y" | "Y" => Ok(BoolPromptOptions::Yes), + "n" | "N" => Ok(BoolPromptOptions::No), + _ => Err(anyhow::anyhow!("Invalid option")), + } + } +} + +impl ToString for BoolPromptOptions { + fn to_string(&self) -> String { + match self { + BoolPromptOptions::Yes => "y".to_string(), + BoolPromptOptions::No => "n".to_string(), + } + } +} + +type Token = String; + +#[derive(Debug, Clone)] +enum AuthorizationState { + TokenSuccess(Token), + Cancelled, + TimedOut, + UnknownMethod, +} + +#[derive(Clone)] +struct AppContext { + server_shutdown_tx: tokio::sync::mpsc::Sender, + token_tx: tokio::sync::mpsc::Sender, +} + +/// Subcommand for log in a user into Wasmer (using a browser or provided a token) +#[derive(Debug, Clone, Parser)] +pub struct Login { + /// Variable to login without opening a browser + #[clap(long, name = "no-browser", default_value = "false")] + pub no_browser: bool, + // Note: This is essentially a copy of WasmerEnv except the token is + // accepted as a main argument instead of via --token. + /// Set Wasmer's home directory + #[clap(long, env = "WASMER_DIR", default_value = WASMER_DIR.as_os_str())] + pub wasmer_dir: PathBuf, + /// The registry to fetch packages from (inferred from the environment by + /// default) + #[clap(long, env = "WASMER_REGISTRY")] + pub registry: Option, + /// The API token to use when communicating with the registry (inferred from + /// the environment by default) + pub token: Option, + /// The directory cached artefacts are saved to. + #[clap(long, env = "WASMER_CACHE_DIR")] + cache_dir: Option, +} + +impl Login { + fn get_token_or_ask_user(&self, env: &WasmerEnv) -> Result { + if let Some(token) = &self.token { + return Ok(AuthorizationState::TokenSuccess(token.clone())); + } + + let registry_host = env.registry_endpoint()?; + let registry_tld = tldextract::TldExtractor::new(tldextract::TldOption::default()) + .extract(registry_host.as_str()) + .map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::Other, + format!("Invalid registry for login {}: {e}", registry_host), + ) + })?; + + let login_prompt = match ( + registry_tld.domain.as_deref(), + registry_tld.suffix.as_deref(), + ) { + (Some(d), Some(s)) => { + format!("Please paste the login token from https://{d}.{s}/settings/access-tokens") + } + _ => "Please paste the login token".to_string(), + }; + #[cfg(test)] + { + Ok(AuthorizationState::TokenSuccess(login_prompt)) + } + #[cfg(not(test))] + { + let token = Input::new().with_prompt(&login_prompt).interact_text()?; + Ok(AuthorizationState::TokenSuccess(token)) + } + } + + async fn get_token_from_browser( + &self, + env: &WasmerEnv, + ) -> Result { + let registry = env.registry_endpoint()?; + + let client = RegistryClient::new(registry.clone(), None, None); + + let (listener, server_url) = Self::setup_listener().await?; + + let (server_shutdown_tx, mut server_shutdown_rx) = tokio::sync::mpsc::channel::(1); + let (token_tx, mut token_rx) = tokio::sync::mpsc::channel::(1); + + // Create a new AppContext + let app_context = AppContext { + server_shutdown_tx, + token_tx, + }; + + let NewNonceOutput { auth_url } = + wasmer_registry::api::create_nonce(&client, WASMER_CLI.to_string(), server_url).await?; + + // if failed to open the browser, then don't error out just print the auth_url with a message + println!("Opening auth link in your default browser: {}", &auth_url); + opener::open_browser(&auth_url).unwrap_or_else(|_| { + println!( + "⚠️ Failed to open the browser.\n + Please open the url: {}", + &auth_url + ); + }); + + // Create a new server + let make_svc = make_service_fn(move |_| { + let context = app_context.clone(); + + // Create a `Service` for responding to the request. + let service = service_fn(move |req| service_router(context.clone(), req)); + + // Return the service to hyper. + async move { Ok(service) } + }); + + print!("Waiting for session... "); + + // start the server + Server::from_tcp(listener)? + .serve(make_svc) + .with_graceful_shutdown(async { + server_shutdown_rx.recv().await; + }) + .await?; + + // receive the token from the server + let token = token_rx + .recv() + .await + .ok_or_else(|| anyhow::anyhow!("❌ Failed to receive token from localhost"))?; + + Ok(token) + } + + fn wasmer_env(&self) -> WasmerEnv { + WasmerEnv::new( + self.wasmer_dir.clone(), + self.registry.clone(), + self.token.clone(), + self.cache_dir.clone(), + ) + } + + async fn setup_listener() -> Result<(TcpListener, String), anyhow::Error> { + let listener = TcpListener::bind("127.0.0.1:0")?; + let addr = listener.local_addr()?; + let port = addr.port(); + + let server_url = format!("http://localhost:{}", port); + + Ok((listener, server_url)) + } + + /// execute [List] + #[tokio::main] + pub async fn execute(&self) -> Result<(), anyhow::Error> { + let env = self.wasmer_env(); + let registry = env.registry_endpoint()?; + + let auth_state = match self.token.clone() { + Some(token) => Ok(AuthorizationState::TokenSuccess(token)), + None => { + let person_wants_to_login = + match wasmer_registry::whoami(env.dir(), Some(registry.as_str()), None) { + std::result::Result::Ok((registry, user)) => { + println!( + "You are already logged in as {:?} on registry {:?}", + user, registry + ); + + #[cfg(not(test))] + { + let login_again = Input::new() + .with_prompt(format!( + "{} {} - [y/{}]", + style("?").yellow().bold(), + style("Do you want to login with another user?") + .bright() + .bold(), + style("N").green().bold() + )) + .show_default(false) + .default(BoolPromptOptions::No) + .interact_text()?; + + login_again == BoolPromptOptions::Yes + } + #[cfg(test)] + { + false + } + } + _ => true, + }; + + if !person_wants_to_login { + Ok(AuthorizationState::Cancelled) + } else if self.no_browser { + self.get_token_or_ask_user(&env) + } else { + // switch between two methods of getting the token. + // start two async processes, 10 minute timeout and get token from browser. Whichever finishes first, use that. + let timeout_future = tokio::time::sleep(Duration::from_secs(60 * 10)); + tokio::select! { + _ = timeout_future => { + Ok(AuthorizationState::TimedOut) + }, + token = self.get_token_from_browser(&env) => { + token + } + } + } + } + }?; + + match auth_state { + AuthorizationState::TokenSuccess(token) => { + match wasmer_registry::login::login_and_save_token( + env.dir(), + registry.as_str(), + &token, + )? { + Some(s) => { + print!("Done!"); + println!("\n✅ Login for Wasmer user {:?} saved", s) + } + None => print!( + "Warning: no user found on {:?} with the provided token.\nToken saved regardless.", + registry.domain().unwrap_or("registry.wasmer.io") + ), + }; + } + AuthorizationState::TimedOut => { + print!("Timed out (10 mins exceeded)"); + } + AuthorizationState::Cancelled => { + println!("Cancelled by the user"); + } + AuthorizationState::UnknownMethod => { + println!("Error: unknown method\n"); + } + }; + Ok(()) + } +} + +async fn preflight(req: Request) -> Result, anyhow::Error> { + let _whole_body = hyper::body::aggregate(req).await?; + let response = Response::builder() + .status(StatusCode::OK) + .header("Access-Control-Allow-Origin", "*") // FIXME: this is not secure, Don't allow all origins. @syrusakbary + .header("Access-Control-Allow-Headers", "Content-Type") + .header("Access-Control-Allow-Methods", "POST, OPTIONS") + .body(Body::default())?; + Ok(response) +} + +async fn handle_post_save_token( + context: AppContext, + req: Request, +) -> Result, anyhow::Error> { + let AppContext { + server_shutdown_tx, + token_tx, + } = context; + let (.., body) = req.into_parts(); + let body = hyper::body::to_bytes(body).await?; + + let ValidatedNonceOutput { + token, + status: token_status, + } = serde_json::from_slice::(&body)?; + + // send the AuthorizationState based on token_status to the main thread and get the response message + let (response_message, parse_failure) = match token_status { + TokenStatus::Cancelled => { + token_tx + .send(AuthorizationState::Cancelled) + .await + .expect("Failed to send token"); + + ("Token Cancelled by the user", false) + } + TokenStatus::Authorized => { + if let Some(token) = token { + token_tx + .send(AuthorizationState::TokenSuccess(token.clone())) + .await + .expect("Failed to send token"); + ("Token Authorized", false) + } else { + ("Token not found", true) + } + } + }; + + server_shutdown_tx + .send(true) + .await + .expect("Failed to send shutdown signal"); + + let status = if parse_failure { + StatusCode::BAD_REQUEST + } else { + StatusCode::OK + }; + + Ok(Response::builder() + .status(status) + .header("Access-Control-Allow-Origin", "*") // FIXME: this is not secure, Don't allow all origins. @syrusakbary + .header("Access-Control-Allow-Headers", "Content-Type") + .header("Access-Control-Allow-Methods", "POST, OPTIONS") + .body(Body::from(response_message))?) +} + +async fn handle_unknown_method(context: AppContext) -> Result, anyhow::Error> { + let AppContext { + server_shutdown_tx, + token_tx, + } = context; + + token_tx + .send(AuthorizationState::UnknownMethod) + .await + .expect("Failed to send token"); + + server_shutdown_tx + .send(true) + .await + .expect("Failed to send shutdown signal"); + + Ok(Response::builder() + .status(StatusCode::METHOD_NOT_ALLOWED) + .body(Body::from("Method not allowed"))?) +} + +/// Handle the preflight headers first - OPTIONS request +/// Then proceed to handle the actual request - POST request +async fn service_router( + context: AppContext, + req: Request, +) -> Result, anyhow::Error> { + match *req.method() { + Method::OPTIONS => preflight(req).await, + Method::POST => handle_post_save_token(context, req).await, + _ => handle_unknown_method(context).await, + } +} + +#[cfg(test)] +mod tests { + use clap::CommandFactory; + use tempfile::TempDir; + + use super::*; + + #[test] + fn interactive_login() { + let temp = TempDir::new().unwrap(); + let login = Login { + no_browser: true, + registry: Some("wasmer.wtf".into()), + wasmer_dir: temp.path().to_path_buf(), + token: None, + cache_dir: None, + }; + let env = login.wasmer_env(); + + let token = login.get_token_or_ask_user(&env).unwrap(); + match token { + AuthorizationState::TokenSuccess(token) => { + assert_eq!( + token, + "Please paste the login token from https://wasmer.wtf/settings/access-tokens" + ); + } + AuthorizationState::Cancelled + | AuthorizationState::TimedOut + | AuthorizationState::UnknownMethod => { + panic!("Should not reach here") + } + } + } + + #[test] + fn login_with_token() { + let temp = TempDir::new().unwrap(); + let login = Login { + no_browser: true, + registry: Some("wasmer.wtf".into()), + wasmer_dir: temp.path().to_path_buf(), + token: Some("abc".to_string()), + cache_dir: None, + }; + let env = login.wasmer_env(); + + let token = login.get_token_or_ask_user(&env).unwrap(); + + match token { + AuthorizationState::TokenSuccess(token) => { + assert_eq!(token, "abc"); + } + AuthorizationState::Cancelled + | AuthorizationState::TimedOut + | AuthorizationState::UnknownMethod => { + panic!("Should not reach here") + } + } + } + + #[test] + fn in_sync_with_wasmer_env() { + let wasmer_env = WasmerEnv::command(); + let login = Login::command(); + + // All options except --token should be the same + let wasmer_env_opts: Vec<_> = wasmer_env + .get_opts() + .filter(|arg| arg.get_id() != "token") + .collect(); + let login_opts: Vec<_> = login.get_opts().collect(); + + assert_eq!(wasmer_env_opts, login_opts); + + // The token argument should have the same message, even if it is an + // argument rather than a --flag. + let wasmer_env_token_help = wasmer_env + .get_opts() + .find(|arg| arg.get_id() == "token") + .unwrap() + .get_help() + .unwrap() + .to_string(); + let login_token_help = login + .get_positionals() + .find(|arg| arg.get_id() == "token") + .unwrap() + .get_help() + .unwrap() + .to_string(); + assert_eq!(wasmer_env_token_help, login_token_help); + } +} From ebe22a068c004ea098980c56e9890e13d281e1b7 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 18 Jul 2023 09:50:08 +1000 Subject: [PATCH 15/52] Renamed two of the projects so they can be externally referenced --- Cargo.lock | 902 +++++----------------- lib/cli/Cargo.toml | 4 +- lib/cli/src/cli.rs | 4 + lib/cli/src/commands.rs | 4 +- lib/cli/src/commands/connect.rs | 520 ------------- lib/cli/src/commands/run/wasi.rs | 6 +- lib/virtual-io/Cargo.toml | 2 +- lib/virtual-net/Cargo.toml | 15 +- lib/virtual-net/src/client.rs | 10 +- lib/virtual-net/src/host.rs | 6 +- lib/virtual-net/src/lib.rs | 10 +- lib/virtual-net/src/meta.rs | 8 - lib/virtual-net/src/server.rs | 6 +- lib/virtual-net/src/tun.rs | 298 ------- lib/wasi-web/Cargo.lock | 10 +- lib/wasi-web/src/runtime.rs | 4 +- lib/wasix/Cargo.toml | 8 +- lib/wasix/src/fs/inode_guard.rs | 4 +- lib/wasix/src/lib.rs | 9 +- lib/wasix/src/net/mod.rs | 2 +- lib/wasix/src/net/socket.rs | 12 +- lib/wasix/src/runtime/mod.rs | 6 +- lib/wasix/src/runtime/task_manager/mod.rs | 2 +- lib/wasix/src/state/env.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 2 +- lib/wasix/src/syscalls/wasix/epoll_ctl.rs | 2 +- 26 files changed, 248 insertions(+), 1610 deletions(-) delete mode 100644 lib/cli/src/commands/connect.rs delete mode 100644 lib/virtual-net/src/tun.rs diff --git a/Cargo.lock b/Cargo.lock index 305a15c151d..9852c9e30d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", "version_check", ] @@ -75,7 +75,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -214,7 +214,7 @@ dependencies = [ "futures-core", "memchr", "pin-project-lite", - "tokio 1.29.1", + "tokio", ] [[package]] @@ -245,7 +245,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi 0.1.19", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -262,7 +262,7 @@ checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" dependencies = [ "addr2line", "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide", "object 0.31.1", @@ -275,12 +275,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.2" @@ -338,7 +332,7 @@ dependencies = [ "arrayref", "arrayvec 0.7.4", "cc", - "cfg-if 1.0.0", + "cfg-if", "constant_time_eq", "digest", ] @@ -423,16 +417,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.4.0" @@ -516,12 +500,6 @@ dependencies = [ "jobserver", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -539,7 +517,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -614,15 +592,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "colorchoice" version = "1.0.0" @@ -637,7 +606,7 @@ checksum = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" dependencies = [ "atty", "lazy_static", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -699,14 +668,14 @@ dependencies = [ "lazy_static", "libc", "log", - "miow 0.3.7", + "miow", "regex", "rustfix", "serde", "serde_derive", "serde_json", "tester", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -728,7 +697,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen", ] @@ -773,7 +742,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9847f90f32a50b0dcbd68bc23ff242798b13080b97b0569f6ed96a45ce4cf2cd" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "libc", "scopeguard", "windows-sys 0.33.0", @@ -824,7 +793,7 @@ dependencies = [ "hashbrown 0.12.3", "log", "regalloc2", - "smallvec 1.11.0", + "smallvec", "target-lexicon 0.12.9", ] @@ -854,7 +823,7 @@ dependencies = [ "hashbrown 0.12.3", "indexmap 1.9.3", "log", - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -872,7 +841,7 @@ dependencies = [ "cranelift-codegen", "hashbrown 0.12.3", "log", - "smallvec 1.11.0", + "smallvec", "target-lexicon 0.12.9", ] @@ -897,7 +866,7 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -948,19 +917,8 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-utils 0.8.16", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "cfg-if", + "crossbeam-utils", ] [[package]] @@ -969,24 +927,9 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if 1.0.0", - "crossbeam-epoch 0.9.15", - "crossbeam-utils 0.8.16", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] @@ -996,41 +939,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", - "cfg-if 1.0.0", - "crossbeam-utils 0.8.16", + "cfg-if", + "crossbeam-utils", "memoffset 0.9.0", "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" -dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", -] - [[package]] name = "crossbeam-utils" version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1042,11 +963,11 @@ dependencies = [ "bitflags 1.3.2", "crossterm_winapi", "libc", - "mio 0.8.8", + "mio", "parking_lot 0.12.1", "signal-hook", "signal-hook-mio", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1055,7 +976,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1220,9 +1141,9 @@ version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "hashbrown 0.14.0", - "lock_api 0.4.10", + "lock_api", "once_cell", "parking_lot_core 0.9.8", ] @@ -1299,7 +1220,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -1311,7 +1232,7 @@ checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1322,7 +1243,7 @@ checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", "redox_users", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1414,7 +1335,7 @@ version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -1541,7 +1462,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.2.16", "windows-sys 0.48.0", @@ -1608,34 +1529,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags 1.3.2", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.28" @@ -1759,7 +1658,7 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "libc", "wasi", @@ -1884,7 +1783,7 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "futures-core", "futures-sink", @@ -1892,7 +1791,7 @@ dependencies = [ "http", "indexmap 1.9.3", "slab", - "tokio 1.29.1", + "tokio", "tokio-util", "tracing", ] @@ -2013,7 +1912,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "itoa", ] @@ -2024,7 +1923,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.4.0", + "bytes", "http", "pin-project-lite", ] @@ -2063,7 +1962,7 @@ version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -2075,7 +1974,7 @@ dependencies = [ "itoa", "pin-project-lite", "socket2", - "tokio 1.29.1", + "tokio", "tower-service", "tracing", "want", @@ -2091,7 +1990,7 @@ dependencies = [ "http", "hyper", "rustls", - "tokio 1.29.1", + "tokio", "tokio-rustls", ] @@ -2101,10 +2000,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.4.0", + "bytes", "hyper", "native-tls", - "tokio 1.29.1", + "tokio", "tokio-native-tls", ] @@ -2280,7 +2179,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -2307,15 +2206,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "ipnet" version = "2.8.0" @@ -2377,16 +2267,6 @@ dependencies = [ "serde", ] -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2407,7 +2287,7 @@ checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec 0.5.2", "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "ryu", "static_assertions", ] @@ -2444,7 +2324,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d580318f95776505201b28cf98eb1fa5e4be3b689633ba6a3e6cd880ff22d8cb" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-sys 0.48.0", ] @@ -2499,15 +2379,6 @@ dependencies = [ "semver 1.0.17", ] -[[package]] -name = "lock_api" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" -dependencies = [ - "scopeguard", -] - [[package]] name = "lock_api" version = "0.4.10" @@ -2583,12 +2454,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md5" version = "0.7.0" @@ -2619,15 +2484,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.5" @@ -2679,7 +2535,7 @@ checksum = "c66a1fdd7e946fe33fe9725012e25836bba3655769bee9ee347cce7de3f396df" dependencies = [ "cc", "dlib", - "futures 0.3.28", + "futures", "instant", "js-sys", "lazy_static", @@ -2693,7 +2549,7 @@ dependencies = [ "wayland-client", "wayland-cursor", "wayland-protocols", - "winapi 0.3.9", + "winapi", "x11-dl", ] @@ -2723,25 +2579,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - [[package]] name = "mio" version = "0.8.8" @@ -2754,36 +2591,13 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio 0.6.23", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - [[package]] name = "miow" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2810,17 +2624,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "net2" -version = "0.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b13b648036a2339d06de780866fbdfda0dde886de7b3af2ddeba8b14f4ee34ac" -dependencies = [ - "cfg-if 0.1.10", - "libc", - "winapi 0.3.9", -] - [[package]] name = "nix" version = "0.24.3" @@ -2828,7 +2631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" dependencies = [ "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "libc", "memoffset 0.6.5", ] @@ -2841,7 +2644,7 @@ checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "libc", "memoffset 0.6.5", "pin-utils", @@ -2890,7 +2693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" dependencies = [ "overload", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -2996,7 +2799,7 @@ checksum = "6c62dcb6174f9cb326eac248f07e955d5d559c272730b6c03e396b443b562788" dependencies = [ "bstr 1.6.0", "normpath", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3006,7 +2809,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ "bitflags 1.3.2", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -3061,17 +2864,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking_lot" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -dependencies = [ - "lock_api 0.3.4", - "parking_lot_core 0.6.3", - "rustc_version 0.2.3", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -3079,7 +2871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api 0.4.10", + "lock_api", "parking_lot_core 0.8.6", ] @@ -3089,37 +2881,22 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api 0.4.10", + "lock_api", "parking_lot_core 0.9.8", ] -[[package]] -name = "parking_lot_core" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66b810a62be75176a80873726630147a5ca780cd33921e0b5709033e66b0a" -dependencies = [ - "cfg-if 0.1.10", - "cloudabi", - "libc", - "redox_syscall 0.1.57", - "rustc_version 0.2.3", - "smallvec 0.6.14", - "winapi 0.3.9", -] - [[package]] name = "parking_lot_core" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "instant", "libc", "redox_syscall 0.2.16", - "smallvec 1.11.0", - "winapi 0.3.9", + "smallvec", + "winapi", ] [[package]] @@ -3128,10 +2905,10 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.3.5", - "smallvec 1.11.0", + "smallvec", "windows-targets 0.48.1", ] @@ -3545,17 +3322,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", - "crossbeam-deque 0.8.3", - "crossbeam-utils 0.8.16", + "crossbeam-deque", + "crossbeam-utils", "num_cpus", ] -[[package]] -name = "redox_syscall" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - [[package]] name = "redox_syscall" version = "0.2.16" @@ -3600,7 +3371,7 @@ dependencies = [ "fxhash", "log", "slice-group-by", - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -3656,7 +3427,7 @@ dependencies = [ "bitflags 1.3.2", "libc", "mach", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3681,8 +3452,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "async-compression", - "base64 0.21.2", - "bytes 1.4.0", + "base64", + "bytes", "encoding_rs", "futures-core", "futures-util", @@ -3706,7 +3477,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "tokio 1.29.1", + "tokio", "tokio-native-tls", "tokio-rustls", "tokio-util", @@ -3732,7 +3503,7 @@ dependencies = [ "spin 0.5.2", "untrusted", "web-sys", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3772,7 +3543,7 @@ checksum = "6678cf63ab3491898c0d021b493c94c9b221d91295294a2a5746eacbe5928322" dependencies = [ "libc", "rtoolbox", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3782,7 +3553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "034e22c514f5c0cb8a10ff341b9b048b5ceb21591f31c8f44c43b960f9b3524a" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -3796,7 +3567,7 @@ dependencies = [ "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -3889,7 +3660,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64", ] [[package]] @@ -3997,12 +3768,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "scoped-tls" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" - [[package]] name = "scoped-tls" version = "1.0.1" @@ -4054,7 +3819,7 @@ version = "0.35.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3586be2cf6c0a8099a79a12b4084357aa9b3e0b0d7980e3b67aaf7a9d55f9f0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "version-compare", ] @@ -4289,17 +4054,6 @@ dependencies = [ "sha1_smol", ] -[[package]] -name = "sha1" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "digest", -] - [[package]] name = "sha1_smol" version = "1.0.0" @@ -4312,7 +4066,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest", ] @@ -4332,7 +4086,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cf61602ee61e2f83dd016b3e6387245291cf728ea071c378b35088125b4d995" dependencies = [ - "bytes 1.4.0", + "bytes", "memmap2 0.6.2", ] @@ -4359,7 +4113,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio 0.8.8", + "mio", "signal-hook", ] @@ -4399,15 +4153,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.11.0" @@ -4421,7 +4166,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4436,7 +4181,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api 0.4.10", + "lock_api", ] [[package]] @@ -4511,7 +4256,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1 0.6.1", + "sha1", "syn 1.0.109", ] @@ -4613,7 +4358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "fastrand", "redox_syscall 0.3.5", "rustix 0.37.23", @@ -4628,7 +4373,7 @@ checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" dependencies = [ "dirs-next", "rustversion", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4638,7 +4383,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e4129646ca0ed8f45d09b929036bafad5377103edd06e50bf574b353d2b08d9" dependencies = [ "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4690,7 +4435,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e8bf7e0eb2dd7b4228cc1b6821fc5114cd6841ae59f652a85488c016091e5f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "getopts", "libc", "num_cpus", @@ -4732,7 +4477,7 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -4748,7 +4493,7 @@ dependencies = [ "stdweb", "time-macros 0.1.1", "version_check", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -4840,30 +4585,6 @@ dependencies = [ "url", ] -[[package]] -name = "tokio" -version = "0.1.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "mio 0.6.23", - "num_cpus", - "tokio-codec", - "tokio-current-thread", - "tokio-executor", - "tokio-fs", - "tokio-io", - "tokio-reactor", - "tokio-sync", - "tokio-tcp", - "tokio-threadpool", - "tokio-timer", - "tokio-udp", - "tokio-uds", -] - [[package]] name = "tokio" version = "1.29.1" @@ -4872,9 +4593,9 @@ checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", "backtrace", - "bytes 1.4.0", + "bytes", "libc", - "mio 0.8.8", + "mio", "num_cpus", "pin-project-lite", "socket2", @@ -4882,78 +4603,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tokio-codec" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "tokio-io", -] - -[[package]] -name = "tokio-core" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87b1395334443abca552f63d4f61d0486f12377c2ba8b368e523f89e828cffd4" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "log", - "mio 0.6.23", - "scoped-tls 0.1.2", - "tokio 0.1.22", - "tokio-executor", - "tokio-io", - "tokio-reactor", - "tokio-timer", -] - -[[package]] -name = "tokio-current-thread" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" -dependencies = [ - "futures 0.1.31", - "tokio-executor", -] - -[[package]] -name = "tokio-executor" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", -] - -[[package]] -name = "tokio-fs" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" -dependencies = [ - "futures 0.1.31", - "tokio-io", - "tokio-threadpool", -] - -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", -] - [[package]] name = "tokio-macros" version = "2.1.0" @@ -4972,26 +4621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", - "tokio 1.29.1", -] - -[[package]] -name = "tokio-reactor" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "mio 0.6.23", - "num_cpus", - "parking_lot 0.9.0", - "slab", - "tokio-executor", - "tokio-io", - "tokio-sync", + "tokio", ] [[package]] @@ -5001,105 +4631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", - "tokio 1.29.1", -] - -[[package]] -name = "tokio-sync" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" -dependencies = [ - "fnv", - "futures 0.1.31", -] - -[[package]] -name = "tokio-tcp" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "mio 0.6.23", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-threadpool" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" -dependencies = [ - "crossbeam-deque 0.7.4", - "crossbeam-queue", - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "lazy_static", - "log", - "num_cpus", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-timer" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" -dependencies = [ - "crossbeam-utils 0.7.2", - "futures 0.1.31", - "slab", - "tokio-executor", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" -dependencies = [ - "futures-util", - "log", - "tokio 1.29.1", - "tungstenite", -] - -[[package]] -name = "tokio-udp" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", - "mio 0.6.23", - "tokio-codec", - "tokio-io", - "tokio-reactor", -] - -[[package]] -name = "tokio-uds" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "iovec", - "libc", - "log", - "mio 0.6.23", - "mio-uds", - "tokio-codec", - "tokio-io", - "tokio-reactor", + "tokio", ] [[package]] @@ -5108,11 +4640,11 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-core", "futures-sink", "pin-project-lite", - "tokio 1.29.1", + "tokio", "tracing", ] @@ -5169,7 +4701,7 @@ dependencies = [ "futures-util", "pin-project", "pin-project-lite", - "tokio 1.29.1", + "tokio", "tower-layer", "tower-service", "tracing", @@ -5182,7 +4714,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ "bitflags 2.3.3", - "bytes 1.4.0", + "bytes", "futures-core", "futures-util", "http", @@ -5213,7 +4745,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -5276,7 +4808,7 @@ dependencies = [ "serde", "serde_json", "sharded-slab", - "smallvec 1.11.0", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -5295,7 +4827,7 @@ dependencies = [ "once_cell", "regex", "sharded-slab", - "smallvec 1.11.0", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -5334,38 +4866,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "tun-tap" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27165dce5c30856f7e11754ff8d3bb6fc965fba6f25f31ecbddc7c23f99e1615" -dependencies = [ - "cc", - "futures 0.1.31", - "libc", - "mio 0.6.23", - "tokio-core", -] - -[[package]] -name = "tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes 1.4.0", - "http", - "httparse", - "log", - "rand", - "sha1 0.10.5", - "thiserror", - "url", - "utf-8", -] - [[package]] name = "typenum" version = "1.16.0" @@ -5495,12 +4995,6 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8parse" version = "0.2.1" @@ -5562,11 +5056,11 @@ version = "0.6.0" dependencies = [ "anyhow", "async-trait", - "bytes 1.4.0", + "bytes", "derivative", "filetime", "fs_extra", - "futures 0.3.28", + "futures", "getrandom", "indexmap 1.9.3", "lazy_static", @@ -5577,52 +5071,12 @@ dependencies = [ "slab", "tempfile", "thiserror", - "tokio 1.29.1", + "tokio", "tracing", "typetag", "webc", ] -[[package]] -name = "virtual-io" -version = "0.1.0" -dependencies = [ - "async-trait", - "bytes 1.4.0", - "derivative", - "futures 0.3.28", - "mio 0.8.8", - "serde", - "socket2", - "thiserror", - "tracing", -] - -[[package]] -name = "virtual-net" -version = "0.3.0" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.21.2", - "bincode", - "bytes 1.4.0", - "derivative", - "futures 0.3.28", - "futures-util", - "libc", - "mio 0.8.8", - "pin-project-lite", - "serde", - "socket2", - "thiserror", - "tokio 1.29.1", - "tokio-tungstenite", - "tracing", - "tun-tap", - "virtual-io", -] - [[package]] name = "void" version = "1.0.2" @@ -5774,7 +5228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d816eaf89496a0eefecb79f43463b07205701165c17790fa6f0eab5ae72bc39" dependencies = [ "anyhow", - "base64 0.21.2", + "base64", "flate2", "indexmap 1.9.3", "json5", @@ -5826,7 +5280,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -5874,7 +5328,7 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -5917,7 +5371,7 @@ checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" dependencies = [ "console_error_panic_hook", "js-sys", - "scoped-tls 1.0.1", + "scoped-tls", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test-macro", @@ -6017,8 +5471,8 @@ name = "wasmer" version = "4.0.0" dependencies = [ "anyhow", - "bytes 1.4.0", - "cfg-if 1.0.0", + "bytes", + "cfg-if", "derivative", "hashbrown 0.11.2", "indexmap 1.9.3", @@ -6046,7 +5500,7 @@ dependencies = [ "wasmparser 0.83.0", "wasmparser 0.95.0", "wat", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6057,7 +5511,7 @@ checksum = "e5f22cf70f93c4690d4e82b325634ad973cd3a02d5a6b6565d442b452ea729bf" dependencies = [ "anyhow", "cynic", - "futures 0.3.28", + "futures", "pin-project-lite", "reqwest", "serde", @@ -6090,7 +5544,7 @@ name = "wasmer-c-api" version = "4.0.0" dependencies = [ "cbindgen", - "cfg-if 1.0.0", + "cfg-if", "enumset", "field-offset", "inline-c", @@ -6098,7 +5552,7 @@ dependencies = [ "libc", "paste", "thiserror", - "tokio 1.29.1", + "tokio", "typetag", "virtual-fs", "wasmer", @@ -6156,10 +5610,10 @@ dependencies = [ "assert_cmd 2.0.11", "async-trait", "atty", - "bytes 1.4.0", + "bytes", "bytesize", "cargo_metadata", - "cfg-if 1.0.0", + "cfg-if", "chrono", "clap 4.3.11", "colored 2.0.4", @@ -6193,14 +5647,13 @@ dependencies = [ "tempfile", "thiserror", "tldextract", - "tokio 1.29.1", + "tokio", "toml 0.5.11", "tracing", "tracing-subscriber 0.3.17", "unix_mode", "url", "virtual-fs", - "virtual-net", "walkdir", "wapm-targz-to-pirita", "wasm-coredump-builder", @@ -6216,6 +5669,7 @@ dependencies = [ "wasmer-registry 5.2.0", "wasmer-toml", "wasmer-types", + "wasmer-virtual-net", "wasmer-vm", "wasmer-wasix", "wasmer-wasix-experimental-io-devices", @@ -6228,7 +5682,7 @@ name = "wasmer-compiler" version = "4.0.0" dependencies = [ "backtrace", - "cfg-if 1.0.0", + "cfg-if", "enum-iterator", "enumset", "hashbrown 0.11.2", @@ -6239,13 +5693,13 @@ dependencies = [ "region", "serde", "serde_bytes", - "smallvec 1.11.0", + "smallvec", "thiserror", "wasmer-object", "wasmer-types", "wasmer-vm", "wasmparser 0.95.0", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6254,7 +5708,7 @@ version = "4.0.0" dependencies = [ "anyhow", "bytesize", - "cfg-if 1.0.0", + "cfg-if", "clap 4.3.11", "colored 2.0.4", "distance", @@ -6281,7 +5735,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec 1.11.0", + "smallvec", "target-lexicon 0.12.9", "tracing", "wasmer-compiler", @@ -6303,7 +5757,7 @@ dependencies = [ "regex", "rustc_version 0.4.0", "semver 1.0.17", - "smallvec 1.11.0", + "smallvec", "target-lexicon 0.12.9", "wasmer-compiler", "wasmer-types", @@ -6323,7 +5777,7 @@ dependencies = [ "lazy_static", "more-asserts", "rayon", - "smallvec 1.11.0", + "smallvec", "target-lexicon 0.12.9", "wasmer-compiler", "wasmer-types", @@ -6340,7 +5794,7 @@ dependencies = [ "clap-verbosity-flag", "comfy-table", "dialoguer", - "futures 0.3.28", + "futures", "is-terminal", "log", "once_cell", @@ -6351,7 +5805,7 @@ dependencies = [ "serde_json", "serde_yaml 0.8.26", "time 0.3.23", - "tokio 1.29.1", + "tokio", "toml 0.7.6", "tracing", "tracing-subscriber 0.3.17", @@ -6458,7 +5912,7 @@ dependencies = [ "derivative", "dirs", "flate2", - "futures 0.3.28", + "futures", "hex", "insta", "md5", @@ -6473,7 +5927,7 @@ dependencies = [ "tar", "target-lexicon 0.12.9", "tempfile", - "tokio 1.29.1", + "tokio", ] [[package]] @@ -6531,7 +5985,7 @@ dependencies = [ "thiserror", "time 0.3.23", "tldextract", - "tokio 1.29.1", + "tokio", "toml 0.5.11", "url", "wasmer-toml", @@ -6573,7 +6027,7 @@ dependencies = [ "thiserror", "time 0.3.23", "tldextract", - "tokio 1.29.1", + "tokio", "toml 0.5.11", "url", "wasmer-toml", @@ -6630,13 +6084,49 @@ dependencies = [ "thiserror", ] +[[package]] +name = "wasmer-virtual-io" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "derivative", + "futures", + "mio", + "serde", + "socket2", + "thiserror", + "tracing", +] + +[[package]] +name = "wasmer-virtual-net" +version = "0.3.0" +dependencies = [ + "async-trait", + "base64", + "bincode", + "bytes", + "derivative", + "futures-util", + "libc", + "mio", + "pin-project-lite", + "serde", + "socket2", + "thiserror", + "tokio", + "tracing", + "wasmer-virtual-io", +] + [[package]] name = "wasmer-vm" version = "4.0.0" dependencies = [ "backtrace", "cc", - "cfg-if 1.0.0", + "cfg-if", "corosensei", "dashmap", "derivative", @@ -6654,7 +6144,7 @@ dependencies = [ "thiserror", "tracing", "wasmer-types", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6664,13 +6154,13 @@ dependencies = [ "anyhow", "async-trait", "bincode", - "bytes 1.4.0", - "cfg-if 1.0.0", + "bytes", + "cfg-if", "chrono", "cooked-waker", "dashmap", "derivative", - "futures 0.3.28", + "futures", "getrandom", "heapless", "hex", @@ -6697,7 +6187,7 @@ dependencies = [ "term_size", "termios", "thiserror", - "tokio 1.29.1", + "tokio", "tower", "tower-http", "tracing", @@ -6707,8 +6197,6 @@ dependencies = [ "url", "urlencoding", "virtual-fs", - "virtual-io", - "virtual-net", "wai-bindgen-wasmer", "waker-fn", "wapm-targz-to-pirita", @@ -6717,12 +6205,14 @@ dependencies = [ "wasmer", "wasmer-emscripten", "wasmer-types", + "wasmer-virtual-io", + "wasmer-virtual-net", "wasmer-wasix-types", "wcgi", "wcgi-host", "webc", "weezl", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -6747,7 +6237,7 @@ dependencies = [ "anyhow", "bitflags 1.3.2", "byteorder", - "cfg-if 1.0.0", + "cfg-if", "num_enum", "pretty_assertions", "serde", @@ -6792,11 +6282,11 @@ name = "wasmer-wast" version = "4.0.0" dependencies = [ "anyhow", - "futures 0.3.28", + "futures", "serde", "tempfile", "thiserror", - "tokio 1.29.1", + "tokio", "virtual-fs", "wasmer", "wasmer-wasix", @@ -6809,7 +6299,7 @@ version = "4.0.0" dependencies = [ "anyhow", "build-deps", - "cfg-if 1.0.0", + "cfg-if", "compiler-test-derive", "criterion", "glob", @@ -6819,7 +6309,7 @@ dependencies = [ "tempfile", "test-generator", "test-log", - "tokio 1.29.1", + "tokio", "tracing", "tracing-subscriber 0.3.17", "wasi-test-generator", @@ -6927,7 +6417,7 @@ dependencies = [ "downcast-rs", "libc", "nix 0.24.3", - "scoped-tls 1.0.1", + "scoped-tls", "wayland-commons", "wayland-scanner", "wayland-sys", @@ -6941,7 +6431,7 @@ checksum = "8691f134d584a33a6606d9d717b95c4fa20065605f798a3f350d78dced02a902" dependencies = [ "nix 0.24.3", "once_cell", - "smallvec 1.11.0", + "smallvec", "wayland-sys", ] @@ -7012,7 +6502,7 @@ dependencies = [ "http", "schemars", "serde", - "tokio 1.29.1", + "tokio", "wasmparser 0.95.0", "wcgi", ] @@ -7034,9 +6524,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02ba1b6e7ad252e691a86d727aa422fbc5ed95e9bca4ec8547869e9b5779f8f3" dependencies = [ "anyhow", - "base64 0.21.2", + "base64", "byteorder", - "bytes 1.4.0", + "bytes", "indexmap 1.9.3", "leb128", "lexical-sort", @@ -7088,12 +6578,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -7104,12 +6588,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -7122,7 +6600,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -7330,17 +6808,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ - "winapi 0.3.9", -] - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] [[package]] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 971f7fea9ff..cf37ad400f5 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -60,7 +60,7 @@ wasmer-object = { version = "=4.0.0", path = "../object", optional = true } virtual-fs = { version = "0.6.0", path = "../virtual-fs", default-features = false, features = [ "host-fs", ] } -virtual-net = { version = "0.3.0", path = "../virtual-net", features = ["tun-tap"] } +wasmer-virtual-net = { version = "0.3.0", path = "../virtual-net" } # Wasmer-owned dependencies. webc = { workspace = true } @@ -170,7 +170,7 @@ coredump = ["wasm-coredump-builder"] sys = ["compiler", "wasmer-vm"] jsc = ["backend", "wasmer/jsc", "wasmer/std"] wast = ["wasmer-wast"] -host-net = ["virtual-net/host-net"] +host-net = ["wasmer-virtual-net/host-net"] wat = ["wasmer/wat"] compiler = [ "backend", diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 7ed4dfabe34..4858faa3897 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -108,6 +108,7 @@ impl Args { Some(Cmd::Init(init)) => init.execute(), Some(Cmd::Login(login)) => login.execute(), Some(Cmd::Publish(publish)) => publish.execute(), + Some(Cmd::Connect(connect)) => connect.execute(), #[cfg(feature = "static-artifact-create")] Some(Cmd::GenCHeader(gen_heder)) => gen_heder.execute(), #[cfg(feature = "wast")] @@ -267,6 +268,9 @@ enum Cmd { /// Create a dynamic on the Deploy Edge, and connect to it through SSH. Ssh(wasmer_deploy_cli::cmd::ssh::CmdSsh), + /// Connect to the Edge. + Connect(wasmer_deploy_cli::cmd::connect::CmdConnect), + /// Manage Wasmer namespaces. #[clap(subcommand, alias = "namespaces")] Namespace(wasmer_deploy_cli::cmd::namespace::CmdNamespace), diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 3d78fa5a899..9f2365837cd 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -6,8 +6,6 @@ mod cache; #[cfg(feature = "compiler")] mod compile; mod config; -mod connect; -mod connect; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] mod create_exe; #[cfg(feature = "static-artifact-create")] @@ -34,7 +32,7 @@ pub use create_exe::*; #[cfg(feature = "wast")] pub use wast::*; pub use { - add::*, cache::*, config::*, connect::*, init::*, inspect::*, login::*, publish::*, run::Run, + add::*, cache::*, config::*, init::*, inspect::*, login::*, publish::*, run::Run, self_update::*, validate::*, whoami::*, }; #[cfg(feature = "static-artifact-create")] diff --git a/lib/cli/src/commands/connect.rs b/lib/cli/src/commands/connect.rs deleted file mode 100644 index 27258c0ea77..00000000000 --- a/lib/cli/src/commands/connect.rs +++ /dev/null @@ -1,520 +0,0 @@ -use std::{net::TcpListener, path::PathBuf, str::FromStr, time::Duration}; - -use anyhow::Ok; - -use clap::Parser; -#[cfg(not(test))] -use dialoguer::{console::style, Input}; -use reqwest::Method; - -use serde::Deserialize; -use wasmer_registry::{ - types::NewNonceOutput, - wasmer_env::{Registry, WasmerEnv, WASMER_DIR}, - RegistryClient, -}; - -use hyper::{ - service::{make_service_fn, service_fn}, - Body, Request, Response, Server, StatusCode, -}; - -const WASMER_CLI: &str = "wasmer-cli"; - -/// Payload from the frontend after the user has authenticated. -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum TokenStatus { - /// Signifying that the token is cancelled - Cancelled, - /// Signifying that the token is authorized - Authorized, -} - -/// Payload from the frontend after the user has authenticated. -/// -/// This has the token that we need to set in the WASMER_TOML file. -#[derive(Clone, Debug, Deserialize)] -pub struct ValidatedNonceOutput { - /// Token Received from the frontend - pub token: Option, - /// Status of the token , whether it is authorized or cancelled - pub status: TokenStatus, -} - -/// Enum for the boolean like prompt options -#[derive(Debug, Clone, PartialEq)] -pub enum BoolPromptOptions { - /// Signifying a yes/true - using `y/Y` - Yes, - /// Signifying a No/false - using `n/N` - No, -} - -impl FromStr for BoolPromptOptions { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "y" | "Y" => Ok(BoolPromptOptions::Yes), - "n" | "N" => Ok(BoolPromptOptions::No), - _ => Err(anyhow::anyhow!("Invalid option")), - } - } -} - -impl ToString for BoolPromptOptions { - fn to_string(&self) -> String { - match self { - BoolPromptOptions::Yes => "y".to_string(), - BoolPromptOptions::No => "n".to_string(), - } - } -} - -type Token = String; - -#[derive(Debug, Clone)] -enum AuthorizationState { - TokenSuccess(Token), - Cancelled, - TimedOut, - UnknownMethod, -} - -#[derive(Clone)] -struct AppContext { - server_shutdown_tx: tokio::sync::mpsc::Sender, - token_tx: tokio::sync::mpsc::Sender, -} - -/// Subcommand for log in a user into Wasmer (using a browser or provided a token) -#[derive(Debug, Clone, Parser)] -pub struct Login { - /// Variable to login without opening a browser - #[clap(long, name = "no-browser", default_value = "false")] - pub no_browser: bool, - // Note: This is essentially a copy of WasmerEnv except the token is - // accepted as a main argument instead of via --token. - /// Set Wasmer's home directory - #[clap(long, env = "WASMER_DIR", default_value = WASMER_DIR.as_os_str())] - pub wasmer_dir: PathBuf, - /// The registry to fetch packages from (inferred from the environment by - /// default) - #[clap(long, env = "WASMER_REGISTRY")] - pub registry: Option, - /// The API token to use when communicating with the registry (inferred from - /// the environment by default) - pub token: Option, - /// The directory cached artefacts are saved to. - #[clap(long, env = "WASMER_CACHE_DIR")] - cache_dir: Option, -} - -impl Login { - fn get_token_or_ask_user(&self, env: &WasmerEnv) -> Result { - if let Some(token) = &self.token { - return Ok(AuthorizationState::TokenSuccess(token.clone())); - } - - let registry_host = env.registry_endpoint()?; - let registry_tld = tldextract::TldExtractor::new(tldextract::TldOption::default()) - .extract(registry_host.as_str()) - .map_err(|e| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Invalid registry for login {}: {e}", registry_host), - ) - })?; - - let login_prompt = match ( - registry_tld.domain.as_deref(), - registry_tld.suffix.as_deref(), - ) { - (Some(d), Some(s)) => { - format!("Please paste the login token from https://{d}.{s}/settings/access-tokens") - } - _ => "Please paste the login token".to_string(), - }; - #[cfg(test)] - { - Ok(AuthorizationState::TokenSuccess(login_prompt)) - } - #[cfg(not(test))] - { - let token = Input::new().with_prompt(&login_prompt).interact_text()?; - Ok(AuthorizationState::TokenSuccess(token)) - } - } - - async fn get_token_from_browser( - &self, - env: &WasmerEnv, - ) -> Result { - let registry = env.registry_endpoint()?; - - let client = RegistryClient::new(registry.clone(), None, None); - - let (listener, server_url) = Self::setup_listener().await?; - - let (server_shutdown_tx, mut server_shutdown_rx) = tokio::sync::mpsc::channel::(1); - let (token_tx, mut token_rx) = tokio::sync::mpsc::channel::(1); - - // Create a new AppContext - let app_context = AppContext { - server_shutdown_tx, - token_tx, - }; - - let NewNonceOutput { auth_url } = - wasmer_registry::api::create_nonce(&client, WASMER_CLI.to_string(), server_url).await?; - - // if failed to open the browser, then don't error out just print the auth_url with a message - println!("Opening auth link in your default browser: {}", &auth_url); - opener::open_browser(&auth_url).unwrap_or_else(|_| { - println!( - "⚠️ Failed to open the browser.\n - Please open the url: {}", - &auth_url - ); - }); - - // Create a new server - let make_svc = make_service_fn(move |_| { - let context = app_context.clone(); - - // Create a `Service` for responding to the request. - let service = service_fn(move |req| service_router(context.clone(), req)); - - // Return the service to hyper. - async move { Ok(service) } - }); - - print!("Waiting for session... "); - - // start the server - Server::from_tcp(listener)? - .serve(make_svc) - .with_graceful_shutdown(async { - server_shutdown_rx.recv().await; - }) - .await?; - - // receive the token from the server - let token = token_rx - .recv() - .await - .ok_or_else(|| anyhow::anyhow!("❌ Failed to receive token from localhost"))?; - - Ok(token) - } - - fn wasmer_env(&self) -> WasmerEnv { - WasmerEnv::new( - self.wasmer_dir.clone(), - self.registry.clone(), - self.token.clone(), - self.cache_dir.clone(), - ) - } - - async fn setup_listener() -> Result<(TcpListener, String), anyhow::Error> { - let listener = TcpListener::bind("127.0.0.1:0")?; - let addr = listener.local_addr()?; - let port = addr.port(); - - let server_url = format!("http://localhost:{}", port); - - Ok((listener, server_url)) - } - - /// execute [List] - #[tokio::main] - pub async fn execute(&self) -> Result<(), anyhow::Error> { - let env = self.wasmer_env(); - let registry = env.registry_endpoint()?; - - let auth_state = match self.token.clone() { - Some(token) => Ok(AuthorizationState::TokenSuccess(token)), - None => { - let person_wants_to_login = - match wasmer_registry::whoami(env.dir(), Some(registry.as_str()), None) { - std::result::Result::Ok((registry, user)) => { - println!( - "You are already logged in as {:?} on registry {:?}", - user, registry - ); - - #[cfg(not(test))] - { - let login_again = Input::new() - .with_prompt(format!( - "{} {} - [y/{}]", - style("?").yellow().bold(), - style("Do you want to login with another user?") - .bright() - .bold(), - style("N").green().bold() - )) - .show_default(false) - .default(BoolPromptOptions::No) - .interact_text()?; - - login_again == BoolPromptOptions::Yes - } - #[cfg(test)] - { - false - } - } - _ => true, - }; - - if !person_wants_to_login { - Ok(AuthorizationState::Cancelled) - } else if self.no_browser { - self.get_token_or_ask_user(&env) - } else { - // switch between two methods of getting the token. - // start two async processes, 10 minute timeout and get token from browser. Whichever finishes first, use that. - let timeout_future = tokio::time::sleep(Duration::from_secs(60 * 10)); - tokio::select! { - _ = timeout_future => { - Ok(AuthorizationState::TimedOut) - }, - token = self.get_token_from_browser(&env) => { - token - } - } - } - } - }?; - - match auth_state { - AuthorizationState::TokenSuccess(token) => { - match wasmer_registry::login::login_and_save_token( - env.dir(), - registry.as_str(), - &token, - )? { - Some(s) => { - print!("Done!"); - println!("\n✅ Login for Wasmer user {:?} saved", s) - } - None => print!( - "Warning: no user found on {:?} with the provided token.\nToken saved regardless.", - registry.domain().unwrap_or("registry.wasmer.io") - ), - }; - } - AuthorizationState::TimedOut => { - print!("Timed out (10 mins exceeded)"); - } - AuthorizationState::Cancelled => { - println!("Cancelled by the user"); - } - AuthorizationState::UnknownMethod => { - println!("Error: unknown method\n"); - } - }; - Ok(()) - } -} - -async fn preflight(req: Request) -> Result, anyhow::Error> { - let _whole_body = hyper::body::aggregate(req).await?; - let response = Response::builder() - .status(StatusCode::OK) - .header("Access-Control-Allow-Origin", "*") // FIXME: this is not secure, Don't allow all origins. @syrusakbary - .header("Access-Control-Allow-Headers", "Content-Type") - .header("Access-Control-Allow-Methods", "POST, OPTIONS") - .body(Body::default())?; - Ok(response) -} - -async fn handle_post_save_token( - context: AppContext, - req: Request, -) -> Result, anyhow::Error> { - let AppContext { - server_shutdown_tx, - token_tx, - } = context; - let (.., body) = req.into_parts(); - let body = hyper::body::to_bytes(body).await?; - - let ValidatedNonceOutput { - token, - status: token_status, - } = serde_json::from_slice::(&body)?; - - // send the AuthorizationState based on token_status to the main thread and get the response message - let (response_message, parse_failure) = match token_status { - TokenStatus::Cancelled => { - token_tx - .send(AuthorizationState::Cancelled) - .await - .expect("Failed to send token"); - - ("Token Cancelled by the user", false) - } - TokenStatus::Authorized => { - if let Some(token) = token { - token_tx - .send(AuthorizationState::TokenSuccess(token.clone())) - .await - .expect("Failed to send token"); - ("Token Authorized", false) - } else { - ("Token not found", true) - } - } - }; - - server_shutdown_tx - .send(true) - .await - .expect("Failed to send shutdown signal"); - - let status = if parse_failure { - StatusCode::BAD_REQUEST - } else { - StatusCode::OK - }; - - Ok(Response::builder() - .status(status) - .header("Access-Control-Allow-Origin", "*") // FIXME: this is not secure, Don't allow all origins. @syrusakbary - .header("Access-Control-Allow-Headers", "Content-Type") - .header("Access-Control-Allow-Methods", "POST, OPTIONS") - .body(Body::from(response_message))?) -} - -async fn handle_unknown_method(context: AppContext) -> Result, anyhow::Error> { - let AppContext { - server_shutdown_tx, - token_tx, - } = context; - - token_tx - .send(AuthorizationState::UnknownMethod) - .await - .expect("Failed to send token"); - - server_shutdown_tx - .send(true) - .await - .expect("Failed to send shutdown signal"); - - Ok(Response::builder() - .status(StatusCode::METHOD_NOT_ALLOWED) - .body(Body::from("Method not allowed"))?) -} - -/// Handle the preflight headers first - OPTIONS request -/// Then proceed to handle the actual request - POST request -async fn service_router( - context: AppContext, - req: Request, -) -> Result, anyhow::Error> { - match *req.method() { - Method::OPTIONS => preflight(req).await, - Method::POST => handle_post_save_token(context, req).await, - _ => handle_unknown_method(context).await, - } -} - -#[cfg(test)] -mod tests { - use clap::CommandFactory; - use tempfile::TempDir; - - use super::*; - - #[test] - fn interactive_login() { - let temp = TempDir::new().unwrap(); - let login = Login { - no_browser: true, - registry: Some("wasmer.wtf".into()), - wasmer_dir: temp.path().to_path_buf(), - token: None, - cache_dir: None, - }; - let env = login.wasmer_env(); - - let token = login.get_token_or_ask_user(&env).unwrap(); - match token { - AuthorizationState::TokenSuccess(token) => { - assert_eq!( - token, - "Please paste the login token from https://wasmer.wtf/settings/access-tokens" - ); - } - AuthorizationState::Cancelled - | AuthorizationState::TimedOut - | AuthorizationState::UnknownMethod => { - panic!("Should not reach here") - } - } - } - - #[test] - fn login_with_token() { - let temp = TempDir::new().unwrap(); - let login = Login { - no_browser: true, - registry: Some("wasmer.wtf".into()), - wasmer_dir: temp.path().to_path_buf(), - token: Some("abc".to_string()), - cache_dir: None, - }; - let env = login.wasmer_env(); - - let token = login.get_token_or_ask_user(&env).unwrap(); - - match token { - AuthorizationState::TokenSuccess(token) => { - assert_eq!(token, "abc"); - } - AuthorizationState::Cancelled - | AuthorizationState::TimedOut - | AuthorizationState::UnknownMethod => { - panic!("Should not reach here") - } - } - } - - #[test] - fn in_sync_with_wasmer_env() { - let wasmer_env = WasmerEnv::command(); - let login = Login::command(); - - // All options except --token should be the same - let wasmer_env_opts: Vec<_> = wasmer_env - .get_opts() - .filter(|arg| arg.get_id() != "token") - .collect(); - let login_opts: Vec<_> = login.get_opts().collect(); - - assert_eq!(wasmer_env_opts, login_opts); - - // The token argument should have the same message, even if it is an - // argument rather than a --flag. - let wasmer_env_token_help = wasmer_env - .get_opts() - .find(|arg| arg.get_id() == "token") - .unwrap() - .get_help() - .unwrap() - .to_string(); - let login_token_help = login - .get_positionals() - .find(|arg| arg.get_id() == "token") - .unwrap() - .get_help() - .unwrap() - .to_string(); - assert_eq!(wasmer_env_token_help, login_token_help); - } -} diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 3a887b584d1..23fe11ede95 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -256,9 +256,11 @@ impl Wasi { let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(handle))); if self.networking { - rt.set_networking_implementation(virtual_net::host::LocalNetworking::default()); + rt.set_networking_implementation(wasmer_virtual_net::host::LocalNetworking::default()); } else { - rt.set_networking_implementation(virtual_net::UnsupportedVirtualNetworking::default()); + rt.set_networking_implementation( + wasmer_virtual_net::UnsupportedVirtualNetworking::default(), + ); } if !self.no_tty { diff --git a/lib/virtual-io/Cargo.toml b/lib/virtual-io/Cargo.toml index 48cdf577209..52b8c2f3a0a 100644 --- a/lib/virtual-io/Cargo.toml +++ b/lib/virtual-io/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "virtual-io" +name = "wasmer-virtual-io" version = "0.1.0" description = "Wasmer Virtual IO Engine" authors.workspace = true diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index ddf3f80e12c..6303fee32fd 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "virtual-net" +name = "wasmer-virtual-net" version = "0.3.0" description = "Wasmer Virtual Networking" authors.workspace = true @@ -19,23 +19,18 @@ libc = { version = "0.2.139", optional = true } mio = { version = "0.8", optional = true } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } -virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } -tun-tap = { version = "0.1.3", features = [ "tokio" ], optional = true } -tokio-tungstenite = { version = "0.18.0", optional = true } +wasmer-virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } base64 = "0.21" bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } pin-project-lite = "0.2.9" futures-util = { version = "0.3" } -anyhow = { version = "1.0", optional = true } -futures = { version = "0.3", optional = true } [features] -default = [ "host-net", "remote", "tun-tap" ] -host-net = [ "tokio", "libc", "virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] +default = [ "host-net", "remote" ] +host-net = [ "tokio", "libc", "wasmer-virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] remote = [ "tokio", "libc", "tokio/io-util", "tokio/sync" ] -tun-tap = [ "remote", "host-net", "dep:tun-tap", "tokio-tungstenite", "anyhow", "futures" ] [package.metadata.docs.rs] -features = ["host-net", "remote", "tun-tap"] +features = ["host-net", "remote"] rustc-args = ["--cfg", "docsrs"] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 5fa5dc98df3..5753b44bac2 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -31,8 +31,8 @@ use tokio::sync::mpsc; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::mpsc::error::TrySendError; use tokio::sync::oneshot; -use virtual_io::InlineWaker; -use virtual_io::InterestType; +use wasmer_virtual_io::InlineWaker; +use wasmer_virtual_io::InterestType; use crate::io_err_into_net_error; use crate::meta; @@ -632,7 +632,7 @@ struct RemoteCommon { recv_tx: Mutex>>>, recv_with_addr_tx: Mutex, SocketAddr)>>>, #[derivative(Debug = "ignore")] - handlers: Mutex>>, + handlers: Mutex>>, // The stall guard will prevent reads while its held and there are background tasks running // (the idea behind this is to create back pressure so that the task list infinitely grow) @@ -1022,7 +1022,7 @@ impl VirtualSocket for RemoteSocket { fn set_handler( &mut self, - handler: Box, + handler: Box, ) -> Result<()> { self.common .handlers @@ -1065,7 +1065,7 @@ impl VirtualTcpListener for RemoteSocket { fn set_handler( &mut self, - handler: Box, + handler: Box, ) -> Result<()> { VirtualSocket::set_handler(self, handler) } diff --git a/lib/virtual-net/src/host.rs b/lib/virtual-net/src/host.rs index 2ad8f28472e..4812210f507 100644 --- a/lib/virtual-net/src/host.rs +++ b/lib/virtual-net/src/host.rs @@ -1,5 +1,5 @@ #![allow(unused_variables)] -use crate::{VirtualIoSource, io_err_into_net_error}; +use crate::{io_err_into_net_error, VirtualIoSource}; #[allow(unused_imports)] use crate::{ IpCidr, IpRoute, NetworkError, Result, SocketStatus, StreamSecurity, VirtualConnectedSocket, @@ -15,7 +15,7 @@ use std::time::Duration; use tokio::runtime::Handle; #[allow(unused_imports, dead_code)] use tracing::{debug, error, info, trace, warn}; -use virtual_io::{InterestGuard, InterestHandler, Selector}; +use wasmer_virtual_io::{InterestGuard, InterestHandler, Selector}; #[derive(Derivative)] #[derivative(Debug)] @@ -313,7 +313,7 @@ impl VirtualSocket for LocalTcpStream { if self.first_handler_writeable { self.first_handler_writeable = false; - handler.interest(virtual_io::InterestType::Writable); + handler.interest(wasmer_virtual_io::InterestType::Writable); } let guard = InterestGuard::new( diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index d1333df7318..6e9f232d61f 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -7,9 +7,6 @@ pub mod host; pub mod meta; #[cfg(any(feature = "remote"))] pub mod server; -#[cfg(target_os = "linux")] -#[cfg(feature = "tun-tap")] -pub mod tun; #[cfg(any(feature = "remote"))] pub use client::{RemoteNetworking, RemoteNetworkingDriver}; @@ -25,16 +22,13 @@ pub use std::net::SocketAddr; use std::sync::Arc; pub use std::time::Duration; use thiserror::Error; -#[cfg(target_os = "linux")] -#[cfg(feature = "tun-tap")] -pub use tun::{TunTapDriver, TunTapSocket}; pub use bytes::Bytes; pub use bytes::BytesMut; use serde::{Deserialize, Serialize}; -pub use virtual_io::{handler_into_waker, InterestHandler}; +pub use wasmer_virtual_io::{handler_into_waker, InterestHandler}; #[cfg(feature = "host-net")] -pub use virtual_io::{InterestGuard, InterestHandlerWaker, InterestType}; +pub use wasmer_virtual_io::{InterestGuard, InterestHandlerWaker, InterestType}; pub type Result = std::result::Result; diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index 8ef8481921d..d32fd2d1b0b 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -32,14 +32,6 @@ pub enum Shutdown { Both, } -#[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -pub struct Hello { - /// The network that we will connect to - pub network_hash: [u8; 16], - /// The encryption key that authorizes the connection - pub encrypt_key: [u8; 32], -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub enum RequestType { /// Bridges this local network with a remote network, which is required in diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index 416593be814..30dce504828 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -25,7 +25,7 @@ use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, }; -use virtual_io::InterestHandler; +use wasmer_virtual_io::InterestHandler; type BackgroundTask = Option>; @@ -1273,7 +1273,7 @@ impl RemoteAdapterHandler { } } impl InterestHandler for RemoteAdapterHandler { - fn interest(&mut self, interest: virtual_io::InterestType) { + fn interest(&mut self, interest: wasmer_virtual_io::InterestType) { let mut guard = self.state.lock().unwrap(); guard.driver_wakers.drain(..).for_each(|w| w.wake()); let socket_id = match self.socket_id.clone() { @@ -1281,7 +1281,7 @@ impl InterestHandler for RemoteAdapterHandler { None => return, }; match interest { - virtual_io::InterestType::Readable => { + wasmer_virtual_io::InterestType::Readable => { guard.readable.insert(socket_id); } _ => {} diff --git a/lib/virtual-net/src/tun.rs b/lib/virtual-net/src/tun.rs deleted file mode 100644 index 691b18c182b..00000000000 --- a/lib/virtual-net/src/tun.rs +++ /dev/null @@ -1,298 +0,0 @@ -use std::collections::VecDeque; -use std::io::{self, ErrorKind}; -use std::mem::MaybeUninit; -use std::ops::DerefMut; -use std::os::fd::{AsRawFd, RawFd}; -use std::pin::Pin; -use std::process::Command; -use std::sync::{Arc, Mutex}; -use std::task::{Context, Poll, Waker}; - -use bytes::Bytes; -use futures_util::Future; -use mio::unix::SourceFd; -use mio::{event, Interest, Registry, Token}; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; -use tokio::net::TcpStream; -use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; -use tun_tap::{Iface, Mode}; -use virtual_io::{InterestGuard, InterestHandler, Selector}; - -use crate::meta::Hello; -use crate::{ - io_err_into_net_error, NetworkError, RemoteNetworking, RemoteNetworkingDriver, - VirtualNetworking, VirtualRawSocket, -}; - -fn cmd(cmd: &str, args: &[&str]) -> anyhow::Result<()> { - let ecode = Command::new(cmd).args(args).spawn()?.wait()?; - if ecode.success() { - Ok(()) - } else { - Err(anyhow::format_err!( - "failed to allocate IP address (code={:?})", - ecode.code() - )) - } -} - -#[derive(Debug, Clone)] -struct TungstensiteAdapter { - stream: Arc>>>, -} -impl TungstensiteAdapter { - fn new(stream: WebSocketStream>) -> Self { - Self { - stream: Arc::new(Mutex::new(stream)), - } - } - fn split(self) -> (Self, Self) { - (self.clone(), self.clone()) - } -} -impl AsyncRead for TungstensiteAdapter { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut tokio::io::ReadBuf<'_>, - ) -> Poll> { - let mut guard = self.stream.lock().unwrap(); - Pin::new(guard.deref_mut().get_mut()).poll_read(cx, buf) - } -} -impl AsyncWrite for TungstensiteAdapter { - fn poll_write_vectored( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - bufs: &[io::IoSlice<'_>], - ) -> Poll> { - let mut guard = self.stream.lock().unwrap(); - Pin::new(guard.deref_mut().get_mut()).poll_write_vectored(cx, bufs) - } - - fn is_write_vectored(&self) -> bool { - let mut guard = self.stream.lock().unwrap(); - Pin::new(guard.deref_mut().get_mut()).is_write_vectored() - } - - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - let mut guard = self.stream.lock().unwrap(); - Pin::new(guard.deref_mut().get_mut()).poll_write(cx, buf) - } - - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut guard = self.stream.lock().unwrap(); - Pin::new(guard.deref_mut().get_mut()).poll_flush(cx) - } - - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut guard = self.stream.lock().unwrap(); - Pin::new(guard.deref_mut().get_mut()).poll_shutdown(cx) - } -} - -pub struct TunTapSocket {} -impl TunTapSocket { - pub async fn create( - selector: Arc, - url: &str, - hello: Hello, - ) -> anyhow::Result { - // Create the remote client - let (stream, _res) = tokio_tungstenite::connect_async(url).await?; - let (mut tx, rx) = TungstensiteAdapter::new(stream).split(); - - // Send the hello message - tx.write_all(&bincode::serialize(&hello)?).await?; - - // Now pass it on to a remote networking adapter - let (remote, remote_driver) = - RemoteNetworking::new_from_stream_via_driver(Box::pin(tx), Box::pin(rx)); - let mut client = remote.bind_raw().await?; - - let iface = Iface::new("wasmer%d", Mode::Tun)?; - cmd("ip", &["link", "set", "up", "dev", iface.name()])?; - - // Set non-blocking and wrap the MIO - set_non_blocking(&iface)?; - let handler = TunTapHandler::default(); - let mut source = MioWrapper { - fd: iface.as_raw_fd(), - }; - - // Register interest in the read and write events - let interest = InterestGuard::new( - &selector, - Box::new(handler.clone()), - &mut source, - mio::Interest::READABLE.add(mio::Interest::WRITABLE), - ) - .map_err(io_err_into_net_error)?; - - // Set the client handler - client.set_handler(Box::new(handler.clone()))?; - - let driver = TunTapDriver { - iface, - handler, - send_queue: Default::default(), - _interest: interest, - client, - remote_driver, - }; - - Ok(driver) - } -} - -fn set_non_blocking(iface: &Iface) -> anyhow::Result<()> { - let fd = iface.as_raw_fd(); - let mut nonblock: libc::c_int = 1; - let result = unsafe { libc::ioctl(fd, libc::FIONBIO, &mut nonblock) }; - if result == -1 { - Err(std::io::Error::last_os_error().into()) - } else { - Ok(()) - } -} - -#[derive(Debug, Default)] -struct TunTapHandlerInner { - wakers: Vec, -} - -#[derive(Debug, Clone, Default)] -struct TunTapHandler { - inner: Arc>, -} -impl InterestHandler for TunTapHandler { - fn interest(&mut self, _interest: virtual_io::InterestType) { - let mut guard = self.inner.lock().unwrap(); - guard.wakers.drain(..).for_each(Waker::wake); - } -} - -pub struct TunTapDriver { - iface: Iface, - handler: TunTapHandler, - send_queue: VecDeque, - _interest: InterestGuard, - client: Box, - remote_driver: RemoteNetworkingDriver, -} - -impl Future for TunTapDriver { - type Output = io::Result<()>; - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - // Run the remote networking - loop { - match Pin::new(&mut self.remote_driver).poll(cx) { - Poll::Ready(()) => return Poll::Ready(Ok(())), - Poll::Pending => break, - } - } - - // Add the waker before we drain all the the events - // we need to read and send - let inner = self.handler.inner.clone(); - let mut guard = inner.lock().unwrap(); - if guard.wakers.iter().any(|w| w.will_wake(cx.waker())) == false { - guard.wakers.push(cx.waker().clone()); - } - - // First we drain the packet - while let Some(packet) = self.send_queue.pop_front() { - if self.client.try_send(&packet).is_ok() == false { - self.send_queue.push_front(packet); - break; - } - } - - // Now we drain all the packets from the interface (but only if we have room) - if self.send_queue.is_empty() { - loop { - let mut chunk: [MaybeUninit; 10240] = - unsafe { MaybeUninit::uninit().assume_init() }; - let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..]; - let chunk_unsafe: &mut [u8] = unsafe { std::mem::transmute(chunk_unsafe) }; - - match self.iface.recv(chunk_unsafe) { - Ok(0) => return Poll::Ready(Err(ErrorKind::ConnectionReset.into())), - Ok(data) => { - // Send the packet down the client - let packet = &chunk_unsafe[..data]; - if self.client.try_send(packet).is_ok() == false { - self.send_queue.push_back(Bytes::copy_from_slice(packet)); - break; - } - continue; - } - Err(err) if err.kind() == ErrorKind::WouldBlock => { - break; - } - Err(err) => return Poll::Ready(Err(err)), - }; - } - } - - // Drain the client into the interfacee - loop { - let mut chunk: [MaybeUninit; 10240] = - unsafe { MaybeUninit::uninit().assume_init() }; - let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..]; - - match self.client.try_recv(chunk_unsafe) { - Ok(0) => return Poll::Ready(Err(ErrorKind::ConnectionReset.into())), - Ok(data) => { - // Send the packet down the client - let chunk_unsafe: &mut [u8] = unsafe { std::mem::transmute(chunk_unsafe) }; - let packet = &chunk_unsafe[..data]; - self.iface.send(packet).ok(); - continue; - } - Err(NetworkError::WouldBlock) => { - break; - } - Err(err) => { - tracing::error!("packet recv error - {}", err); - return Poll::Ready(Err(ErrorKind::Other.into())); - } - } - } - - // Wait for some more interest or something to send - Poll::Pending - } -} - -struct MioWrapper { - fd: RawFd, -} - -impl event::Source for MioWrapper { - fn register( - &mut self, - registry: &Registry, - token: Token, - interests: Interest, - ) -> io::Result<()> { - SourceFd(&self.fd).register(registry, token, interests) - } - - fn reregister( - &mut self, - registry: &Registry, - token: Token, - interests: Interest, - ) -> io::Result<()> { - SourceFd(&self.fd).reregister(registry, token, interests) - } - - fn deregister(&mut self, registry: &Registry) -> io::Result<()> { - SourceFd(&self.fd).deregister(registry) - } -} diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 2e50d9d00af..6a71e54a5a9 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -2047,7 +2047,7 @@ dependencies = [ ] [[package]] -name = "virtual-io" +name = "wasmer-virtual-io" version = "0.1.0" dependencies = [ "async-trait", @@ -2060,7 +2060,7 @@ dependencies = [ ] [[package]] -name = "virtual-net" +name = "wasmer-virtual-net" version = "0.3.0" dependencies = [ "async-trait", @@ -2073,7 +2073,7 @@ dependencies = [ "serde", "thiserror", "tracing", - "virtual-io", + "wasmer-virtual-io", ] [[package]] @@ -2462,8 +2462,8 @@ dependencies = [ "url", "urlencoding", "virtual-fs", - "virtual-io", - "virtual-net", + "wasmer-virtual-io", + "wasmer-virtual-net", "wai-bindgen-wasmer", "waker-fn", "wasm-bindgen", diff --git a/lib/wasi-web/src/runtime.rs b/lib/wasi-web/src/runtime.rs index 586633dde7f..7dce9200f1f 100644 --- a/lib/wasi-web/src/runtime.rs +++ b/lib/wasi-web/src/runtime.rs @@ -57,7 +57,7 @@ pub(crate) struct WebRuntime { package_loader: Arc, source: Arc, module_cache: Arc, - net: wasmer_wasix::virtual_net::DynVirtualNetworking, + net: wasmer_wasix::wasmer_virtual_net::DynVirtualNetworking, tasks: Arc, } @@ -426,7 +426,7 @@ impl VirtualFile for TermLog { } impl wasmer_wasix::Runtime for WebRuntime { - fn networking(&self) -> &wasmer_wasix::virtual_net::DynVirtualNetworking { + fn networking(&self) -> &wasmer_wasix::wasmer_virtual_net::DynVirtualNetworking { &self.net } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index bcd9544bf28..fcc1d35cd86 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -20,9 +20,9 @@ getrandom = "0.2" wasmer-wasix-types = { path = "../wasi-types", version = "0.9.0", features = [ "enable-serde" ] } wasmer-types = { path = "../types", version = "=4.0.0", default-features = false } wasmer = { path = "../api", version = "=4.0.0", default-features = false, features = ["wat", "js-serializable-module"] } -virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } +wasmer-virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } virtual-fs = { path = "../virtual-fs", version = "0.6.0", default-features = false, features = ["webc-fs"] } -virtual-net = { path = "../virtual-net", version = "0.3.0", default-features = false } +wasmer-virtual-net = { path = "../virtual-net", version = "0.3.0", default-features = false } wasmer-emscripten = { path = "../emscripten", version = "=4.0.0", optional = true } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"] } @@ -128,11 +128,11 @@ js = ["virtual-fs/no-time", "getrandom/js", "chrono"] js-default = ["js"] test-js = ["js", "wasmer/wat"] -host-vnet = ["virtual-net/host-net"] +host-vnet = ["wasmer-virtual-net/host-net"] host-threads = [] host-reqwest = ["reqwest"] host-fs = ["virtual-fs/host-fs"] -remote-vnet = ["virtual-net/remote"] +remote-vnet = ["wasmer-virtual-net/remote"] logging = ["tracing/log"] disable-all-logging = ["tracing/release_max_level_off", "tracing/max_level_off"] diff --git a/lib/wasix/src/fs/inode_guard.rs b/lib/wasix/src/fs/inode_guard.rs index 33344f14e75..97bb6f4a755 100644 --- a/lib/wasix/src/fs/inode_guard.rs +++ b/lib/wasix/src/fs/inode_guard.rs @@ -11,8 +11,8 @@ use std::{ use futures::future::BoxFuture; use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite}; use virtual_fs::{FsError, Pipe as VirtualPipe, VirtualFile}; -use virtual_io::{InterestType, StatefulHandler}; -use virtual_net::NetworkError; +use wasmer_virtual_io::{InterestType, StatefulHandler}; +use wasmer_virtual_net::NetworkError; use wasmer_wasix_types::{ types::Eventtype, wasi::{self, EpollType}, diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 3ea2b4cd836..beef7f1869a 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -70,12 +70,13 @@ use wasmer::{ pub use virtual_fs; pub use virtual_fs::{DuplexPipe, FsError, Pipe, VirtualFile, WasiBidirectionalSharedPipePair}; -pub use virtual_net; -pub use virtual_net::{UnsupportedVirtualNetworking, VirtualNetworking}; +pub use wasmer_virtual_net; +pub use wasmer_virtual_net::{UnsupportedVirtualNetworking, VirtualNetworking}; #[cfg(feature = "host-vnet")] -pub use virtual_net::host::{ - io_err_into_net_error, LocalNetworking, LocalTcpListener, LocalTcpStream, LocalUdpSocket, +pub use wasmer_virtual_net::{ + host::{LocalNetworking, LocalTcpListener, LocalTcpStream, LocalUdpSocket}, + io_err_into_net_error, }; use wasmer_wasix_types::wasi::{Errno, ExitCode}; diff --git a/lib/wasix/src/net/mod.rs b/lib/wasix/src/net/mod.rs index a59d71508cf..7845d22f2de 100644 --- a/lib/wasix/src/net/mod.rs +++ b/lib/wasix/src/net/mod.rs @@ -4,9 +4,9 @@ use std::{ time::Duration, }; -use virtual_net::{IpCidr, IpRoute, NetworkError}; use wasmer::{MemoryView, WasmPtr}; use wasmer_types::MemorySize; +use wasmer_virtual_net::{IpCidr, IpRoute, NetworkError}; use wasmer_wasix_types::{ types::{ OptionTag, OptionTimestamp, Route, __wasi_addr_ip4_t, __wasi_addr_ip6_t, diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index fc6872d1bf5..38c9c7fff90 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -10,15 +10,15 @@ use std::{ #[cfg(feature = "enable-serde")] use serde_derive::{Deserialize, Serialize}; -use virtual_io::{ +use wasmer_types::MemorySize; +use wasmer_virtual_io::{ FilteredHandler, FilteredHandlerSubscriptions, InterestHandler, InterestType, StatefulHandlerState, }; -use virtual_net::{ +use wasmer_virtual_net::{ NetworkError, VirtualIcmpSocket, VirtualNetworking, VirtualRawSocket, VirtualTcpListener, VirtualTcpSocket, VirtualUdpSocket, }; -use wasmer_types::MemorySize; use wasmer_wasix_types::wasi::{Addressfamily, Errno, Rights, SockProto, Sockoption, Socktype}; use crate::{net::net_error_into_wasi_err, VirtualTaskManager}; @@ -1266,7 +1266,9 @@ impl InodeSocketProtected { InodeSocketKind::UdpSocket { socket, .. } => socket.set_handler(handler), InodeSocketKind::Raw(socket) => socket.set_handler(handler), InodeSocketKind::Icmp(socket) => socket.set_handler(handler), - InodeSocketKind::PreSocket { .. } => Err(virtual_net::NetworkError::NotConnected), + InodeSocketKind::PreSocket { .. } => { + Err(wasmer_virtual_net::NetworkError::NotConnected) + } } } @@ -1274,7 +1276,7 @@ impl InodeSocketProtected { &mut self, handler: Box, interest: InterestType, - ) -> virtual_net::Result<()> { + ) -> wasmer_virtual_net::Result<()> { if self.aggregate_handler.is_none() { let upper = FilteredHandler::new(); let subs = upper.subscriptions().clone(); diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 3cb1b552ffc..4f8268fad7b 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -16,8 +16,8 @@ use std::{ use derivative::Derivative; use futures::future::BoxFuture; -use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; +use wasmer_virtual_net::{DynVirtualNetworking, VirtualNetworking}; use crate::{ http::{DynHttpClient, HttpClient}, @@ -187,9 +187,9 @@ impl PluggableRuntime { // TODO: the cfg flags below should instead be handled by separate implementations. cfg_if::cfg_if! { if #[cfg(feature = "host-vnet")] { - let networking = Arc::new(virtual_net::host::LocalNetworking::default()); + let networking = Arc::new(wasmer_virtual_net::host::LocalNetworking::default()); } else { - let networking = Arc::new(virtual_net::UnsupportedVirtualNetworking::default()); + let networking = Arc::new(wasmer_virtual_net::UnsupportedVirtualNetworking::default()); } } let http_client = diff --git a/lib/wasix/src/runtime/task_manager/mod.rs b/lib/wasix/src/runtime/task_manager/mod.rs index c69c43dab81..40fd1d5dbad 100644 --- a/lib/wasix/src/runtime/task_manager/mod.rs +++ b/lib/wasix/src/runtime/task_manager/mod.rs @@ -16,7 +16,7 @@ use crate::os::task::thread::WasiThreadError; use crate::syscalls::AsyncifyFuture; use crate::{capture_snapshot, InstanceSnapshot, WasiEnv, WasiFunctionEnv, WasiThread}; -pub use virtual_io::waker::*; +pub use wasmer_virtual_io::waker::*; #[derive(Debug)] pub enum SpawnMemoryType<'a> { diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index b8a895302fb..c1a4314c924 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -11,11 +11,11 @@ use futures::future::BoxFuture; use rand::Rng; use tracing::{trace, warn}; use virtual_fs::{FileSystem, FsError, StaticFile, VirtualFile}; -use virtual_net::DynVirtualNetworking; use wasmer::{ AsStoreMut, AsStoreRef, FunctionEnvMut, Global, Instance, Memory, MemoryType, MemoryView, Module, TypedFunction, }; +use wasmer_virtual_net::DynVirtualNetworking; use wasmer_wasix_types::{ types::Signal, wasi::{Errno, ExitCode, Snapshot0Clockid}, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index ef24583bafd..62659aa259d 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -69,12 +69,12 @@ pub use wasm::*; pub(crate) use virtual_fs::{ AsyncSeekExt, AsyncWriteExt, DuplexPipe, FileSystem, FsError, VirtualFile, }; -pub(crate) use virtual_net::StreamSecurity; pub(crate) use wasmer::{ AsStoreMut, AsStoreRef, Extern, Function, FunctionEnv, FunctionEnvMut, Global, Instance, Memory, Memory32, Memory64, MemoryAccessError, MemoryError, MemorySize, MemoryView, Module, OnCalledAction, Pages, RuntimeError, Store, TypedFunction, Value, WasmPtr, WasmSlice, }; +pub(crate) use wasmer_virtual_net::StreamSecurity; pub(crate) use wasmer_wasix_types::{asyncify::__wasi_asyncify_t, wasi::EventUnion}; #[cfg(any(target_os = "windows"))] pub use windows::*; diff --git a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs index 98cc6dff8b1..c0456cfd51e 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; use tokio::sync::{mpsc::UnboundedSender, watch}; -use virtual_io::{InterestHandler, InterestType}; +use wasmer_virtual_io::{InterestHandler, InterestType}; use wasmer_wasix_types::wasi::{ EpollCtl, EpollEvent, EpollType, SubscriptionClock, SubscriptionUnion, Userdata, }; From 5c5f7ec30f7f8950f28b237ea72164b575363dd5 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 18 Jul 2023 18:29:34 +1000 Subject: [PATCH 16/52] Fixed an issue with the borrow checker on async calls to remote networking --- lib/virtual-net/src/client.rs | 128 +++++++++++++++---------------- lib/virtual-net/src/lib.rs | 1 + lib/virtual-net/src/server.rs | 4 +- lib/virtual-net/src/write_all.rs | 69 +++++++++++++++++ 4 files changed, 135 insertions(+), 67 deletions(-) create mode 100644 lib/virtual-net/src/write_all.rs diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 5753b44bac2..ba667c8d57f 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -17,6 +17,7 @@ use std::task::Waker; use std::time::Duration; use bytes::Buf; +use bytes::Bytes; use bytes::BytesMut; use derivative::Derivative; use futures_util::future::BoxFuture; @@ -25,7 +26,6 @@ use futures_util::StreamExt; use serde::Serialize; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; -use tokio::io::AsyncWriteExt; use tokio::io::ReadBuf; use tokio::sync::mpsc; use tokio::sync::mpsc::error::TryRecvError; @@ -55,6 +55,8 @@ use crate::VirtualTcpListener; use crate::VirtualTcpSocket; use crate::VirtualUdpSocket; +use crate::write_all::locking_write_all; +use crate::write_all::locking_write_all_many; use crate::Result; #[derive(Debug, Clone)] @@ -109,15 +111,15 @@ impl RemoteNetworking { /// This version will run the async read and write operations /// only the driver (this is needed for mixed runtimes) pub fn new_from_stream_via_driver( - tx: Pin>, - rx: Pin>, + tx: Pin>, + rx: Pin>, ) -> (Self, RemoteNetworkingDriver) { Self::new_from_stream_internal(tx, rx, true) } fn new_from_stream_internal( - tx: Pin>, - rx: Pin>, + tx: Pin>, + rx: Pin>, via_driver: bool, ) -> (Self, RemoteNetworkingDriver) { let (tx_work, rx_work) = mpsc::unbounded_channel(); @@ -125,12 +127,12 @@ impl RemoteNetworking { let common = RemoteCommon { tx: if via_driver { RemoteTx::StreamViaDriver { - tx: Arc::new(tokio::sync::Mutex::new(tx)), + tx: Arc::new(Mutex::new(tx)), work: tx_work, } } else { RemoteTx::Stream { - tx: tokio::sync::Mutex::new(tx), + tx: Arc::new(Mutex::new(tx)), } }, rx: Mutex::new(RemoteRx::Stream { @@ -325,17 +327,17 @@ impl Future for RemoteNetworkingDriver { } } -pub(crate) type RemoteTxStream = Pin>; +pub(crate) type RemoteTxStream = Pin>; pub(crate) enum RemoteTx where T: Serialize, { Mpsc(mpsc::Sender), Stream { - tx: tokio::sync::Mutex, + tx: Arc>, }, StreamViaDriver { - tx: Arc>, + tx: Arc>, work: mpsc::UnboundedSender>, }, } @@ -350,34 +352,26 @@ where .await .map_err(|_| NetworkError::ConnectionAborted), RemoteTx::Stream { tx, .. } => { - let mut tx = tx.lock().await; let data = bincode::serialize(&req).map_err(|err| { tracing::warn!("failed to serialize message - {}", err); NetworkError::IOError })?; - let data_len = data.len() as u64; - let data_len_buf = data_len.to_le_bytes(); - tx.write_all(&data_len_buf) + locking_write_all(tx, data.into()) .await - .map_err(io_err_into_net_error)?; - tx.write_all(&data).await.map_err(io_err_into_net_error) + .map_err(io_err_into_net_error) } RemoteTx::StreamViaDriver { tx, work, .. } => { let (tx_done, rx_done) = oneshot::channel(); let tx = tx.clone(); work.send(Box::pin(async move { let job = async { - let mut tx = tx.lock().await; let data = bincode::serialize(&req).map_err(|err| { tracing::warn!("failed to serialize message - {}", err); NetworkError::IOError })?; - let data_len = data.len() as u64; - let data_len_buf = data_len.to_le_bytes(); - tx.write_all(&data_len_buf) + locking_write_all(&tx, data.into()) .await - .map_err(io_err_into_net_error)?; - tx.write_all(&data).await.map_err(io_err_into_net_error) + .map_err(io_err_into_net_error) }; tx_done.send(job.await).ok(); })) @@ -404,30 +398,36 @@ where let data_len = data.len() as u64; let data_len_buf = data_len.to_le_bytes(); - let mut tx = InlineWaker::block_on(tx.lock()); - let data_len_buf_left = + let data_len_remaining = { + let mut tx = tx.lock().unwrap(); match Pin::new(tx.deref_mut()).poll_write(interest_cx, &data_len_buf) { Poll::Ready(Ok(0)) => { return Err(NetworkError::ConnectionAborted); } - Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => 0, - Poll::Ready(Ok(amt)) => data_len_buf.len() - amt, + Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => None, + Poll::Ready(Ok(amt)) => { + let remaining = &data_len_buf[amt..]; + Some(remaining.to_vec()) + } Poll::Ready(Err(err)) => { return Err(io_err_into_net_error(err)); } Poll::Pending => { return Err(NetworkError::WouldBlock); } - }; + } + }; + + let bufs = if let Some(data_len_remaining) = data_len_remaining { + vec![Bytes::from(data_len_remaining), Bytes::from(data)] + } else { + vec![Bytes::from(data)] + }; InlineWaker::block_on(Box::pin(async move { - if data_len_buf_left > 0 { - let offset = data_len_buf.len() - data_len_buf_left; - tx.write_all(&data_len_buf[offset..]) - .await - .map_err(io_err_into_net_error)?; - } - tx.write_all(&data).await.map_err(io_err_into_net_error) + locking_write_all_many(&tx, bufs) + .await + .map_err(io_err_into_net_error) })) } RemoteTx::StreamViaDriver { tx, work } => { @@ -438,40 +438,38 @@ where let data_len = data.len() as u64; let data_len_buf = data_len.to_le_bytes(); - let interest_waker = interest_cx.waker().clone(); + let data_len_remaining = { + let mut tx = tx.lock().unwrap(); + match Pin::new(tx.deref_mut()).poll_write(interest_cx, &data_len_buf) { + Poll::Ready(Ok(0)) => { + return Err(NetworkError::ConnectionAborted); + } + Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => None, + Poll::Ready(Ok(amt)) => { + let remaining = &data_len_buf[amt..]; + Some(remaining.to_vec()) + } + Poll::Ready(Err(err)) => { + return Err(io_err_into_net_error(err)); + } + Poll::Pending => { + return Err(NetworkError::WouldBlock); + } + } + }; + + let bufs = if let Some(data_len_remaining) = data_len_remaining { + vec![Bytes::from(data_len_remaining), Bytes::from(data)] + } else { + vec![Bytes::from(data)] + }; let (tx_done, rx_done) = std::sync::mpsc::channel(); let tx = tx.clone(); work.send(Box::pin(async move { - let job = async { - let mut tx = tx.lock().await; - - let mut interest_cx = Context::from_waker(&interest_waker); - let data_len_buf_left = match Pin::new(tx.deref_mut()) - .poll_write(&mut interest_cx, &data_len_buf) - { - Poll::Ready(Ok(0)) => { - return Err(NetworkError::ConnectionAborted); - } - Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => 0, - Poll::Ready(Ok(amt)) => data_len_buf.len() - amt, - Poll::Ready(Err(err)) => { - return Err(io_err_into_net_error(err)); - } - Poll::Pending => { - return Err(NetworkError::WouldBlock); - } - }; - - if data_len_buf_left > 0 { - let offset = data_len_buf.len() - data_len_buf_left; - tx.write_all(&data_len_buf[offset..]) - .await - .map_err(io_err_into_net_error)?; - } - tx.write_all(&data).await.map_err(io_err_into_net_error) - }; - let ret = job.await; + let ret = locking_write_all_many(&tx, bufs) + .await + .map_err(io_err_into_net_error); tx_done.send(ret).ok(); })) .map_err(|_| NetworkError::ConnectionAborted)?; @@ -535,7 +533,7 @@ where { Mpsc(mpsc::Receiver), Stream { - rx: Pin>, + rx: Pin>, next: Option, buf: BytesMut, }, diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index 6e9f232d61f..56a7b2e6f27 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -7,6 +7,7 @@ pub mod host; pub mod meta; #[cfg(any(feature = "remote"))] pub mod server; +pub mod write_all; #[cfg(any(feature = "remote"))] pub use client::{RemoteNetworking, RemoteNetworkingDriver}; diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index 30dce504828..e65dc82025a 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -97,12 +97,12 @@ impl RemoteNetworkingAdapter { let common = RemoteAdapterCommon { tx: if via_driver { RemoteTx::StreamViaDriver { - tx: Arc::new(tokio::sync::Mutex::new(tx)), + tx: Arc::new(Mutex::new(tx)), work: tx_work, } } else { RemoteTx::Stream { - tx: tokio::sync::Mutex::new(tx), + tx: Arc::new(Mutex::new(tx)), } }, rx: Mutex::new(RemoteRx::Stream { diff --git a/lib/virtual-net/src/write_all.rs b/lib/virtual-net/src/write_all.rs new file mode 100644 index 00000000000..880d2571ae1 --- /dev/null +++ b/lib/virtual-net/src/write_all.rs @@ -0,0 +1,69 @@ +use bytes::Buf; +use bytes::Bytes; +use pin_project_lite::pin_project; +use std::collections::VecDeque; +use std::future::Future; +use std::io; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::Mutex; +use std::task::{Context, Poll}; +use tokio::io::AsyncWrite; + +pin_project! { + #[must_use = "futures do nothing unless you `.await` or poll them"] + pub struct WriteAll { + writer: Arc>>>, + bufs: VecDeque, + } +} + +pub(crate) fn locking_write_all( + writer: &Arc>>>, + buf: Bytes, +) -> WriteAll { + WriteAll { + writer: writer.clone(), + bufs: vec![buf].into(), + } +} + +pub(crate) fn locking_write_all_many( + writer: &Arc>>>, + bufs: I, +) -> WriteAll +where + I: IntoIterator, +{ + WriteAll { + writer: writer.clone(), + bufs: bufs.into_iter().collect(), + } +} + +impl Future for WriteAll { + type Output = io::Result<()>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let me = self.project(); + + let mut writer = me.writer.lock().unwrap(); + while let Some(first) = me.bufs.front_mut() { + match writer.as_mut().poll_write(cx, &first) { + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Ready(Ok(0)) => { + return Poll::Ready(Err(io::ErrorKind::ConnectionAborted.into())) + } + Poll::Ready(Ok(amt)) => { + first.advance(amt); + if first.is_empty() { + me.bufs.pop_front(); + continue; + } + } + Poll::Pending => return Poll::Pending, + } + } + Poll::Ready(Ok(())) + } +} From e816d66083cbbada38ff9a154e7735100a25c5f2 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 19 Jul 2023 06:46:23 +1000 Subject: [PATCH 17/52] Finished the main client and server setup and added unit tests --- Cargo.lock | 107 +++- lib/cli/src/commands/run.rs | 2 +- lib/virtual-io/src/waker.rs | 4 +- lib/virtual-net/Cargo.toml | 9 +- lib/virtual-net/src/client.rs | 935 +++++++++++++------------------ lib/virtual-net/src/lib.rs | 336 ++++++++++- lib/virtual-net/src/meta.rs | 25 +- lib/virtual-net/src/rx_tx.rs | 241 ++++++++ lib/virtual-net/src/server.rs | 530 +++++++++++++----- lib/virtual-net/src/tests.rs | 154 +++++ lib/virtual-net/src/write_all.rs | 69 --- lib/wasix/src/fs/inode_guard.rs | 39 -- 12 files changed, 1640 insertions(+), 811 deletions(-) create mode 100644 lib/virtual-net/src/rx_tx.rs create mode 100644 lib/virtual-net/src/tests.rs delete mode 100644 lib/virtual-net/src/write_all.rs diff --git a/Cargo.lock b/Cargo.lock index 9852c9e30d5..13a2ecb2d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1311,6 +1311,18 @@ dependencies = [ "memmap2 0.5.10", ] +[[package]] +name = "educe" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "079044df30bb07de7d846d41a184c4b00e66ebdac93ee459253474f3a47e50ae" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "either" version = "1.8.1" @@ -1358,6 +1370,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "enum-ordinalize" +version = "3.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4f76552f53cefc9a7f64987c3701b99d982f7690606fd67de1d09712fbf52f1" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.25", +] + [[package]] name = "enumset" version = "1.1.2" @@ -1792,7 +1817,7 @@ dependencies = [ "indexmap 1.9.3", "slab", "tokio", - "tokio-util", + "tokio-util 0.7.8", "tracing", ] @@ -2702,6 +2727,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6842d8099b88d19a64158a6cfdc3e9ad82c738c041dab98280ef7ba98d64fa" +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -3480,7 +3526,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls", - "tokio-util", + "tokio-util 0.7.8", "tower-service", "url", "wasm-bindgen", @@ -4634,6 +4680,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-serde" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" +dependencies = [ + "bincode", + "bytes", + "educe", + "futures-core", + "futures-sink", + "pin-project", + "serde", + "serde_json", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.8" @@ -4834,6 +4910,29 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "tracing-test" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" +dependencies = [ + "lazy_static", + "tracing-core", + "tracing-subscriber 0.3.17", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" +dependencies = [ + "lazy_static", + "quote", + "syn 1.0.109", +] + [[package]] name = "tracing-wasm" version = "0.2.1" @@ -6103,6 +6202,7 @@ dependencies = [ name = "wasmer-virtual-net" version = "0.3.0" dependencies = [ + "anyhow", "async-trait", "base64", "bincode", @@ -6116,7 +6216,10 @@ dependencies = [ "socket2", "thiserror", "tokio", + "tokio-serde", + "tokio-util 0.6.10", "tracing", + "tracing-test", "wasmer-virtual-io", ] diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index c3fc5d37d46..e16273fc383 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -803,7 +803,7 @@ impl MonitoringRuntime { } impl wasmer_wasix::Runtime for MonitoringRuntime { - fn networking(&self) -> &virtual_net::DynVirtualNetworking { + fn networking(&self) -> &wasmer_virtual_net::DynVirtualNetworking { self.runtime.networking() } diff --git a/lib/virtual-io/src/waker.rs b/lib/virtual-io/src/waker.rs index 124c14afbda..f9fe822635d 100644 --- a/lib/virtual-io/src/waker.rs +++ b/lib/virtual-io/src/waker.rs @@ -11,7 +11,7 @@ pub struct InlineWaker { condvar: Condvar, } impl InlineWaker { - fn new() -> Arc { + pub fn new() -> Arc { Arc::new(Self { lock: Mutex::new(()), condvar: Condvar::new(), @@ -28,7 +28,7 @@ impl InlineWaker { self.condvar.notify_all(); } - fn as_waker(self: &Arc) -> Waker { + pub fn as_waker(self: &Arc) -> Waker { let s: *const Self = Arc::into_raw(Arc::clone(self)); let raw_waker = RawWaker::new(s as *const (), &VTABLE); unsafe { Waker::from_raw(raw_waker) } diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index 6303fee32fd..bb4afbe6b12 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -25,11 +25,18 @@ bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } pin-project-lite = "0.2.9" futures-util = { version = "0.3" } +anyhow = "1.0" +tokio-serde = { version = "0.8", features = [ "bincode", "json" ], optional = true } +tokio-util = { version = "0.6", features = ["codec"], optional = true } + +[dev-dependencies] +tokio = { version = "1", default_features = false, features = [ "macros" ] } +tracing-test = { version = "0.2" } [features] default = [ "host-net", "remote" ] host-net = [ "tokio", "libc", "wasmer-virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] -remote = [ "tokio", "libc", "tokio/io-util", "tokio/sync" ] +remote = [ "tokio", "libc", "tokio/io-util", "tokio/sync", "tokio-serde", "tokio-util" ] [package.metadata.docs.rs] features = ["host-net", "remote"] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index ba667c8d57f..ac38bc333f3 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -1,9 +1,7 @@ use std::collections::HashMap; use std::future::Future; -use std::mem::MaybeUninit; use std::net::IpAddr; use std::net::SocketAddr; -use std::ops::DerefMut; use std::pin::Pin; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; @@ -17,25 +15,29 @@ use std::task::Waker; use std::time::Duration; use bytes::Buf; -use bytes::Bytes; use bytes::BytesMut; use derivative::Derivative; use futures_util::future::BoxFuture; use futures_util::stream::FuturesOrdered; +use futures_util::Sink; +use futures_util::Stream; use futures_util::StreamExt; -use serde::Serialize; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; -use tokio::io::ReadBuf; use tokio::sync::mpsc; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::mpsc::error::TrySendError; -use tokio::sync::oneshot; +use tokio_serde::formats::SymmetricalBincode; +use tokio_serde::formats::SymmetricalJson; +use tokio_serde::SymmetricallyFramed; +use tokio_util::codec::FramedRead; +use tokio_util::codec::FramedWrite; +use tokio_util::codec::LengthDelimitedCodec; use wasmer_virtual_io::InlineWaker; use wasmer_virtual_io::InterestType; -use crate::io_err_into_net_error; use crate::meta; +use crate::meta::FrameSerializationFormat; use crate::meta::RequestType; use crate::meta::ResponseType; use crate::meta::SocketId; @@ -55,38 +57,48 @@ use crate::VirtualTcpListener; use crate::VirtualTcpSocket; use crate::VirtualUdpSocket; -use crate::write_all::locking_write_all; -use crate::write_all::locking_write_all_many; +use crate::rx_tx::RemoteRx; +use crate::rx_tx::RemoteTx; +use crate::rx_tx::RemoteTxWakers; use crate::Result; #[derive(Debug, Clone)] -pub struct RemoteNetworking { +pub struct RemoteNetworkingClient { common: Arc, } -impl RemoteNetworking { +impl RemoteNetworkingClient { /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels pub fn new_from_mpsc( tx: mpsc::Sender, rx: mpsc::Receiver, - ) -> (Self, RemoteNetworkingDriver) { - let (_, rx_work) = mpsc::unbounded_channel(); + ) -> (Self, RemoteNetworkingClientDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + let tx_wakers = RemoteTxWakers::default(); let common = RemoteCommon { - tx: RemoteTx::Mpsc(tx), - rx: Mutex::new(RemoteRx::Mpsc(rx)), + tx: RemoteTx::Mpsc { + tx, + work: tx_work, + wakers: tx_wakers.clone(), + }, + rx: Mutex::new(RemoteRx::Mpsc { + rx, + wakers: tx_wakers, + }), request_seed: AtomicU64::new(1), requests: Default::default(), socket_seed: AtomicU64::new(1), recv_tx: Default::default(), recv_with_addr_tx: Default::default(), + accept_tx: Default::default(), handlers: Default::default(), stall: Default::default(), }; let common = Arc::new(common); - let driver = RemoteNetworkingDriver { + let driver = RemoteNetworkingClientDriver { more_work: rx_work, tasks: Default::default(), common: common.clone(), @@ -96,61 +108,64 @@ impl RemoteNetworking { (networking, driver) } - /// Creates a new interface on the remote location using - /// a unique interface ID and a pair of channels - pub fn new_from_stream( - tx: Pin>, - rx: Pin>, - ) -> (Self, RemoteNetworkingDriver) { - Self::new_from_stream_internal(tx, rx, false) - } - /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels /// /// This version will run the async read and write operations /// only the driver (this is needed for mixed runtimes) - pub fn new_from_stream_via_driver( - tx: Pin>, - rx: Pin>, - ) -> (Self, RemoteNetworkingDriver) { - Self::new_from_stream_internal(tx, rx, true) - } - - fn new_from_stream_internal( - tx: Pin>, - rx: Pin>, - via_driver: bool, - ) -> (Self, RemoteNetworkingDriver) { - let (tx_work, rx_work) = mpsc::unbounded_channel(); + pub fn new_from_stream( + tx: TX, + rx: RX, + format: FrameSerializationFormat, + ) -> (Self, RemoteNetworkingClientDriver) + where + TX: AsyncWrite + Send + 'static, + RX: AsyncRead + Send + 'static, + { + let tx = FramedWrite::new(tx, LengthDelimitedCodec::new()); + let tx: Pin + Send + 'static>> = + match format { + FrameSerializationFormat::Bincode => { + Box::pin(SymmetricallyFramed::new(tx, SymmetricalBincode::default())) + } + FrameSerializationFormat::Json => { + Box::pin(SymmetricallyFramed::new(tx, SymmetricalJson::default())) + } + }; - let common = RemoteCommon { - tx: if via_driver { - RemoteTx::StreamViaDriver { - tx: Arc::new(Mutex::new(tx)), - work: tx_work, + let rx = FramedRead::new(rx, LengthDelimitedCodec::new()); + let rx: Pin> + Send + 'static>> = + match format { + FrameSerializationFormat::Bincode => { + Box::pin(SymmetricallyFramed::new(rx, SymmetricalBincode::default())) } - } else { - RemoteTx::Stream { - tx: Arc::new(Mutex::new(tx)), + FrameSerializationFormat::Json => { + Box::pin(SymmetricallyFramed::new(rx, SymmetricalJson::default())) } + }; + + let (tx_work, rx_work) = mpsc::unbounded_channel(); + let tx_wakers = RemoteTxWakers::default(); + + let common = RemoteCommon { + tx: RemoteTx::Stream { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + wakers: tx_wakers.clone(), }, - rx: Mutex::new(RemoteRx::Stream { - rx, - next: None, - buf: BytesMut::new(), - }), + rx: Mutex::new(RemoteRx::Stream { rx }), request_seed: AtomicU64::new(1), requests: Default::default(), socket_seed: AtomicU64::new(1), recv_tx: Default::default(), recv_with_addr_tx: Default::default(), + accept_tx: Default::default(), handlers: Default::default(), stall: Default::default(), }; let common = Arc::new(common); - let driver = RemoteNetworkingDriver { + let driver = RemoteNetworkingClientDriver { more_work: rx_work, tasks: Default::default(), common: common.clone(), @@ -167,19 +182,24 @@ impl RemoteNetworking { let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); + let (tx, rx_accept) = tokio::sync::mpsc::channel(100); + self.common.accept_tx.lock().unwrap().insert(id, tx); + RemoteSocket { socket_id: id, common: self.common.clone(), rx_buffer: BytesMut::new(), rx_recv, rx_recv_with_addr, + rx_accept, tx_waker: TxWaker::new(&self.common).as_waker(), + pending_accept: None, } } } pin_project_lite::pin_project! { - pub struct RemoteNetworkingDriver { + pub struct RemoteNetworkingClientDriver { common: Arc, more_work: mpsc::UnboundedReceiver>, #[pin] @@ -187,7 +207,7 @@ pin_project_lite::pin_project! { } } -impl Future for RemoteNetworkingDriver { +impl Future for RemoteNetworkingClientDriver { type Output = (); fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { @@ -303,6 +323,32 @@ impl Future for RemoteNetworkingDriver { .map(|h| h.interest(InterestType::Writable)); } }, + + MessageResponse::FinishAccept { + socket_id, + child_id, + addr, + } => { + let common = self.common.clone(); + self.tasks.push_back(Box::pin(async move { + let tx = common + .accept_tx + .lock() + .unwrap() + .get(&socket_id) + .map(|tx| tx.clone()); + if let Some(tx) = tx { + tx.send((child_id, addr)).await.ok(); + } + + common + .handlers + .lock() + .unwrap() + .get_mut(&socket_id) + .map(|h| h.interest(InterestType::Readable)); + })); + } MessageResponse::Closed { socket_id } => { self.common .handlers @@ -327,161 +373,6 @@ impl Future for RemoteNetworkingDriver { } } -pub(crate) type RemoteTxStream = Pin>; -pub(crate) enum RemoteTx -where - T: Serialize, -{ - Mpsc(mpsc::Sender), - Stream { - tx: Arc>, - }, - StreamViaDriver { - tx: Arc>, - work: mpsc::UnboundedSender>, - }, -} -impl RemoteTx -where - T: Serialize + Send + Sync + 'static, -{ - pub(crate) async fn send(&self, req: T) -> Result<()> { - match self { - RemoteTx::Mpsc(tx) => tx - .send(req) - .await - .map_err(|_| NetworkError::ConnectionAborted), - RemoteTx::Stream { tx, .. } => { - let data = bincode::serialize(&req).map_err(|err| { - tracing::warn!("failed to serialize message - {}", err); - NetworkError::IOError - })?; - locking_write_all(tx, data.into()) - .await - .map_err(io_err_into_net_error) - } - RemoteTx::StreamViaDriver { tx, work, .. } => { - let (tx_done, rx_done) = oneshot::channel(); - let tx = tx.clone(); - work.send(Box::pin(async move { - let job = async { - let data = bincode::serialize(&req).map_err(|err| { - tracing::warn!("failed to serialize message - {}", err); - NetworkError::IOError - })?; - locking_write_all(&tx, data.into()) - .await - .map_err(io_err_into_net_error) - }; - tx_done.send(job.await).ok(); - })) - .map_err(|_| NetworkError::ConnectionAborted)?; - - rx_done - .await - .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) - } - } - } - pub(crate) fn try_send(&self, interest_cx: &mut Context<'_>, req: T) -> Result<()> { - match self { - RemoteTx::Mpsc(tx) => match tx.try_send(req) { - Ok(()) => Ok(()), - Err(TrySendError::Closed(_)) => Err(NetworkError::ConnectionAborted), - Err(TrySendError::Full(_)) => Err(NetworkError::WouldBlock), - }, - RemoteTx::Stream { tx } => { - let data = bincode::serialize(&req).map_err(|err| { - tracing::warn!("failed to serialize message - {}", err); - NetworkError::IOError - })?; - let data_len = data.len() as u64; - let data_len_buf = data_len.to_le_bytes(); - - let data_len_remaining = { - let mut tx = tx.lock().unwrap(); - match Pin::new(tx.deref_mut()).poll_write(interest_cx, &data_len_buf) { - Poll::Ready(Ok(0)) => { - return Err(NetworkError::ConnectionAborted); - } - Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => None, - Poll::Ready(Ok(amt)) => { - let remaining = &data_len_buf[amt..]; - Some(remaining.to_vec()) - } - Poll::Ready(Err(err)) => { - return Err(io_err_into_net_error(err)); - } - Poll::Pending => { - return Err(NetworkError::WouldBlock); - } - } - }; - - let bufs = if let Some(data_len_remaining) = data_len_remaining { - vec![Bytes::from(data_len_remaining), Bytes::from(data)] - } else { - vec![Bytes::from(data)] - }; - - InlineWaker::block_on(Box::pin(async move { - locking_write_all_many(&tx, bufs) - .await - .map_err(io_err_into_net_error) - })) - } - RemoteTx::StreamViaDriver { tx, work } => { - let data = bincode::serialize(&req).map_err(|err| { - tracing::warn!("failed to serialize message - {}", err); - NetworkError::IOError - })?; - let data_len = data.len() as u64; - let data_len_buf = data_len.to_le_bytes(); - - let data_len_remaining = { - let mut tx = tx.lock().unwrap(); - match Pin::new(tx.deref_mut()).poll_write(interest_cx, &data_len_buf) { - Poll::Ready(Ok(0)) => { - return Err(NetworkError::ConnectionAborted); - } - Poll::Ready(Ok(amt)) if amt == data_len_buf.len() => None, - Poll::Ready(Ok(amt)) => { - let remaining = &data_len_buf[amt..]; - Some(remaining.to_vec()) - } - Poll::Ready(Err(err)) => { - return Err(io_err_into_net_error(err)); - } - Poll::Pending => { - return Err(NetworkError::WouldBlock); - } - } - }; - - let bufs = if let Some(data_len_remaining) = data_len_remaining { - vec![Bytes::from(data_len_remaining), Bytes::from(data)] - } else { - vec![Bytes::from(data)] - }; - - let (tx_done, rx_done) = std::sync::mpsc::channel(); - let tx = tx.clone(); - work.send(Box::pin(async move { - let ret = locking_write_all_many(&tx, bufs) - .await - .map_err(io_err_into_net_error); - tx_done.send(ret).ok(); - })) - .map_err(|_| NetworkError::ConnectionAborted)?; - - rx_done - .recv() - .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) - } - } - } -} - #[derive(Debug)] struct TxWaker { common: Arc, @@ -527,82 +418,6 @@ const VTABLE: RawWakerVTable = unsafe { ) }; -pub(crate) enum RemoteRx -where - T: serde::de::DeserializeOwned, -{ - Mpsc(mpsc::Receiver), - Stream { - rx: Pin>, - next: Option, - buf: BytesMut, - }, -} -impl RemoteRx -where - T: serde::de::DeserializeOwned, -{ - pub(crate) fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { - loop { - return match self { - RemoteRx::Mpsc(rx) => Pin::new(rx).poll_recv(cx), - RemoteRx::Stream { rx, next, buf } => { - match next.clone() { - Some(next) if (buf.len() as u64) >= next => { - let next = next as usize; - let msg = match bincode::deserialize(&buf[..next]) { - Ok(m) => m, - Err(err) => { - tracing::warn!("failed to deserialize message - {}", err); - return Poll::Ready(None); - } - }; - buf.advance(next); - if buf.is_empty() { - buf.clear(); - } - return Poll::Ready(Some(msg)); - } - None if buf.len() >= 8 => { - let mut data_len_buf = [0u8; 8]; - data_len_buf.copy_from_slice(&buf[..8]); - buf.advance(8); - if buf.is_empty() { - buf.clear(); - } - next.replace(u64::from_le_bytes(data_len_buf)); - continue; - } - _ => {} - } - - let mut chunk: [MaybeUninit; 10240] = - unsafe { MaybeUninit::uninit().assume_init() }; - let chunk_unsafe: &mut [MaybeUninit] = &mut chunk[..]; - let chunk_unsafe: &mut [u8] = unsafe { std::mem::transmute(chunk_unsafe) }; - - let mut read_buf = ReadBuf::new(chunk_unsafe); - match rx.as_mut().poll_read(cx, &mut read_buf) { - Poll::Ready(Ok(_)) => { - let filled = read_buf.filled(); - if filled.is_empty() { - return Poll::Ready(None); - } - buf.extend_from_slice(&filled); - continue; - } - Poll::Ready(Err(err)) => { - tracing::warn!("failed to read from channel - {}", err); - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } - }; - } - } -} - #[derive(Debug)] struct RequestTx { tx: mpsc::Sender, @@ -629,6 +444,7 @@ struct RemoteCommon { socket_seed: AtomicU64, recv_tx: Mutex>>>, recv_with_addr_tx: Mutex, SocketAddr)>>>, + accept_tx: Mutex>>, #[derivative(Debug = "ignore")] handlers: Mutex>>, @@ -648,7 +464,10 @@ impl RemoteCommon { }; if let Err(err) = self .tx - .send(MessageRequest::Interface { req_id, req }) + .send(MessageRequest::Interface { + req_id: Some(req_id), + req, + }) .await { return ResponseType::Err(err); @@ -656,13 +475,14 @@ impl RemoteCommon { req_rx.recv().await.unwrap() } - fn blocking_io_iface(&self, req: RequestType) -> ResponseType { - InlineWaker::block_on(self.io_iface(req)) + fn io_iface_fire_and_forget(&self, req: RequestType) -> Result<()> { + self.tx + .send_with_driver(MessageRequest::Interface { req_id: None, req }) } } #[async_trait::async_trait] -impl VirtualNetworking for RemoteNetworking { +impl VirtualNetworking for RemoteNetworkingClient { async fn bridge( &self, network: &str, @@ -680,7 +500,10 @@ impl VirtualNetworking for RemoteNetworking { { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to bridge request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -688,7 +511,10 @@ impl VirtualNetworking for RemoteNetworking { match self.common.io_iface(RequestType::Unbridge).await { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to unbridge request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -696,59 +522,52 @@ impl VirtualNetworking for RemoteNetworking { match self.common.io_iface(RequestType::DhcpAcquire).await { ResponseType::Err(err) => Err(err), ResponseType::IpAddressList(ips) => Ok(ips), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to DHCP acquire request - {res:?}"); + Err(NetworkError::IOError) + } } } fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<()> { - match self - .common - .blocking_io_iface(RequestType::IpAdd { ip, prefix }) - { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.common + .io_iface_fire_and_forget(RequestType::IpAdd { ip, prefix }) } fn ip_remove(&self, ip: IpAddr) -> Result<()> { - match self.common.blocking_io_iface(RequestType::IpRemove(ip)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.common + .io_iface_fire_and_forget(RequestType::IpRemove(ip)) } fn ip_clear(&self) -> Result<()> { - match self.common.blocking_io_iface(RequestType::IpClear) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.common.io_iface_fire_and_forget(RequestType::IpClear) } fn ip_list(&self) -> Result> { - match self.common.blocking_io_iface(RequestType::Unbridge) { + match InlineWaker::block_on(self.common.io_iface(RequestType::Unbridge)) { ResponseType::Err(err) => Err(err), ResponseType::CidrList(routes) => Ok(routes), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to IP list request - {res:?}"); + Err(NetworkError::IOError) + } } } fn mac(&self) -> Result<[u8; 6]> { - match self.common.blocking_io_iface(RequestType::GetMac) { + match InlineWaker::block_on(self.common.io_iface(RequestType::GetMac)) { ResponseType::Err(err) => Err(err), ResponseType::Mac(mac) => Ok(mac), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to MAC request - {res:?}"); + Err(NetworkError::IOError) + } } } fn gateway_set(&self, ip: IpAddr) -> Result<()> { - match self.common.blocking_io_iface(RequestType::GatewaySet(ip)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.common + .io_iface_fire_and_forget(RequestType::GatewaySet(ip)) } fn route_add( @@ -758,42 +577,32 @@ impl VirtualNetworking for RemoteNetworking { preferred_until: Option, expires_at: Option, ) -> Result<()> { - match self.common.blocking_io_iface(RequestType::RouteAdd { + self.common.io_iface_fire_and_forget(RequestType::RouteAdd { cidr, via_router, preferred_until, expires_at, - }) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + }) } fn route_remove(&self, cidr: IpAddr) -> Result<()> { - match self - .common - .blocking_io_iface(RequestType::RouteRemove(cidr)) - { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.common + .io_iface_fire_and_forget(RequestType::RouteRemove(cidr)) } fn route_clear(&self) -> Result<()> { - match self.common.blocking_io_iface(RequestType::RouteClear) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.common + .io_iface_fire_and_forget(RequestType::RouteClear) } fn route_list(&self) -> Result> { - match self.common.blocking_io_iface(RequestType::GetRouteList) { + match InlineWaker::block_on(self.common.io_iface(RequestType::GetRouteList)) { ResponseType::Err(err) => Err(err), ResponseType::RouteList(routes) => Ok(routes), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to route list request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -806,7 +615,11 @@ impl VirtualNetworking for RemoteNetworking { match self.common.io_iface(RequestType::BindRaw(socket_id)).await { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), - _ => Err(NetworkError::IOError), + ResponseType::Socket(socket_id) => Ok(Box::new(self.new_socket(socket_id))), + res => { + tracing::debug!("invalid response to bind RAw request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -835,7 +648,15 @@ impl VirtualNetworking for RemoteNetworking { { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), - _ => Err(NetworkError::IOError), + ResponseType::Socket(socket_id) => { + let mut socket = self.new_socket(socket_id); + socket.touch_begin_accept().ok(); + Ok(Box::new(socket)) + } + res => { + tracing::debug!("invalid response to listen TCP request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -862,7 +683,11 @@ impl VirtualNetworking for RemoteNetworking { { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), - _ => Err(NetworkError::IOError), + ResponseType::Socket(socket_id) => Ok(Box::new(self.new_socket(socket_id))), + res => { + tracing::debug!("invalid response to bind UDP request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -879,7 +704,11 @@ impl VirtualNetworking for RemoteNetworking { { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), - _ => Err(NetworkError::IOError), + ResponseType::Socket(socket_id) => Ok(Box::new(self.new_socket(socket_id))), + res => { + tracing::debug!("invalid response to bind ICMP request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -904,7 +733,11 @@ impl VirtualNetworking for RemoteNetworking { { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(Box::new(self.new_socket(socket_id))), - _ => Err(NetworkError::IOError), + ResponseType::Socket(socket_id) => Ok(Box::new(self.new_socket(socket_id))), + res => { + tracing::debug!("invalid response to connect TCP request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -925,7 +758,10 @@ impl VirtualNetworking for RemoteNetworking { { ResponseType::Err(err) => Err(err), ResponseType::IpAddressList(ips) => Ok(ips), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to resolve request - {res:?}"); + Err(NetworkError::IOError) + } } } } @@ -938,6 +774,8 @@ struct RemoteSocket { rx_recv: mpsc::Receiver>, rx_recv_with_addr: mpsc::Receiver<(Vec, SocketAddr)>, tx_waker: Waker, + rx_accept: mpsc::Receiver<(SocketId, SocketAddr)>, + pending_accept: Option, } impl Drop for RemoteSocket { fn drop(&mut self) { @@ -964,7 +802,7 @@ impl RemoteSocket { .tx .send(MessageRequest::Socket { socket: self.socket_id, - req_id, + req_id: Some(req_id), req, }) .await @@ -974,8 +812,26 @@ impl RemoteSocket { req_rx.recv().await.unwrap() } - fn blocking_io_socket(&self, req: RequestType) -> ResponseType { - InlineWaker::block_on(self.io_socket(req)) + fn io_socket_fire_and_forget(&self, req: RequestType) -> Result<()> { + self.common.tx.send_with_driver(MessageRequest::Socket { + socket: self.socket_id, + req_id: None, + req, + }) + } + + fn touch_begin_accept(&mut self) -> Result<()> { + if self.pending_accept.is_some() { + return Ok(()); + } + let child_id: SocketId = self + .common + .socket_seed + .fetch_add(1, Ordering::SeqCst) + .into(); + self.io_socket_fire_and_forget(RequestType::BeginAccept(child_id))?; + self.pending_accept.replace(child_id); + Ok(()) } } @@ -987,34 +843,39 @@ impl VirtualIoSource for RemoteSocket { impl VirtualSocket for RemoteSocket { fn set_ttl(&mut self, ttl: u32) -> Result<()> { - match self.blocking_io_socket(RequestType::SetTtl(ttl)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetTtl(ttl)) } fn ttl(&self) -> Result { - match self.blocking_io_socket(RequestType::GetTtl) { + match InlineWaker::block_on(self.io_socket(RequestType::GetTtl)) { ResponseType::Err(err) => Err(err), ResponseType::Ttl(ttl) => Ok(ttl), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get TTL request - {res:?}"); + Err(NetworkError::IOError) + } } } fn addr_local(&self) -> Result { - match self.blocking_io_socket(RequestType::GetAddrLocal) { + match InlineWaker::block_on(self.io_socket(RequestType::GetAddrLocal)) { ResponseType::Err(err) => Err(err), ResponseType::SocketAddr(addr) => Ok(addr), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to address local request - {res:?}"); + Err(NetworkError::IOError) + } } } fn status(&self) -> Result { - match self.blocking_io_socket(RequestType::GetStatus) { + match InlineWaker::block_on(self.io_socket(RequestType::GetStatus)) { ResponseType::Err(err) => Err(err), ResponseType::Status(status) => Ok(status), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to status request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -1033,32 +894,40 @@ impl VirtualSocket for RemoteSocket { impl VirtualTcpListener for RemoteSocket { fn try_accept(&mut self) -> Result<(Box, SocketAddr)> { - let child_id: SocketId = self - .common - .socket_seed - .fetch_add(1, Ordering::SeqCst) - .into(); - match self.blocking_io_socket(RequestType::TryAccept(child_id)) { - ResponseType::Err(err) => Err(err), - ResponseType::SocketWithAddr { id, addr } => { - let (tx, rx_recv) = tokio::sync::mpsc::channel(100); - self.common.recv_tx.lock().unwrap().insert(id, tx); - - let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); - self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); - - let socket = RemoteSocket { - socket_id: id, - common: self.common.clone(), - rx_buffer: BytesMut::new(), - rx_recv, - rx_recv_with_addr, - tx_waker: TxWaker::new(&self.common).as_waker(), - }; - Ok((Box::new(socket), addr)) - } - _ => Err(NetworkError::IOError), - } + self.touch_begin_accept()?; + + let (id, addr) = self.rx_accept.try_recv().map_err(|err| match err { + TryRecvError::Empty => NetworkError::WouldBlock, + TryRecvError::Disconnected => NetworkError::ConnectionAborted, + })?; + + // This placed here will mean there is always an accept request pending at the + // server as the constructor invokes this method and we invoke it here after + // receiving a child connection. + self.pending_accept.take(); + self.touch_begin_accept().ok(); + + // Now we construct the child + let (tx, rx_recv) = tokio::sync::mpsc::channel(100); + self.common.recv_tx.lock().unwrap().insert(id, tx); + + let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); + self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); + + let (tx, rx_accept) = tokio::sync::mpsc::channel(100); + self.common.accept_tx.lock().unwrap().insert(id, tx); + + let socket = RemoteSocket { + socket_id: id, + common: self.common.clone(), + rx_buffer: BytesMut::new(), + rx_recv, + rx_recv_with_addr, + rx_accept, + pending_accept: None, + tx_waker: TxWaker::new(&self.common).as_waker(), + }; + Ok((Box::new(socket), addr)) } fn set_handler( @@ -1069,53 +938,62 @@ impl VirtualTcpListener for RemoteSocket { } fn addr_local(&self) -> Result { - match self.blocking_io_socket(RequestType::GetAddrLocal) { + match InlineWaker::block_on(self.io_socket(RequestType::GetAddrLocal)) { ResponseType::Err(err) => Err(err), ResponseType::SocketAddr(addr) => Ok(addr), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to addr local request - {res:?}"); + Err(NetworkError::IOError) + } } } fn set_ttl(&mut self, ttl: u8) -> Result<()> { - match self.blocking_io_socket(RequestType::SetTtl(ttl as u32)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetTtl(ttl as u32)) } fn ttl(&self) -> Result { - match self.blocking_io_socket(RequestType::GetTtl) { + match InlineWaker::block_on(self.io_socket(RequestType::GetTtl)) { ResponseType::Err(err) => Err(err), ResponseType::Ttl(val) => Ok(val.try_into().map_err(|_| NetworkError::InvalidData)?), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get TTL request - {res:?}"); + Err(NetworkError::IOError) + } } } } impl VirtualRawSocket for RemoteSocket { fn try_send(&mut self, data: &[u8]) -> Result { - let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); - let mut cx = Context::from_waker(&self.tx_waker); - self.common - .tx - .try_send( - &mut cx, - MessageRequest::Send { - socket: self.socket_id, - data: data.to_vec(), - req_id, - }, - ) - .map(|_| data.len()) + match self.common.tx.poll_send( + &mut cx, + MessageRequest::Send { + socket: self.socket_id, + data: data.to_vec(), + req_id: None, + }, + ) { + Poll::Ready(Ok(())) => Ok(data.len()), + Poll::Ready(Err(err)) => Err(err), + Poll::Pending => Err(NetworkError::WouldBlock), + } } fn try_flush(&mut self) -> Result<()> { - match self.blocking_io_socket(RequestType::Flush) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), + let mut cx = Context::from_waker(&self.tx_waker); + match self.common.tx.poll_send( + &mut cx, + MessageRequest::Socket { + socket: self.socket_id, + req: RequestType::Flush, + req_id: None, + }, + ) { + Poll::Ready(Ok(())) => Ok(()), + Poll::Ready(Err(err)) => Err(err), + Poll::Pending => Err(NetworkError::WouldBlock), } } @@ -1137,18 +1015,17 @@ impl VirtualRawSocket for RemoteSocket { } fn set_promiscuous(&mut self, promiscuous: bool) -> Result<()> { - match self.blocking_io_socket(RequestType::SetPromiscuous(promiscuous)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetPromiscuous(promiscuous)) } fn promiscuous(&self) -> Result { - match self.blocking_io_socket(RequestType::GetPromiscuous) { + match InlineWaker::block_on(self.io_socket(RequestType::GetPromiscuous)) { ResponseType::Err(err) => Err(err), ResponseType::Flag(val) => Ok(val), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get promiscuous request - {res:?}"); + Err(NetworkError::IOError) + } } } } @@ -1157,18 +1034,19 @@ impl VirtualConnectionlessSocket for RemoteSocket { fn try_send_to(&mut self, data: &[u8], addr: SocketAddr) -> Result { let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); let mut cx = Context::from_waker(&self.tx_waker); - self.common - .tx - .try_send( - &mut cx, - MessageRequest::SendTo { - socket: self.socket_id, - data: data.to_vec(), - addr, - req_id, - }, - ) - .map(|_| data.len()) + match self.common.tx.poll_send( + &mut cx, + MessageRequest::SendTo { + socket: self.socket_id, + data: data.to_vec(), + addr, + req_id: Some(req_id), + }, + ) { + Poll::Ready(Ok(())) => Ok(data.len()), + Poll::Ready(Err(err)) => Err(err), + Poll::Pending => Err(NetworkError::WouldBlock), + } } fn try_recv_from( @@ -1190,66 +1068,62 @@ impl VirtualConnectionlessSocket for RemoteSocket { impl VirtualUdpSocket for RemoteSocket { fn set_broadcast(&mut self, broadcast: bool) -> Result<()> { - match self.blocking_io_socket(RequestType::SetBroadcast(broadcast)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetBroadcast(broadcast)) } fn broadcast(&self) -> Result { - match self.blocking_io_socket(RequestType::GetBroadcast) { + match InlineWaker::block_on(self.io_socket(RequestType::GetBroadcast)) { ResponseType::Err(err) => Err(err), ResponseType::Flag(val) => Ok(val), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get broadcast request - {res:?}"); + Err(NetworkError::IOError) + } } } fn set_multicast_loop_v4(&mut self, val: bool) -> Result<()> { - match self.blocking_io_socket(RequestType::SetMulticastLoopV4(val)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetMulticastLoopV4(val)) } fn multicast_loop_v4(&self) -> Result { - match self.blocking_io_socket(RequestType::GetMulticastLoopV4) { + match InlineWaker::block_on(self.io_socket(RequestType::GetMulticastLoopV4)) { ResponseType::Err(err) => Err(err), ResponseType::Flag(val) => Ok(val), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get multicast loop v4 request - {res:?}"); + Err(NetworkError::IOError) + } } } fn set_multicast_loop_v6(&mut self, val: bool) -> Result<()> { - match self.blocking_io_socket(RequestType::SetMulticastLoopV6(val)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetMulticastLoopV6(val)) } fn multicast_loop_v6(&self) -> Result { - match self.blocking_io_socket(RequestType::GetMulticastLoopV6) { + match InlineWaker::block_on(self.io_socket(RequestType::GetMulticastLoopV6)) { ResponseType::Err(err) => Err(err), ResponseType::Flag(val) => Ok(val), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get multicast loop v6 request - {res:?}"); + Err(NetworkError::IOError) + } } } fn set_multicast_ttl_v4(&mut self, ttl: u32) -> Result<()> { - match self.blocking_io_socket(RequestType::SetMulticastTtlV4(ttl)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetMulticastTtlV4(ttl)) } fn multicast_ttl_v4(&self) -> Result { - match self.blocking_io_socket(RequestType::GetMulticastTtlV4) { + match InlineWaker::block_on(self.io_socket(RequestType::GetMulticastTtlV4)) { ResponseType::Err(err) => Err(err), ResponseType::Ttl(ttl) => Ok(ttl), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get multicast TTL v4 request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -1258,11 +1132,7 @@ impl VirtualUdpSocket for RemoteSocket { multiaddr: std::net::Ipv4Addr, iface: std::net::Ipv4Addr, ) -> Result<()> { - match self.blocking_io_socket(RequestType::JoinMulticastV4 { multiaddr, iface }) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::JoinMulticastV4 { multiaddr, iface }) } fn leave_multicast_v4( @@ -1270,35 +1140,26 @@ impl VirtualUdpSocket for RemoteSocket { multiaddr: std::net::Ipv4Addr, iface: std::net::Ipv4Addr, ) -> Result<()> { - match self.blocking_io_socket(RequestType::LeaveMulticastV4 { multiaddr, iface }) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::LeaveMulticastV4 { multiaddr, iface }) } fn join_multicast_v6(&mut self, multiaddr: std::net::Ipv6Addr, iface: u32) -> Result<()> { - match self.blocking_io_socket(RequestType::JoinMulticastV6 { multiaddr, iface }) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::JoinMulticastV6 { multiaddr, iface }) } fn leave_multicast_v6(&mut self, multiaddr: std::net::Ipv6Addr, iface: u32) -> Result<()> { - match self.blocking_io_socket(RequestType::LeaveMulticastV6 { multiaddr, iface }) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::LeaveMulticastV6 { multiaddr, iface }) } fn addr_peer(&self) -> Result> { - match self.blocking_io_socket(RequestType::GetAddrPeer) { + match InlineWaker::block_on(self.io_socket(RequestType::GetAddrPeer)) { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(None), ResponseType::SocketAddr(addr) => Ok(Some(addr)), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to addr peer request - {res:?}"); + Err(NetworkError::IOError) + } } } } @@ -1307,52 +1168,56 @@ impl VirtualIcmpSocket for RemoteSocket {} impl VirtualConnectedSocket for RemoteSocket { fn set_linger(&mut self, linger: Option) -> Result<()> { - match self.blocking_io_socket(RequestType::SetLinger(linger)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetLinger(linger)) } fn linger(&self) -> Result> { - match self.blocking_io_socket(RequestType::GetLinger) { + match InlineWaker::block_on(self.io_socket(RequestType::GetLinger)) { ResponseType::Err(err) => Err(err), ResponseType::None => Ok(None), ResponseType::Duration(val) => Ok(Some(val)), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get linger request - {res:?}"); + Err(NetworkError::IOError) + } } } fn try_send(&mut self, data: &[u8]) -> Result { let req_id = self.common.request_seed.fetch_add(1, Ordering::SeqCst); let mut cx = Context::from_waker(&self.tx_waker); - self.common - .tx - .try_send( - &mut cx, - MessageRequest::Send { - socket: self.socket_id, - data: data.to_vec(), - req_id, - }, - ) - .map(|_| data.len()) + match self.common.tx.poll_send( + &mut cx, + MessageRequest::Send { + socket: self.socket_id, + data: data.to_vec(), + req_id: Some(req_id), + }, + ) { + Poll::Ready(Ok(())) => Ok(data.len()), + Poll::Ready(Err(err)) => Err(err), + Poll::Pending => Err(NetworkError::WouldBlock), + } } fn try_flush(&mut self) -> Result<()> { - match self.blocking_io_socket(RequestType::Flush) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), + let mut cx = Context::from_waker(&self.tx_waker); + match self.common.tx.poll_send( + &mut cx, + MessageRequest::Socket { + socket: self.socket_id, + req: RequestType::Flush, + req_id: None, + }, + ) { + Poll::Ready(Ok(())) => Ok(()), + Poll::Ready(Err(err)) => Err(err), + Poll::Pending => Err(NetworkError::WouldBlock), } } fn close(&mut self) -> Result<()> { - match self.blocking_io_socket(RequestType::Close) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::Close) } fn try_recv(&mut self, buf: &mut [std::mem::MaybeUninit]) -> Result { @@ -1375,58 +1240,58 @@ impl VirtualConnectedSocket for RemoteSocket { impl VirtualTcpSocket for RemoteSocket { fn set_recv_buf_size(&mut self, size: usize) -> Result<()> { - match self.blocking_io_socket(RequestType::SetRecvBufSize(size as u64)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetRecvBufSize(size as u64)) } fn recv_buf_size(&self) -> Result { - match self.blocking_io_socket(RequestType::GetRecvBufSize) { + match InlineWaker::block_on(self.io_socket(RequestType::GetRecvBufSize)) { ResponseType::Err(err) => Err(err), ResponseType::Amount(amt) => Ok(amt.try_into().map_err(|_| NetworkError::IOError)?), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get recv buf size request - {res:?}"); + Err(NetworkError::IOError) + } } } fn set_send_buf_size(&mut self, size: usize) -> Result<()> { - match self.blocking_io_socket(RequestType::SetSendBufSize(size as u64)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetSendBufSize(size as u64)) } fn send_buf_size(&self) -> Result { - match self.blocking_io_socket(RequestType::GetSendBufSize) { + match InlineWaker::block_on(self.io_socket(RequestType::GetSendBufSize)) { ResponseType::Err(err) => Err(err), ResponseType::Amount(val) => Ok(val.try_into().map_err(|_| NetworkError::IOError)?), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get send buf size request - {res:?}"); + Err(NetworkError::IOError) + } } } fn set_nodelay(&mut self, reuse: bool) -> Result<()> { - match self.blocking_io_socket(RequestType::SetNoDelay(reuse)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::SetNoDelay(reuse)) } fn nodelay(&self) -> Result { - match self.blocking_io_socket(RequestType::GetNoDelay) { + match InlineWaker::block_on(self.io_socket(RequestType::GetNoDelay)) { ResponseType::Err(err) => Err(err), ResponseType::Flag(val) => Ok(val), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to get nodelay request - {res:?}"); + Err(NetworkError::IOError) + } } } fn addr_peer(&self) -> Result { - match self.blocking_io_socket(RequestType::GetAddrPeer) { + match InlineWaker::block_on(self.io_socket(RequestType::GetAddrPeer)) { ResponseType::Err(err) => Err(err), ResponseType::SocketAddr(addr) => Ok(addr), - _ => Err(NetworkError::IOError), + res => { + tracing::debug!("invalid response to addr peer request - {res:?}"); + Err(NetworkError::IOError) + } } } @@ -1436,15 +1301,11 @@ impl VirtualTcpSocket for RemoteSocket { std::net::Shutdown::Write => meta::Shutdown::Write, std::net::Shutdown::Both => meta::Shutdown::Both, }; - match self.blocking_io_socket(RequestType::Shutdown(shutdown)) { - ResponseType::Err(err) => Err(err), - ResponseType::None => Ok(()), - _ => Err(NetworkError::IOError), - } + self.io_socket_fire_and_forget(RequestType::Shutdown(shutdown)) } fn is_closed(&self) -> bool { - match self.blocking_io_socket(RequestType::IsClosed) { + match InlineWaker::block_on(self.io_socket(RequestType::IsClosed)) { ResponseType::Flag(val) => val, _ => false, } diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index 56a7b2e6f27..09faad7f7d6 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -6,13 +6,17 @@ pub mod client; pub mod host; pub mod meta; #[cfg(any(feature = "remote"))] +pub mod rx_tx; +#[cfg(any(feature = "remote"))] pub mod server; -pub mod write_all; +#[cfg(test)] +mod tests; #[cfg(any(feature = "remote"))] -pub use client::{RemoteNetworking, RemoteNetworkingDriver}; +pub use client::{RemoteNetworkingClient, RemoteNetworkingClientDriver}; +use pin_project_lite::pin_project; #[cfg(any(feature = "remote"))] -pub use server::{RemoteNetworkingAdapter, RemoteNetworkingAdapterDriver}; +pub use server::{RemoteNetworkingServer, RemoteNetworkingServerDriver}; use std::fmt; use std::mem::MaybeUninit; pub use std::net::IpAddr; @@ -20,9 +24,14 @@ pub use std::net::Ipv4Addr; pub use std::net::Ipv6Addr; use std::net::Shutdown; pub use std::net::SocketAddr; +use std::pin::Pin; use std::sync::Arc; +use std::task::Context; +use std::task::Poll; pub use std::time::Duration; use thiserror::Error; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; pub use bytes::Bytes; pub use bytes::BytesMut; @@ -213,6 +222,42 @@ pub trait VirtualTcpListener: VirtualIoSource + fmt::Debug + Send + Sync + 'stat fn ttl(&self) -> Result; } +#[async_trait::async_trait] +pub trait VirtualTcpListenerExt: VirtualTcpListener { + /// Accepts a new connection from the TCP listener + async fn accept(&mut self) -> Result<(Box, SocketAddr)>; +} + +#[async_trait::async_trait] +impl VirtualTcpListenerExt for R { + async fn accept(&mut self) -> Result<(Box, SocketAddr)> { + struct Poller<'a, R> + where + R: VirtualTcpListener + ?Sized, + { + listener: &'a mut R, + } + impl<'a, R> std::future::Future for Poller<'a, R> + where + R: VirtualTcpListener + ?Sized, + { + type Output = Result<(Box, SocketAddr)>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let handler: Box = cx.waker().into(); + if let Err(err) = self.listener.set_handler(handler) { + return Poll::Ready(Err(err)); + } + match self.listener.try_accept() { + Ok(ret) => Poll::Ready(Ok(ret)), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(err)), + } + } + } + Poller { listener: self }.await + } +} + pub trait VirtualSocket: VirtualIoSource + fmt::Debug + Send + Sync + 'static { /// Sets how many network hops the packets are permitted for new connections fn set_ttl(&mut self, ttl: u32) -> Result<()>; @@ -273,6 +318,109 @@ pub trait VirtualConnectedSocket: VirtualSocket + fmt::Debug + Send + Sync + 'st fn try_recv(&mut self, buf: &mut [MaybeUninit]) -> Result; } +#[async_trait::async_trait] +pub trait VirtualConnectedSocketExt: VirtualConnectedSocket { + async fn send(&mut self, data: &[u8]) -> Result; + + async fn recv(&mut self, buf: &mut [MaybeUninit]) -> Result; + + async fn flush(&mut self) -> Result<()>; +} + +#[async_trait::async_trait] +impl VirtualConnectedSocketExt for R { + async fn send(&mut self, data: &[u8]) -> Result { + pin_project! { + struct Poller<'a, 'b, R: ?Sized> + where + R: VirtualConnectedSocket, + { + socket: &'a mut R, + data: &'b [u8], + } + } + impl<'a, 'b, R> std::future::Future for Poller<'a, 'b, R> + where + R: VirtualConnectedSocket + ?Sized, + { + type Output = Result; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + let handler: Box = cx.waker().into(); + if let Err(err) = this.socket.set_handler(handler) { + return Poll::Ready(Err(err)); + } + match this.socket.try_send(this.data) { + Ok(ret) => Poll::Ready(Ok(ret)), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(err)), + } + } + } + Poller { socket: self, data }.await + } + + async fn recv(&mut self, buf: &mut [MaybeUninit]) -> Result { + pin_project! { + struct Poller<'a, 'b, R: ?Sized> + where + R: VirtualConnectedSocket, + { + socket: &'a mut R, + buf: &'b mut [MaybeUninit], + } + } + impl<'a, 'b, R> std::future::Future for Poller<'a, 'b, R> + where + R: VirtualConnectedSocket + ?Sized, + { + type Output = Result; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + let handler: Box = cx.waker().into(); + if let Err(err) = this.socket.set_handler(handler) { + return Poll::Ready(Err(err)); + } + match this.socket.try_recv(this.buf) { + Ok(ret) => Poll::Ready(Ok(ret)), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(err)), + } + } + } + Poller { socket: self, buf }.await + } + + async fn flush(&mut self) -> Result<()> { + struct Poller<'a, R> + where + R: VirtualConnectedSocket + ?Sized, + { + socket: &'a mut R, + } + impl<'a, R> std::future::Future for Poller<'a, R> + where + R: VirtualConnectedSocket + ?Sized, + { + type Output = Result<()>; + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let handler: Box = cx.waker().into(); + if let Err(err) = self.socket.set_handler(handler) { + return Poll::Ready(Err(err)); + } + match self.socket.try_flush() { + Ok(ret) => Poll::Ready(Ok(ret)), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(err)), + } + } + } + Poller { socket: self }.await + } +} + /// Connectionless sockets are able to send and receive datagrams and stream /// bytes to multiple addresses at the same time (peer-to-peer) pub trait VirtualConnectionlessSocket: VirtualSocket + fmt::Debug + Send + Sync + 'static { @@ -284,6 +432,86 @@ pub trait VirtualConnectionlessSocket: VirtualSocket + fmt::Debug + Send + Sync fn try_recv_from(&mut self, buf: &mut [MaybeUninit]) -> Result<(usize, SocketAddr)>; } +#[async_trait::async_trait] +pub trait VirtualConnectionlessSocketExt: VirtualConnectionlessSocket { + async fn send_to(&mut self, data: &[u8], addr: SocketAddr) -> Result; + + async fn recv_from(&mut self, buf: &mut [MaybeUninit]) -> Result<(usize, SocketAddr)>; +} + +#[async_trait::async_trait] +impl VirtualConnectionlessSocketExt for R { + async fn send_to(&mut self, data: &[u8], addr: SocketAddr) -> Result { + pin_project! { + struct Poller<'a, 'b, R: ?Sized> + where + R: VirtualConnectionlessSocket, + { + socket: &'a mut R, + data: &'b [u8], + addr: SocketAddr, + } + } + impl<'a, 'b, R> std::future::Future for Poller<'a, 'b, R> + where + R: VirtualConnectionlessSocket + ?Sized, + { + type Output = Result; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + let handler: Box = cx.waker().into(); + if let Err(err) = this.socket.set_handler(handler) { + return Poll::Ready(Err(err)); + } + match this.socket.try_send_to(this.data, *this.addr) { + Ok(ret) => Poll::Ready(Ok(ret)), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(err)), + } + } + } + Poller { + socket: self, + data, + addr, + } + .await + } + + async fn recv_from(&mut self, buf: &mut [MaybeUninit]) -> Result<(usize, SocketAddr)> { + pin_project! { + struct Poller<'a, 'b, R: ?Sized> + where + R: VirtualConnectionlessSocket, + { + socket: &'a mut R, + buf: &'b mut [MaybeUninit], + } + } + impl<'a, 'b, R> std::future::Future for Poller<'a, 'b, R> + where + R: VirtualConnectionlessSocket + ?Sized, + { + type Output = Result<(usize, SocketAddr)>; + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + let handler: Box = cx.waker().into(); + if let Err(err) = this.socket.set_handler(handler) { + return Poll::Ready(Err(err)); + } + match this.socket.try_recv_from(this.buf) { + Ok(ret) => Poll::Ready(Ok(ret)), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(err)), + } + } + } + Poller { socket: self, buf }.await + } +} + /// ICMP sockets are low level devices bound to a specific address /// that can send and receive ICMP packets pub trait VirtualIcmpSocket: @@ -353,6 +581,69 @@ pub trait VirtualTcpSocket: VirtualConnectedSocket + fmt::Debug + Send + Sync + fn is_closed(&self) -> bool; } +impl<'a> AsyncRead for Box { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + let this = self.get_mut(); + let handler: Box = cx.waker().into(); + if let Err(err) = this.set_handler(handler) { + return Poll::Ready(Err(net_error_into_io_err(err))); + } + let buf_unsafe = unsafe { buf.unfilled_mut() }; + match this.try_recv(buf_unsafe) { + Ok(ret) => { + unsafe { buf.assume_init(ret) }; + buf.set_filled(ret); + Poll::Ready(Ok(())) + } + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(net_error_into_io_err(err))), + } + } +} + +impl<'a> AsyncWrite for Box { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + let this = self.get_mut(); + let handler: Box = cx.waker().into(); + if let Err(err) = this.set_handler(handler) { + return Poll::Ready(Err(net_error_into_io_err(err))); + } + match this.try_send(buf) { + Ok(ret) => Poll::Ready(Ok(ret)), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(net_error_into_io_err(err))), + } + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + let handler: Box = cx.waker().into(); + if let Err(err) = this.set_handler(handler) { + return Poll::Ready(Err(net_error_into_io_err(err))); + } + match this.try_flush() { + Ok(()) => Poll::Ready(Ok(())), + Err(NetworkError::WouldBlock) => Poll::Pending, + Err(err) => Poll::Ready(Err(net_error_into_io_err(err))), + } + } + + fn poll_shutdown(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready( + self.shutdown(Shutdown::Write) + .map_err(net_error_into_io_err), + ) + } +} + pub trait VirtualUdpSocket: VirtualConnectionlessSocket + fmt::Debug + Send + Sync + 'static { @@ -548,3 +839,42 @@ pub fn io_err_into_net_error(net_error: std::io::Error) -> NetworkError { _ => NetworkError::UnknownError, } } + +pub fn net_error_into_io_err(net_error: NetworkError) -> std::io::Error { + use std::io::ErrorKind; + match net_error { + NetworkError::InvalidFd => ErrorKind::BrokenPipe.into(), + NetworkError::AlreadyExists => ErrorKind::AlreadyExists.into(), + NetworkError::Lock => ErrorKind::BrokenPipe.into(), + NetworkError::IOError => ErrorKind::BrokenPipe.into(), + NetworkError::AddressInUse => ErrorKind::AddrInUse.into(), + NetworkError::AddressNotAvailable => ErrorKind::AddrNotAvailable.into(), + NetworkError::BrokenPipe => ErrorKind::BrokenPipe.into(), + NetworkError::ConnectionAborted => ErrorKind::ConnectionAborted.into(), + NetworkError::ConnectionRefused => ErrorKind::ConnectionRefused.into(), + NetworkError::ConnectionReset => ErrorKind::ConnectionReset.into(), + NetworkError::Interrupted => ErrorKind::Interrupted.into(), + NetworkError::InvalidData => ErrorKind::InvalidData.into(), + NetworkError::InvalidInput => ErrorKind::InvalidInput.into(), + NetworkError::NotConnected => ErrorKind::NotConnected.into(), + NetworkError::NoDevice => ErrorKind::BrokenPipe.into(), + NetworkError::PermissionDenied => ErrorKind::PermissionDenied.into(), + NetworkError::TimedOut => ErrorKind::TimedOut.into(), + NetworkError::UnexpectedEof => ErrorKind::UnexpectedEof.into(), + NetworkError::WouldBlock => ErrorKind::WouldBlock.into(), + NetworkError::WriteZero => ErrorKind::WriteZero.into(), + NetworkError::Unsupported => ErrorKind::Unsupported.into(), + NetworkError::UnknownError => ErrorKind::BrokenPipe.into(), + NetworkError::InsufficientMemory => ErrorKind::OutOfMemory.into(), + NetworkError::TooManyOpenFiles => { + #[cfg(target_family = "unix")] + { + std::io::Error::from_raw_os_error(libc::EMFILE) + } + #[cfg(not(target_family = "unix"))] + { + ErrorKind::Other.into() + } + } + } +} diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index d32fd2d1b0b..4a6ebd7c8f4 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -21,6 +21,12 @@ impl From for SocketId { } } +#[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub enum FrameSerializationFormat { + Bincode, + Json +} + /// Possible values which can be passed to the [`TcpStream::shutdown`] method. #[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] pub enum Shutdown { @@ -111,8 +117,8 @@ pub enum RequestType { }, /// Closes the socket Close, - /// Tries to accept a new connection - TryAccept(SocketId), + /// Begins the process of accepting a socket and returns it later + BeginAccept(SocketId), /// Returns the local address of this TCP listener GetAddrLocal, /// Returns the address (IP and Port) of the peer socket that this @@ -243,8 +249,6 @@ pub enum ResponseType { RouteList(Vec), /// Reference to a socket Socket(SocketId), - /// Reference to a socket with an address - SocketWithAddr { id: SocketId, addr: SocketAddr }, /// The TTL of a packet Ttl(u32), /// The status of the socket @@ -256,23 +260,23 @@ pub enum ResponseType { pub enum MessageRequest { Interface { req: RequestType, - req_id: u64, + req_id: Option, }, Socket { socket: SocketId, req: RequestType, - req_id: u64, + req_id: Option, }, Send { socket: SocketId, data: Vec, - req_id: u64, + req_id: Option, }, SendTo { socket: SocketId, data: Vec, addr: SocketAddr, - req_id: u64, + req_id: Option, }, } @@ -302,6 +306,11 @@ pub enum MessageResponse { req_id: u64, error: NetworkError, }, + FinishAccept { + socket_id: SocketId, + child_id: SocketId, + addr: SocketAddr, + }, Closed { socket_id: SocketId, }, diff --git a/lib/virtual-net/src/rx_tx.rs b/lib/virtual-net/src/rx_tx.rs new file mode 100644 index 00000000000..9145192e3a2 --- /dev/null +++ b/lib/virtual-net/src/rx_tx.rs @@ -0,0 +1,241 @@ +use std::{ + io, + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll, Waker}, +}; + +use crate::Result; +use futures_util::{future::BoxFuture, Future, Sink, SinkExt, Stream}; +use serde::Serialize; +use tokio::{ + io::AsyncWrite, + sync::{ + mpsc::{self, error::TrySendError}, + oneshot, + }, +}; +use wasmer_virtual_io::InlineWaker; + +use crate::{io_err_into_net_error, NetworkError}; + +#[derive(Debug, Clone, Default)] +pub(crate) struct RemoteTxWakers { + wakers: Arc>>, +} +impl RemoteTxWakers { + pub fn add(&self, waker: &Waker) { + let mut guard = self.wakers.lock().unwrap(); + if guard.iter().any(|w| w.will_wake(waker)) == false { + guard.push(waker.clone()); + } + } + pub fn wake(&self) { + let mut guard = self.wakers.lock().unwrap(); + guard.drain(..).for_each(|w| w.wake()); + } +} + +#[derive(Debug, Default)] +struct FailOnWrite {} +impl AsyncWrite for FailOnWrite { + fn poll_write( + self: Pin<&mut Self>, + _cx: &mut Context<'_>, + _buf: &[u8], + ) -> Poll> { + Poll::Ready(Err(io::ErrorKind::ConnectionAborted.into())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(io::ErrorKind::ConnectionAborted.into())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Err(io::ErrorKind::ConnectionAborted.into())) + } +} + +pub(crate) enum RemoteTx +where + T: Serialize, +{ + Mpsc { + tx: mpsc::Sender, + work: mpsc::UnboundedSender>, + wakers: RemoteTxWakers, + }, + Stream { + tx: Arc + Send + 'static>>>>, + work: mpsc::UnboundedSender>, + wakers: RemoteTxWakers, + }, +} +impl RemoteTx +where + T: Serialize + Send + Sync + 'static, +{ + pub(crate) async fn send(&self, req: T) -> Result<()> { + match self { + RemoteTx::Mpsc { tx, .. } => tx + .send(req) + .await + .map_err(|_| NetworkError::ConnectionAborted), + RemoteTx::Stream { tx, work, .. } => { + let (tx_done, rx_done) = oneshot::channel(); + let tx = tx.clone(); + work.send(Box::pin(async move { + let job = async { + let mut tx_guard = tx.lock_owned().await; + tx_guard.send(req).await.map_err(io_err_into_net_error) + }; + tx_done.send(job.await).ok(); + })) + .map_err(|_| NetworkError::ConnectionAborted)?; + + rx_done + .await + .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) + } + } + } + + pub(crate) fn poll_send(&self, cx: &mut Context<'_>, req: T) -> Poll> { + match self { + RemoteTx::Mpsc { tx, wakers, .. } => match tx.try_send(req) { + Ok(()) => Poll::Ready(Ok(())), + Err(TrySendError::Closed(_)) => Poll::Ready(Err(NetworkError::ConnectionAborted)), + Err(TrySendError::Full(_)) => { + wakers.add(cx.waker()); + Poll::Pending + } + }, + RemoteTx::Stream { tx, work, wakers } => { + let mut tx_guard = match tx.clone().try_lock_owned() { + Ok(lock) => lock, + Err(_) => { + wakers.add(cx.waker()); + return Poll::Pending; + } + }; + match tx_guard.poll_ready_unpin(cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(io_err_into_net_error(err))), + Poll::Pending => return Poll::Pending, + } + let mut job = Box::pin(async move { + if let Err(err) = tx_guard.send(req).await.map_err(io_err_into_net_error) { + tracing::error!("failed to send remaining bytes for request - {}", err); + } + }); + + // First we try to finish it synchronously + if job.as_mut().poll(cx).is_ready() { + return Poll::Ready(Ok(())); + } + + // Otherwise we push it to the driver which will block all future send + // operations until it finishes + work.send(job).map_err(|err| { + tracing::error!("failed to send remaining bytes for request - {}", err); + NetworkError::ConnectionAborted + })?; + Poll::Ready(Ok(())) + } + } + } + + pub(crate) fn send_with_driver(&self, req: T) -> Result<()> { + match self { + RemoteTx::Mpsc { tx, work, .. } => match tx.try_send(req) { + Ok(()) => Ok(()), + Err(TrySendError::Closed(_)) => Err(NetworkError::ConnectionAborted), + Err(TrySendError::Full(req)) => { + let tx = tx.clone(); + work.send(Box::pin(async move { + tx.send(req).await.ok(); + })) + .ok(); + Ok(()) + } + }, + RemoteTx::Stream { tx, work, .. } => { + let mut tx_guard = match tx.clone().try_lock_owned() { + Ok(lock) => lock, + Err(_) => { + let tx = tx.clone(); + work.send(Box::pin(async move { + let mut tx_guard = tx.lock().await; + tx_guard.send(req).await.ok(); + })) + .ok(); + return Ok(()); + } + }; + + let inline_waker = InlineWaker::new(); + let waker = inline_waker.as_waker(); + let mut cx = Context::from_waker(&waker); + + let mut job = Box::pin(async move { + if let Err(err) = tx_guard.send(req).await.map_err(io_err_into_net_error) { + tracing::error!("failed to send remaining bytes for request - {}", err); + } + }); + + // First we try to finish it synchronously + if job.as_mut().poll(&mut cx).is_ready() { + return Ok(()); + } + + // Otherwise we push it to the driver which will block all future send + // operations until it finishes + work.send(job).map_err(|err| { + tracing::error!("failed to send remaining bytes for request - {}", err); + NetworkError::ConnectionAborted + })?; + Ok(()) + } + } + } +} + +pub(crate) enum RemoteRx +where + T: serde::de::DeserializeOwned, +{ + Mpsc { + rx: mpsc::Receiver, + wakers: RemoteTxWakers, + }, + Stream { + rx: Pin> + Send + 'static>>, + }, +} +impl RemoteRx +where + T: serde::de::DeserializeOwned, +{ + pub(crate) fn poll(&mut self, cx: &mut Context<'_>) -> Poll> { + loop { + return match self { + RemoteRx::Mpsc { rx, wakers } => { + let ret = Pin::new(rx).poll_recv(cx); + if ret.is_ready() { + wakers.wake(); + } + ret + } + RemoteRx::Stream { rx } => match rx.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(msg))) => Poll::Ready(Some(msg)), + Poll::Ready(Some(Err(err))) => { + tracing::warn!("failed to read from channel - {}", err); + Poll::Ready(None) + } + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => Poll::Pending, + }, + }; + } + } +} diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index e65dc82025a..a15599f1ed4 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -1,17 +1,19 @@ -use crate::meta::ResponseType; +use crate::meta::{FrameSerializationFormat, ResponseType}; +use crate::rx_tx::{RemoteRx, RemoteTx, RemoteTxWakers}; use crate::{ - client::{RemoteRx, RemoteTx}, meta::{MessageRequest, MessageResponse, RequestType, SocketId}, VirtualNetworking, VirtualRawSocket, VirtualTcpListener, VirtualTcpSocket, VirtualUdpSocket, }; -use crate::{NetworkError, VirtualIcmpSocket}; -use bytes::BytesMut; +use crate::{IpCidr, IpRoute, NetworkError, StreamSecurity, VirtualIcmpSocket}; use derivative::Derivative; use futures_util::stream::FuturesOrdered; use futures_util::{future::BoxFuture, StreamExt}; +use futures_util::{Sink, Stream}; use std::collections::HashSet; use std::mem::MaybeUninit; +use std::net::IpAddr; use std::task::Waker; +use std::time::Duration; use std::{ collections::HashMap, future::Future, @@ -25,111 +27,261 @@ use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, }; +use tokio_serde::formats::{SymmetricalBincode, SymmetricalJson}; +use tokio_serde::SymmetricallyFramed; +use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec}; use wasmer_virtual_io::InterestHandler; type BackgroundTask = Option>; #[derive(Debug, Clone)] -pub struct RemoteNetworkingAdapter { +pub struct RemoteNetworkingServer { #[allow(dead_code)] common: Arc, + inner: Arc>>, } -impl RemoteNetworkingAdapter { +impl RemoteNetworkingServer { /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels pub fn new_from_mpsc( tx: mpsc::Sender, rx: mpsc::Receiver, inner: Box, - ) -> (Self, RemoteNetworkingAdapterDriver) { - let (_, rx_work) = mpsc::unbounded_channel(); + ) -> (Self, RemoteNetworkingServerDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + let tx_wakers = RemoteTxWakers::default(); let common = RemoteAdapterCommon { - tx: RemoteTx::Mpsc(tx), - rx: Mutex::new(RemoteRx::Mpsc(rx)), + tx: RemoteTx::Mpsc { + tx, + work: tx_work, + wakers: tx_wakers.clone(), + }, + rx: Mutex::new(RemoteRx::Mpsc { + rx, + wakers: tx_wakers, + }), sockets: Default::default(), + socket_accept: Default::default(), handler: Default::default(), stall_rx: Default::default(), }; let common = Arc::new(common); - let driver = RemoteNetworkingAdapterDriver { + let inner = Arc::new(tokio::sync::Mutex::new(inner)); + let driver = RemoteNetworkingServerDriver { more_work: rx_work, tasks: Default::default(), common: common.clone(), - inner: Arc::new(tokio::sync::Mutex::new(inner)), + inner: inner.clone(), }; - let networking = Self { common }; + let networking = Self { common, inner }; (networking, driver) } /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels - pub fn new_from_stream( - tx: Pin>, - rx: Pin>, + pub fn new_from_stream( + tx: TX, + rx: RX, + format: FrameSerializationFormat, inner: Box, - ) -> (Self, RemoteNetworkingAdapterDriver) { - Self::new_from_stream_internal(tx, rx, inner, false) - } + ) -> (Self, RemoteNetworkingServerDriver) + where + TX: AsyncWrite + Send + 'static, + RX: AsyncRead + Send + 'static, + { + let tx = FramedWrite::new(tx, LengthDelimitedCodec::new()); + let tx: Pin + Send + 'static>> = + match format { + FrameSerializationFormat::Bincode => { + Box::pin(SymmetricallyFramed::new(tx, SymmetricalBincode::default())) + } + FrameSerializationFormat::Json => { + Box::pin(SymmetricallyFramed::new(tx, SymmetricalJson::default())) + } + }; - /// Creates a new interface on the remote location using - /// a unique interface ID and a pair of channels - pub fn new_from_stream_via_driver( - tx: Pin>, - rx: Pin>, - inner: Box, - ) -> (Self, RemoteNetworkingAdapterDriver) { - Self::new_from_stream_internal(tx, rx, inner, true) - } + let rx = FramedRead::new(rx, LengthDelimitedCodec::new()); + let rx: Pin> + Send + 'static>> = + match format { + FrameSerializationFormat::Bincode => { + Box::pin(SymmetricallyFramed::new(rx, SymmetricalBincode::default())) + } + FrameSerializationFormat::Json => { + Box::pin(SymmetricallyFramed::new(rx, SymmetricalJson::default())) + } + }; - fn new_from_stream_internal( - tx: Pin>, - rx: Pin>, - inner: Box, - via_driver: bool, - ) -> (Self, RemoteNetworkingAdapterDriver) { let (tx_work, rx_work) = mpsc::unbounded_channel(); + let tx_wakers = RemoteTxWakers::default(); let handler = RemoteAdapterHandler::default(); let common = RemoteAdapterCommon { - tx: if via_driver { - RemoteTx::StreamViaDriver { - tx: Arc::new(Mutex::new(tx)), - work: tx_work, - } - } else { - RemoteTx::Stream { - tx: Arc::new(Mutex::new(tx)), - } + tx: RemoteTx::Stream { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + wakers: tx_wakers.clone(), }, - rx: Mutex::new(RemoteRx::Stream { - rx, - next: None, - buf: BytesMut::new(), - }), + rx: Mutex::new(RemoteRx::Stream { rx }), sockets: Default::default(), + socket_accept: Default::default(), handler, stall_rx: Default::default(), }; let common = Arc::new(common); - let driver = RemoteNetworkingAdapterDriver { + let inner = Arc::new(tokio::sync::Mutex::new(inner)); + let driver = RemoteNetworkingServerDriver { more_work: rx_work, tasks: Default::default(), common: common.clone(), - inner: Arc::new(tokio::sync::Mutex::new(inner)), + inner: inner.clone(), }; - let networking = Self { common }; + let networking = Self { common, inner }; (networking, driver) } } +#[async_trait::async_trait] +impl VirtualNetworking for RemoteNetworkingServer { + async fn bridge( + &self, + network: &str, + access_token: &str, + security: StreamSecurity, + ) -> Result<(), NetworkError> { + let inner = self.inner.lock().await; + inner.bridge(network, access_token, security).await + } + + async fn unbridge(&self) -> Result<(), NetworkError> { + let inner = self.inner.lock().await; + inner.unbridge().await + } + + async fn dhcp_acquire(&self) -> Result, NetworkError> { + let inner = self.inner.lock().await; + inner.dhcp_acquire().await + } + + fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<(), NetworkError> { + let inner = self.inner.blocking_lock(); + inner.ip_add(ip, prefix) + } + + fn ip_remove(&self, ip: IpAddr) -> Result<(), NetworkError> { + let inner = self.inner.blocking_lock(); + inner.ip_remove(ip) + } + + fn ip_clear(&self) -> Result<(), NetworkError> { + let inner = self.inner.blocking_lock(); + inner.ip_clear() + } + + fn ip_list(&self) -> Result, NetworkError> { + let inner = self.inner.blocking_lock(); + inner.ip_list() + } + + fn mac(&self) -> Result<[u8; 6], NetworkError> { + let inner = self.inner.blocking_lock(); + inner.mac() + } + + fn gateway_set(&self, ip: IpAddr) -> Result<(), NetworkError> { + let inner = self.inner.blocking_lock(); + inner.gateway_set(ip) + } + + fn route_add( + &self, + cidr: IpCidr, + via_router: IpAddr, + preferred_until: Option, + expires_at: Option, + ) -> Result<(), NetworkError> { + let inner = self.inner.blocking_lock(); + inner.route_add(cidr, via_router, preferred_until, expires_at) + } + + fn route_remove(&self, cidr: IpAddr) -> Result<(), NetworkError> { + let inner = self.inner.blocking_lock(); + inner.route_remove(cidr) + } + + fn route_clear(&self) -> Result<(), NetworkError> { + let inner = self.inner.blocking_lock(); + inner.route_clear() + } + + fn route_list(&self) -> Result, NetworkError> { + let inner = self.inner.blocking_lock(); + inner.route_list() + } + + async fn bind_raw(&self) -> Result, NetworkError> { + let inner = self.inner.lock().await; + inner.bind_raw().await + } + + async fn listen_tcp( + &self, + addr: SocketAddr, + only_v6: bool, + reuse_port: bool, + reuse_addr: bool, + ) -> Result, NetworkError> { + let inner = self.inner.lock().await; + inner + .listen_tcp(addr, only_v6, reuse_port, reuse_addr) + .await + } + + async fn bind_udp( + &self, + addr: SocketAddr, + reuse_port: bool, + reuse_addr: bool, + ) -> Result, NetworkError> { + let inner = self.inner.lock().await; + inner.bind_udp(addr, reuse_port, reuse_addr).await + } + + async fn bind_icmp( + &self, + addr: IpAddr, + ) -> Result, NetworkError> { + let inner = self.inner.lock().await; + inner.bind_icmp(addr).await + } + + async fn connect_tcp( + &self, + addr: SocketAddr, + peer: SocketAddr, + ) -> Result, NetworkError> { + let inner = self.inner.lock().await; + inner.connect_tcp(addr, peer).await + } + + async fn resolve( + &self, + host: &str, + port: Option, + dns_server: Option, + ) -> Result, NetworkError> { + let inner = self.inner.lock().await; + inner.resolve(host, port, dns_server).await + } +} + pin_project_lite::pin_project! { - pub struct RemoteNetworkingAdapterDriver { + pub struct RemoteNetworkingServerDriver { common: Arc, more_work: mpsc::UnboundedReceiver>, #[pin] @@ -138,7 +290,7 @@ pin_project_lite::pin_project! { } } -impl Future for RemoteNetworkingAdapterDriver { +impl Future for RemoteNetworkingServerDriver { type Output = (); fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { @@ -161,7 +313,7 @@ impl Future for RemoteNetworkingAdapterDriver { for socket_id in readable { if let Some(task) = guard .get_mut(&socket_id) - .map(|s| s.drain_reads(&common, socket_id)) + .map(|s| s.drain_reads_and_accepts(&common, socket_id)) .unwrap_or(None) { self.tasks.push_back(task); @@ -221,7 +373,7 @@ impl Future for RemoteNetworkingAdapterDriver { } } -impl RemoteNetworkingAdapterDriver { +impl RemoteNetworkingServerDriver { fn process(&mut self, msg: MessageRequest) -> BackgroundTask { match msg { MessageRequest::Send { @@ -244,11 +396,20 @@ impl RemoteNetworkingAdapterDriver { } } - fn process_send(&mut self, socket_id: SocketId, data: Vec, req_id: u64) -> BackgroundTask { + fn process_send( + &mut self, + socket_id: SocketId, + data: Vec, + req_id: Option, + ) -> BackgroundTask { let mut guard = self.common.sockets.lock().unwrap(); guard .get_mut(&socket_id) - .map(|s| s.send(&self.common, socket_id, data, req_id)) + .map(|s| { + req_id + .map(|req_id| s.send(&self.common, socket_id, data, req_id)) + .flatten() + }) .unwrap_or(None) } @@ -257,12 +418,16 @@ impl RemoteNetworkingAdapterDriver { socket_id: SocketId, data: Vec, addr: SocketAddr, - req_id: u64, + req_id: Option, ) -> BackgroundTask { let mut guard = self.common.sockets.lock().unwrap(); guard .get_mut(&socket_id) - .map(|s| s.send_to(&self.common, socket_id, data, addr, req_id)) + .map(|s| { + req_id + .map(|req_id| s.send_to(&self.common, socket_id, data, addr, req_id)) + .flatten() + }) .unwrap_or(None) } @@ -278,7 +443,12 @@ impl RemoteNetworkingAdapterDriver { })) } - fn process_async_inner(&self, work: F, transmute: T, req_id: u64) -> BackgroundTask + fn process_async_inner( + &self, + work: F, + transmute: T, + req_id: Option, + ) -> BackgroundTask where F: FnOnce(OwnedMutexGuard>) -> Fut + Send @@ -292,14 +462,18 @@ impl RemoteNetworkingAdapterDriver { let inner = inner.lock_owned().await; let future = work(inner); let ret = future.await; - common.send(MessageResponse::ResponseToRequest { - req_id, - res: transmute(ret), - }) + req_id + .map(|req_id| { + common.send(MessageResponse::ResponseToRequest { + req_id, + res: transmute(ret), + }) + }) + .flatten() }) } - fn process_async_noop(&self, work: F, req_id: u64) -> BackgroundTask + fn process_async_noop(&self, work: F, req_id: Option) -> BackgroundTask where F: FnOnce(OwnedMutexGuard>) -> Fut + Send @@ -316,11 +490,11 @@ impl RemoteNetworkingAdapterDriver { ) } - fn process_async_socket( + fn process_async_new_socket( &self, work: F, socket_id: SocketId, - req_id: u64, + req_id: Option, ) -> BackgroundTask where F: FnOnce(OwnedMutexGuard>) -> Fut @@ -336,7 +510,12 @@ impl RemoteNetworkingAdapterDriver { let handler = Box::new(common.handler.clone().for_socket(socket_id)); let err = match &mut socket { - RemoteAdapterSocket::TcpListener(s) => s.set_handler(handler), + RemoteAdapterSocket::TcpListener { .. } => { + // we do not attach the handler immediately with new TPC listeners as we + // only want it to trigger when the BeginAccept message is received with + // a child ID we can actually use + Ok(()) + } RemoteAdapterSocket::TcpSocket(s) => s.set_handler(handler), RemoteAdapterSocket::UdpSocket(s) => s.set_handler(handler), RemoteAdapterSocket::IcmpSocket(s) => s.set_handler(handler), @@ -362,7 +541,7 @@ impl RemoteNetworkingAdapterDriver { work: F, transmute: T, socket_id: SocketId, - req_id: u64, + req_id: Option, ) -> BackgroundTask where F: FnOnce(&mut RemoteAdapterSocket) -> R + Send + 'static, @@ -373,21 +552,34 @@ impl RemoteNetworkingAdapterDriver { let socket = match guard.get_mut(&socket_id) { Some(s) => s, None => { - return self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: ResponseType::Err(NetworkError::InvalidFd), - }) + return req_id + .map(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::InvalidFd), + }) + }) + .flatten() } }; work(socket) }; - self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: transmute(ret), - }) + req_id + .map(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: transmute(ret), + }) + }) + .flatten() } - fn process_inner_noop(&self, work: F, socket_id: SocketId, req_id: u64) -> BackgroundTask + fn process_inner_noop( + &self, + work: F, + socket_id: SocketId, + req_id: Option, + ) -> BackgroundTask where F: FnOnce(&mut RemoteAdapterSocket) -> Result<(), NetworkError> + Send + 'static, { @@ -402,51 +594,46 @@ impl RemoteNetworkingAdapterDriver { ) } - fn process_inner_socket( + fn process_inner_begin_accept( &self, - work: F, socket_id: SocketId, child_id: SocketId, - req_id: u64, - ) -> BackgroundTask - where - F: FnOnce( - &mut RemoteAdapterSocket, - ) -> Result<(RemoteAdapterSocket, SocketAddr), NetworkError> - + Send - + 'static, - { - let common = self.common.clone(); - self.process_inner( - work, - move |ret| match ret { - Ok((mut socket, addr)) => { - let handler = Box::new(common.handler.clone().for_socket(child_id)); - - let err = match &mut socket { - RemoteAdapterSocket::TcpListener(s) => s.set_handler(handler), - RemoteAdapterSocket::TcpSocket(s) => s.set_handler(handler), - RemoteAdapterSocket::UdpSocket(s) => s.set_handler(handler), - RemoteAdapterSocket::IcmpSocket(s) => s.set_handler(handler), - RemoteAdapterSocket::RawSocket(s) => s.set_handler(handler), - }; - if let Err(err) = err { - return ResponseType::Err(err); - } - - let mut guard = common.sockets.lock().unwrap(); - guard.insert(child_id, socket); + req_id: Option, + ) -> BackgroundTask { + // We record the child socket so it can be used on the next accepted socket + { + let mut guard = self.common.socket_accept.lock().unwrap(); + guard.insert(socket_id, child_id); + } - ResponseType::SocketWithAddr { id: child_id, addr } + // Now we attach the handler to the main listening socket + let mut handler = Box::new(self.common.handler.clone().for_socket(socket_id)); + handler.interest(wasmer_virtual_io::InterestType::Readable); + self.process_inner_noop( + move |socket| match socket { + RemoteAdapterSocket::TcpListener { + socket: s, + next_accept, + .. + } => { + next_accept.replace(child_id); + s.set_handler(handler) + } + _ => { + // only the TCP listener needs its socket set as the other sockets + // set their handlers when the socket is created instead. we need to + // delay setting the handler so we have a child ID to use and return + // to the client when a socket is accepted, thus we can not accept them + // immediately + Err(NetworkError::Unsupported) } - Err(err) => ResponseType::Err(err), }, socket_id, req_id, ) } - fn process_interface(&mut self, req: RequestType, req_id: u64) -> BackgroundTask { + fn process_interface(&mut self, req: RequestType, req_id: Option) -> BackgroundTask { match req { RequestType::Bridge { network, @@ -544,7 +731,7 @@ impl RemoteNetworkingAdapterDriver { }, req_id, ), - RequestType::BindRaw(socket_id) => self.process_async_socket( + RequestType::BindRaw(socket_id) => self.process_async_new_socket( move |inner: OwnedMutexGuard>| async move { Ok(RemoteAdapterSocket::RawSocket(inner.bind_raw().await?)) }, @@ -557,13 +744,14 @@ impl RemoteNetworkingAdapterDriver { only_v6, reuse_port, reuse_addr, - } => self.process_async_socket( + } => self.process_async_new_socket( move |inner: OwnedMutexGuard>| async move { - Ok(RemoteAdapterSocket::TcpListener( - inner + Ok(RemoteAdapterSocket::TcpListener { + socket: inner .listen_tcp(addr, only_v6, reuse_port, reuse_addr) .await?, - )) + next_accept: None, + }) }, socket_id, req_id, @@ -573,7 +761,7 @@ impl RemoteNetworkingAdapterDriver { addr, reuse_port, reuse_addr, - } => self.process_async_socket( + } => self.process_async_new_socket( move |inner: OwnedMutexGuard>| async move { Ok(RemoteAdapterSocket::UdpSocket( inner.bind_udp(addr, reuse_port, reuse_addr).await?, @@ -582,7 +770,7 @@ impl RemoteNetworkingAdapterDriver { socket_id, req_id, ), - RequestType::BindIcmp { socket_id, addr } => self.process_async_socket( + RequestType::BindIcmp { socket_id, addr } => self.process_async_new_socket( move |inner: OwnedMutexGuard>| async move { Ok(RemoteAdapterSocket::IcmpSocket( inner.bind_icmp(addr).await?, @@ -595,7 +783,7 @@ impl RemoteNetworkingAdapterDriver { socket_id, addr, peer, - } => self.process_async_socket( + } => self.process_async_new_socket( move |inner: OwnedMutexGuard>| async move { Ok(RemoteAdapterSocket::TcpSocket( inner.connect_tcp(addr, peer).await?, @@ -618,10 +806,14 @@ impl RemoteNetworkingAdapterDriver { }, req_id, ), - _ => self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: ResponseType::Err(NetworkError::Unsupported), - }), + _ => req_id + .map(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::Unsupported), + }) + }) + .flatten(), } } @@ -629,7 +821,7 @@ impl RemoteNetworkingAdapterDriver { &mut self, socket_id: SocketId, req: RequestType, - req_id: u64, + req_id: Option, ) -> BackgroundTask { match req { RequestType::Flush => self.process_inner_noop( @@ -649,22 +841,13 @@ impl RemoteNetworkingAdapterDriver { socket_id, req_id, ), - RequestType::TryAccept(child_id) => self.process_inner_socket( - move |socket| match socket { - RemoteAdapterSocket::TcpListener(s) => match s.try_accept() { - Ok((socket, addr)) => Ok((RemoteAdapterSocket::TcpSocket(socket), addr)), - Err(err) => Err(err), - }, - _ => Err(NetworkError::Unsupported), - }, - socket_id, - child_id, - req_id, - ), + RequestType::BeginAccept(child_id) => { + self.process_inner_begin_accept(socket_id, child_id, req_id) + } RequestType::GetAddrLocal => self.process_inner( move |socket| match socket { RemoteAdapterSocket::TcpSocket(s) => s.addr_local(), - RemoteAdapterSocket::TcpListener(s) => s.addr_local(), + RemoteAdapterSocket::TcpListener { socket: s, .. } => s.addr_local(), RemoteAdapterSocket::UdpSocket(s) => s.addr_local(), RemoteAdapterSocket::IcmpSocket(s) => s.addr_local(), RemoteAdapterSocket::RawSocket(s) => s.addr_local(), @@ -679,7 +862,7 @@ impl RemoteNetworkingAdapterDriver { RequestType::GetAddrPeer => self.process_inner( move |socket| match socket { RemoteAdapterSocket::TcpSocket(s) => s.addr_peer().map(Some), - RemoteAdapterSocket::TcpListener(_) => Err(NetworkError::Unsupported), + RemoteAdapterSocket::TcpListener { .. } => Err(NetworkError::Unsupported), RemoteAdapterSocket::UdpSocket(s) => s.addr_peer(), RemoteAdapterSocket::IcmpSocket(_) => Err(NetworkError::Unsupported), RemoteAdapterSocket::RawSocket(_) => Err(NetworkError::Unsupported), @@ -695,7 +878,7 @@ impl RemoteNetworkingAdapterDriver { RequestType::SetTtl(ttl) => self.process_inner_noop( move |socket| match socket { RemoteAdapterSocket::TcpSocket(s) => s.set_ttl(ttl), - RemoteAdapterSocket::TcpListener(s) => { + RemoteAdapterSocket::TcpListener { socket: s, .. } => { s.set_ttl(ttl.try_into().unwrap_or_default()) } RemoteAdapterSocket::UdpSocket(s) => s.set_ttl(ttl), @@ -708,7 +891,7 @@ impl RemoteNetworkingAdapterDriver { RequestType::GetTtl => self.process_inner( move |socket| match socket { RemoteAdapterSocket::TcpSocket(s) => s.ttl(), - RemoteAdapterSocket::TcpListener(s) => s.ttl().map(|t| t as u32), + RemoteAdapterSocket::TcpListener { socket: s, .. } => s.ttl().map(|t| t as u32), RemoteAdapterSocket::UdpSocket(s) => s.ttl(), RemoteAdapterSocket::IcmpSocket(s) => s.ttl(), RemoteAdapterSocket::RawSocket(s) => s.ttl(), @@ -723,7 +906,7 @@ impl RemoteNetworkingAdapterDriver { RequestType::GetStatus => self.process_inner( move |socket| match socket { RemoteAdapterSocket::TcpSocket(s) => s.status(), - RemoteAdapterSocket::TcpListener(_) => Err(NetworkError::Unsupported), + RemoteAdapterSocket::TcpListener { .. } => Err(NetworkError::Unsupported), RemoteAdapterSocket::UdpSocket(s) => s.status(), RemoteAdapterSocket::IcmpSocket(s) => s.status(), RemoteAdapterSocket::RawSocket(s) => s.status(), @@ -976,16 +1159,23 @@ impl RemoteNetworkingAdapterDriver { socket_id, req_id, ), - _ => self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: ResponseType::Err(NetworkError::Unsupported), - }), + _ => req_id + .map(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::Unsupported), + }) + }) + .flatten(), } } } enum RemoteAdapterSocket { - TcpListener(Box), + TcpListener { + socket: Box, + next_accept: Option, + }, TcpSocket(Box), UdpSocket(Box), RawSocket(Box), @@ -1133,7 +1323,7 @@ impl RemoteAdapterSocket { }), } } - pub fn drain_reads( + pub fn drain_reads_and_accepts( &mut self, common: &Arc, socket_id: SocketId, @@ -1143,6 +1333,48 @@ impl RemoteAdapterSocket { let mut ret: FuturesOrdered> = Default::default(); loop { break match self { + Self::TcpListener { + socket, + next_accept, + } => { + if next_accept.is_some() { + match socket.try_accept() { + Ok((mut child_socket, addr)) => { + let child_id = next_accept.take().unwrap(); + + // We set the handler on the socket so that it can + let handler = Box::new(common.handler.clone().for_socket(child_id)); + child_socket.set_handler(handler).ok(); + + // We will fix up the socket in the background then notify + // the client of the new socket + let common = common.clone(); + ret.push_back(Box::pin(async move { + // Next we record the socket so that it is active + { + let child_socket = + RemoteAdapterSocket::TcpSocket(child_socket); + let mut guard = common.sockets.lock().unwrap(); + guard.insert(child_id, child_socket); + } + + // Lastly we tell the client about the new socket + if let Some(task) = common.send(MessageResponse::FinishAccept { + socket_id, + child_id, + addr, + }) { + task.await; + } + })); + } + Err(NetworkError::WouldBlock) => {} + Err(err) => { + tracing::error!("failed to accept socket - {}", err); + } + } + } + } Self::TcpSocket(this) => { let mut chunk: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; @@ -1222,7 +1454,6 @@ impl RemoteAdapterSocket { Err(_) => {} } } - _ => {} }; } @@ -1298,6 +1529,7 @@ struct RemoteAdapterCommon { rx: Mutex>, #[derivative(Debug = "ignore")] sockets: Mutex>, + socket_accept: Mutex>, handler: RemoteAdapterHandler, // The stall guard will prevent reads while its held and there are background tasks running diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs new file mode 100644 index 00000000000..493171dcd3d --- /dev/null +++ b/lib/virtual-net/src/tests.rs @@ -0,0 +1,154 @@ +use std::{ + net::{Ipv4Addr, SocketAddrV4}, + sync::atomic::{AtomicU16, Ordering}, +}; + +use tracing_test::traced_test; + +use crate::{ + host::LocalNetworking, meta::FrameSerializationFormat, RemoteNetworkingServer, + VirtualConnectedSocketExt, VirtualTcpListenerExt, +}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use super::*; + +async fn setup_mpsc() -> (RemoteNetworkingClient, RemoteNetworkingServer) { + tracing::info!("building MPSC channels"); + let (tx1, rx1) = tokio::sync::mpsc::channel(100); + let (tx2, rx2) = tokio::sync::mpsc::channel(100); + + tracing::info!("constructing remote client (mpsc)"); + let (client, client_driver) = RemoteNetworkingClient::new_from_mpsc(tx1, rx2); + + tracing::info!("spawning driver for remote client"); + tokio::task::spawn(client_driver); + + tracing::info!("create local networking provider"); + let local_networking = LocalNetworking::new(); + + tracing::info!("constructing remote server (mpsc)"); + let (server, server_driver) = + RemoteNetworkingServer::new_from_mpsc(tx2, rx1, Box::new(local_networking)); + + tracing::info!("spawning driver for remote server"); + tokio::task::spawn(server_driver); + + (client, server) +} + +async fn setup_pipe( + buf_size: usize, + format: FrameSerializationFormat, +) -> (RemoteNetworkingClient, RemoteNetworkingServer) { + tracing::info!("building duplex streams"); + let (tx1, rx1) = tokio::io::duplex(buf_size); + let (tx2, rx2) = tokio::io::duplex(buf_size); + + tracing::info!("constructing remote client (mpsc)"); + let (client, client_driver) = RemoteNetworkingClient::new_from_stream(tx1, rx2, format); + + tracing::info!("spawning driver for remote client"); + tokio::task::spawn(client_driver); + + tracing::info!("create local networking provider"); + let local_networking = LocalNetworking::new(); + + tracing::info!("constructing remote server (mpsc)"); + let (server, server_driver) = + RemoteNetworkingServer::new_from_stream(tx2, rx1, format, Box::new(local_networking)); + + tracing::info!("spawning driver for remote server"); + tokio::task::spawn(server_driver); + + (client, server) +} + +async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServer) { + static PORT: AtomicU16 = AtomicU16::new(8000); + let addr = SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::LOCALHOST, + PORT.fetch_add(1, Ordering::SeqCst), + )); + tracing::info!("listening on {addr}"); + let mut listener = client + .listen_tcp(addr.clone(), false, false, false) + .await + .unwrap(); + + const TEST1: &'static str = "the cat ran up the wall!"; + const TEST2: &'static str = "...and fell off the roof! raise the roof! oop oop"; + + tracing::info!("spawning acceptor worker thread"); + tokio::task::spawn(async move { + tracing::info!("waiting for connection"); + let (mut socket, addr) = listener.accept().await.unwrap(); + tracing::info!("accepted connection from {addr}"); + + tracing::info!("receiving data from client"); + let mut buf = [0u8; TEST1.len()]; + socket.read_exact(&mut buf).await.unwrap(); + + let msg = String::from_utf8_lossy(&buf); + assert_eq!(msg.as_ref(), TEST1); + + tracing::info!("sending back test string - {TEST2}"); + socket.send(TEST2.as_bytes()).await.unwrap(); + }); + + tracing::info!("connecting to listening socket"); + let mut socket = client + .connect_tcp( + SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0)), + addr, + ) + .await + .unwrap(); + + tracing::info!("sending test string - {TEST1}"); + socket.write_all(TEST1.as_bytes()).await.unwrap(); + + tracing::info!("receiving data from server"); + let mut buf = [0u8; TEST2.len()]; + socket.read_exact(&mut buf).await.unwrap(); + + let msg = String::from_utf8_lossy(&buf); + assert_eq!(msg.as_ref(), TEST2); + + tracing::info!("all good"); +} + +#[traced_test] +#[tokio::test] +async fn test_tcp_with_mpsc() { + let (client, server) = setup_mpsc().await; + test_tcp(client, server).await +} + +#[traced_test] +#[tokio::test] +async fn test_tcp_with_small_pipe_using_bincode() { + let (client, server) = setup_pipe(10, FrameSerializationFormat::Bincode).await; + test_tcp(client, server).await +} + +#[traced_test] +#[tokio::test] +async fn test_tcp_with_large_pipe_using_bincode() { + let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Bincode).await; + test_tcp(client, server).await +} + +#[traced_test] +#[tokio::test] +async fn test_tcp_with_small_pipe_using_json() { + let (client, server) = setup_pipe(10, FrameSerializationFormat::Json).await; + test_tcp(client, server).await +} + +#[traced_test] +#[tokio::test] +async fn test_tcp_with_large_pipe_json_using_json() { + let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Json).await; + test_tcp(client, server).await +} diff --git a/lib/virtual-net/src/write_all.rs b/lib/virtual-net/src/write_all.rs deleted file mode 100644 index 880d2571ae1..00000000000 --- a/lib/virtual-net/src/write_all.rs +++ /dev/null @@ -1,69 +0,0 @@ -use bytes::Buf; -use bytes::Bytes; -use pin_project_lite::pin_project; -use std::collections::VecDeque; -use std::future::Future; -use std::io; -use std::pin::Pin; -use std::sync::Arc; -use std::sync::Mutex; -use std::task::{Context, Poll}; -use tokio::io::AsyncWrite; - -pin_project! { - #[must_use = "futures do nothing unless you `.await` or poll them"] - pub struct WriteAll { - writer: Arc>>>, - bufs: VecDeque, - } -} - -pub(crate) fn locking_write_all( - writer: &Arc>>>, - buf: Bytes, -) -> WriteAll { - WriteAll { - writer: writer.clone(), - bufs: vec![buf].into(), - } -} - -pub(crate) fn locking_write_all_many( - writer: &Arc>>>, - bufs: I, -) -> WriteAll -where - I: IntoIterator, -{ - WriteAll { - writer: writer.clone(), - bufs: bufs.into_iter().collect(), - } -} - -impl Future for WriteAll { - type Output = io::Result<()>; - - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let me = self.project(); - - let mut writer = me.writer.lock().unwrap(); - while let Some(first) = me.bufs.front_mut() { - match writer.as_mut().poll_write(cx, &first) { - Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), - Poll::Ready(Ok(0)) => { - return Poll::Ready(Err(io::ErrorKind::ConnectionAborted.into())) - } - Poll::Ready(Ok(amt)) => { - first.advance(amt); - if first.is_empty() { - me.bufs.pop_front(); - continue; - } - } - Poll::Pending => return Poll::Pending, - } - } - Poll::Ready(Ok(())) - } -} diff --git a/lib/wasix/src/fs/inode_guard.rs b/lib/wasix/src/fs/inode_guard.rs index 97bb6f4a755..ddc534c8a39 100644 --- a/lib/wasix/src/fs/inode_guard.rs +++ b/lib/wasix/src/fs/inode_guard.rs @@ -748,42 +748,3 @@ fn is_err_closed(err: &std::io::Error) -> bool { || err.kind() == std::io::ErrorKind::NotConnected || err.kind() == std::io::ErrorKind::UnexpectedEof } - -pub fn net_error_into_io_err(net_error: NetworkError) -> std::io::Error { - use std::io::ErrorKind; - match net_error { - NetworkError::InvalidFd => ErrorKind::BrokenPipe.into(), - NetworkError::AlreadyExists => ErrorKind::AlreadyExists.into(), - NetworkError::Lock => ErrorKind::BrokenPipe.into(), - NetworkError::IOError => ErrorKind::BrokenPipe.into(), - NetworkError::AddressInUse => ErrorKind::AddrInUse.into(), - NetworkError::AddressNotAvailable => ErrorKind::AddrNotAvailable.into(), - NetworkError::BrokenPipe => ErrorKind::BrokenPipe.into(), - NetworkError::ConnectionAborted => ErrorKind::ConnectionAborted.into(), - NetworkError::ConnectionRefused => ErrorKind::ConnectionRefused.into(), - NetworkError::ConnectionReset => ErrorKind::ConnectionReset.into(), - NetworkError::Interrupted => ErrorKind::Interrupted.into(), - NetworkError::InvalidData => ErrorKind::InvalidData.into(), - NetworkError::InvalidInput => ErrorKind::InvalidInput.into(), - NetworkError::NotConnected => ErrorKind::NotConnected.into(), - NetworkError::NoDevice => ErrorKind::BrokenPipe.into(), - NetworkError::PermissionDenied => ErrorKind::PermissionDenied.into(), - NetworkError::TimedOut => ErrorKind::TimedOut.into(), - NetworkError::UnexpectedEof => ErrorKind::UnexpectedEof.into(), - NetworkError::WouldBlock => ErrorKind::WouldBlock.into(), - NetworkError::WriteZero => ErrorKind::WriteZero.into(), - NetworkError::Unsupported => ErrorKind::Unsupported.into(), - NetworkError::UnknownError => ErrorKind::BrokenPipe.into(), - NetworkError::InsufficientMemory => ErrorKind::OutOfMemory.into(), - NetworkError::TooManyOpenFiles => { - #[cfg(target_family = "unix")] - { - std::io::Error::from_raw_os_error(libc::EMFILE) - } - #[cfg(not(target_family = "unix"))] - { - ErrorKind::Other.into() - } - } - } -} From 558d41be538ecf01ba40086f756ebdee511c3a7c Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 19 Jul 2023 07:03:16 +1000 Subject: [PATCH 18/52] Added more serialization formats --- Cargo.lock | 24 ++++++++++++++++ lib/virtual-net/Cargo.toml | 9 ++++-- lib/virtual-net/src/client.rs | 25 +++++++++++++++++ lib/virtual-net/src/lib.rs | 9 ++++-- lib/virtual-net/src/meta.rs | 7 ++++- lib/virtual-net/src/server.rs | 28 +++++++++++++++++- lib/virtual-net/src/tests.rs | 53 +++++++++++++++++++++++++++++++++-- 7 files changed, 146 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13a2ecb2d0e..db19008474d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3581,6 +3581,28 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rmp" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44519172358fd6d58656c86ab8e7fbc9e1490c3e8f14d35ed78ca0dd07403c9f" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723ecff9ad04f4ad92fe1c8ca6c20d2196d9286e9c60727c4cb5511629260e9d" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + [[package]] name = "rpassword" version = "7.2.0" @@ -4692,7 +4714,9 @@ dependencies = [ "futures-core", "futures-sink", "pin-project", + "rmp-serde", "serde", + "serde_cbor", "serde_json", ] diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index bb4afbe6b12..bd759e39890 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -26,7 +26,7 @@ serde = { version = "1.0", default-features = false, features = ["derive"] } pin-project-lite = "0.2.9" futures-util = { version = "0.3" } anyhow = "1.0" -tokio-serde = { version = "0.8", features = [ "bincode", "json" ], optional = true } +tokio-serde = { version = "0.8", features = [ "bincode" ], optional = true } tokio-util = { version = "0.6", features = ["codec"], optional = true } [dev-dependencies] @@ -34,9 +34,12 @@ tokio = { version = "1", default_features = false, features = [ "macros" ] } tracing-test = { version = "0.2" } [features] -default = [ "host-net", "remote" ] -host-net = [ "tokio", "libc", "wasmer-virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] +default = [ "host-net", "remote", "json", "messagepack", "cbor" ] +host-net = [ "tokio", "libc", "tokio/io-util", "wasmer-virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] remote = [ "tokio", "libc", "tokio/io-util", "tokio/sync", "tokio-serde", "tokio-util" ] +json = [ "tokio-serde/json" ] +messagepack = [ "tokio-serde/messagepack" ] +cbor = [ "tokio-serde/cbor" ] [package.metadata.docs.rs] features = ["host-net", "remote"] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index ac38bc333f3..5a6844f9876 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -28,7 +28,12 @@ use tokio::sync::mpsc; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::mpsc::error::TrySendError; use tokio_serde::formats::SymmetricalBincode; +#[cfg(feature = "cbor")] +use tokio_serde::formats::SymmetricalCbor; +#[cfg(feature = "json")] use tokio_serde::formats::SymmetricalJson; +#[cfg(feature = "messagepack")] +use tokio_serde::formats::SymmetricalMessagePack; use tokio_serde::SymmetricallyFramed; use tokio_util::codec::FramedRead; use tokio_util::codec::FramedWrite; @@ -128,9 +133,19 @@ impl RemoteNetworkingClient { FrameSerializationFormat::Bincode => { Box::pin(SymmetricallyFramed::new(tx, SymmetricalBincode::default())) } + #[cfg(feature = "json")] FrameSerializationFormat::Json => { Box::pin(SymmetricallyFramed::new(tx, SymmetricalJson::default())) } + #[cfg(feature = "messagepack")] + FrameSerializationFormat::MessagePack => Box::pin(SymmetricallyFramed::new( + tx, + SymmetricalMessagePack::default(), + )), + #[cfg(feature = "cbor")] + FrameSerializationFormat::Cbor => { + Box::pin(SymmetricallyFramed::new(tx, SymmetricalCbor::default())) + } }; let rx = FramedRead::new(rx, LengthDelimitedCodec::new()); @@ -139,9 +154,19 @@ impl RemoteNetworkingClient { FrameSerializationFormat::Bincode => { Box::pin(SymmetricallyFramed::new(rx, SymmetricalBincode::default())) } + #[cfg(feature = "json")] FrameSerializationFormat::Json => { Box::pin(SymmetricallyFramed::new(rx, SymmetricalJson::default())) } + #[cfg(feature = "messagepack")] + FrameSerializationFormat::MessagePack => Box::pin(SymmetricallyFramed::new( + rx, + SymmetricalMessagePack::default(), + )), + #[cfg(feature = "cbor")] + FrameSerializationFormat::Cbor => { + Box::pin(SymmetricallyFramed::new(rx, SymmetricalCbor::default())) + } }; let (tx_work, rx_work) = mpsc::unbounded_channel(); diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index 09faad7f7d6..0416b49e5d6 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -9,6 +9,7 @@ pub mod meta; pub mod rx_tx; #[cfg(any(feature = "remote"))] pub mod server; +#[cfg(feature = "tokio")] #[cfg(test)] mod tests; @@ -30,7 +31,9 @@ use std::task::Context; use std::task::Poll; pub use std::time::Duration; use thiserror::Error; +#[cfg(feature = "tokio")] use tokio::io::AsyncRead; +#[cfg(feature = "tokio")] use tokio::io::AsyncWrite; pub use bytes::Bytes; @@ -581,6 +584,7 @@ pub trait VirtualTcpSocket: VirtualConnectedSocket + fmt::Debug + Send + Sync + fn is_closed(&self) -> bool; } +#[cfg(feature = "tokio")] impl<'a> AsyncRead for Box { fn poll_read( self: Pin<&mut Self>, @@ -605,6 +609,7 @@ impl<'a> AsyncRead for Box { } } +#[cfg(feature = "tokio")] impl<'a> AsyncWrite for Box { fn poll_write( self: Pin<&mut Self>, @@ -867,11 +872,11 @@ pub fn net_error_into_io_err(net_error: NetworkError) -> std::io::Error { NetworkError::UnknownError => ErrorKind::BrokenPipe.into(), NetworkError::InsufficientMemory => ErrorKind::OutOfMemory.into(), NetworkError::TooManyOpenFiles => { - #[cfg(target_family = "unix")] + #[cfg(all(target_family = "unix", feature = "libc"))] { std::io::Error::from_raw_os_error(libc::EMFILE) } - #[cfg(not(target_family = "unix"))] + #[cfg(not(all(target_family = "unix", feature = "libc")))] { ErrorKind::Other.into() } diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index 4a6ebd7c8f4..4b5c99afcb1 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -24,7 +24,12 @@ impl From for SocketId { #[derive(Copy, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] pub enum FrameSerializationFormat { Bincode, - Json + #[cfg(feature = "json")] + Json, + #[cfg(feature = "messagepack")] + MessagePack, + #[cfg(feature = "cbor")] + Cbor, } /// Possible values which can be passed to the [`TcpStream::shutdown`] method. diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index a15599f1ed4..819f134097b 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -27,7 +27,13 @@ use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, }; -use tokio_serde::formats::{SymmetricalBincode, SymmetricalJson}; +use tokio_serde::formats::SymmetricalBincode; +#[cfg(feature = "cbor")] +use tokio_serde::formats::SymmetricalCbor; +#[cfg(feature = "json")] +use tokio_serde::formats::SymmetricalJson; +#[cfg(feature = "messagepack")] +use tokio_serde::formats::SymmetricalMessagePack; use tokio_serde::SymmetricallyFramed; use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec}; use wasmer_virtual_io::InterestHandler; @@ -99,9 +105,19 @@ impl RemoteNetworkingServer { FrameSerializationFormat::Bincode => { Box::pin(SymmetricallyFramed::new(tx, SymmetricalBincode::default())) } + #[cfg(feature = "json")] FrameSerializationFormat::Json => { Box::pin(SymmetricallyFramed::new(tx, SymmetricalJson::default())) } + #[cfg(feature = "messagepack")] + FrameSerializationFormat::MessagePack => Box::pin(SymmetricallyFramed::new( + tx, + SymmetricalMessagePack::default(), + )), + #[cfg(feature = "cbor")] + FrameSerializationFormat::Cbor => { + Box::pin(SymmetricallyFramed::new(tx, SymmetricalCbor::default())) + } }; let rx = FramedRead::new(rx, LengthDelimitedCodec::new()); @@ -110,9 +126,19 @@ impl RemoteNetworkingServer { FrameSerializationFormat::Bincode => { Box::pin(SymmetricallyFramed::new(rx, SymmetricalBincode::default())) } + #[cfg(feature = "json")] FrameSerializationFormat::Json => { Box::pin(SymmetricallyFramed::new(rx, SymmetricalJson::default())) } + #[cfg(feature = "messagepack")] + FrameSerializationFormat::MessagePack => Box::pin(SymmetricallyFramed::new( + rx, + SymmetricalMessagePack::default(), + )), + #[cfg(feature = "cbor")] + FrameSerializationFormat::Cbor => { + Box::pin(SymmetricallyFramed::new(rx, SymmetricalCbor::default())) + } }; let (tx_work, rx_work) = mpsc::unbounded_channel(); diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs index 493171dcd3d..5dc6fd9eb49 100644 --- a/lib/virtual-net/src/tests.rs +++ b/lib/virtual-net/src/tests.rs @@ -1,3 +1,4 @@ +#![allow(unused)] use std::{ net::{Ipv4Addr, SocketAddrV4}, sync::atomic::{AtomicU16, Ordering}, @@ -5,14 +6,17 @@ use std::{ use tracing_test::traced_test; +#[cfg(feature = "remote")] +use crate::RemoteNetworkingServer; use crate::{ - host::LocalNetworking, meta::FrameSerializationFormat, RemoteNetworkingServer, - VirtualConnectedSocketExt, VirtualTcpListenerExt, + host::LocalNetworking, meta::FrameSerializationFormat, VirtualConnectedSocketExt, + VirtualTcpListenerExt, }; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use super::*; +#[cfg(feature = "remote")] async fn setup_mpsc() -> (RemoteNetworkingClient, RemoteNetworkingServer) { tracing::info!("building MPSC channels"); let (tx1, rx1) = tokio::sync::mpsc::channel(100); @@ -37,6 +41,7 @@ async fn setup_mpsc() -> (RemoteNetworkingClient, RemoteNetworkingServer) { (client, server) } +#[cfg(feature = "remote")] async fn setup_pipe( buf_size: usize, format: FrameSerializationFormat, @@ -64,6 +69,7 @@ async fn setup_pipe( (client, server) } +#[cfg(feature = "remote")] async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServer) { static PORT: AtomicU16 = AtomicU16::new(8000); let addr = SocketAddr::V4(SocketAddrV4::new( @@ -118,6 +124,7 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe tracing::info!("all good"); } +#[cfg(feature = "remote")] #[traced_test] #[tokio::test] async fn test_tcp_with_mpsc() { @@ -125,6 +132,7 @@ async fn test_tcp_with_mpsc() { test_tcp(client, server).await } +#[cfg(feature = "remote")] #[traced_test] #[tokio::test] async fn test_tcp_with_small_pipe_using_bincode() { @@ -132,6 +140,7 @@ async fn test_tcp_with_small_pipe_using_bincode() { test_tcp(client, server).await } +#[cfg(feature = "remote")] #[traced_test] #[tokio::test] async fn test_tcp_with_large_pipe_using_bincode() { @@ -139,6 +148,8 @@ async fn test_tcp_with_large_pipe_using_bincode() { test_tcp(client, server).await } +#[cfg(feature = "remote")] +#[cfg(feature = "json")] #[traced_test] #[tokio::test] async fn test_tcp_with_small_pipe_using_json() { @@ -146,9 +157,47 @@ async fn test_tcp_with_small_pipe_using_json() { test_tcp(client, server).await } +#[cfg(feature = "remote")] +#[cfg(feature = "json")] #[traced_test] #[tokio::test] async fn test_tcp_with_large_pipe_json_using_json() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Json).await; test_tcp(client, server).await } + +#[cfg(feature = "remote")] +#[cfg(feature = "messagepack")] +#[traced_test] +#[tokio::test] +async fn test_tcp_with_small_pipe_using_messagepack() { + let (client, server) = setup_pipe(10, FrameSerializationFormat::MessagePack).await; + test_tcp(client, server).await +} + +#[cfg(feature = "remote")] +#[cfg(feature = "messagepack")] +#[traced_test] +#[tokio::test] +async fn test_tcp_with_large_pipe_json_using_messagepack() { + let (client, server) = setup_pipe(1024000, FrameSerializationFormat::MessagePack).await; + test_tcp(client, server).await +} + +#[cfg(feature = "remote")] +#[cfg(feature = "cbor")] +#[traced_test] +#[tokio::test] +async fn test_tcp_with_small_pipe_using_cbor() { + let (client, server) = setup_pipe(10, FrameSerializationFormat::Cbor).await; + test_tcp(client, server).await +} + +#[cfg(feature = "remote")] +#[cfg(feature = "cbor")] +#[traced_test] +#[tokio::test] +async fn test_tcp_with_large_pipe_json_using_cbor() { + let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Cbor).await; + test_tcp(client, server).await +} From 4ffd1e7fa7b7077239360825d68b0c7d2824a245 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 19 Jul 2023 07:09:32 +1000 Subject: [PATCH 19/52] Renamed the construction methods --- lib/virtual-net/src/client.rs | 2 +- lib/virtual-net/src/server.rs | 2 +- lib/virtual-net/src/tests.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 5a6844f9876..c144888fdda 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -118,7 +118,7 @@ impl RemoteNetworkingClient { /// /// This version will run the async read and write operations /// only the driver (this is needed for mixed runtimes) - pub fn new_from_stream( + pub fn new_from_async_io( tx: TX, rx: RX, format: FrameSerializationFormat, diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index 819f134097b..d8f530d97c2 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -89,7 +89,7 @@ impl RemoteNetworkingServer { /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels - pub fn new_from_stream( + pub fn new_from_async_io( tx: TX, rx: RX, format: FrameSerializationFormat, diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs index 5dc6fd9eb49..4af10f7d134 100644 --- a/lib/virtual-net/src/tests.rs +++ b/lib/virtual-net/src/tests.rs @@ -51,7 +51,7 @@ async fn setup_pipe( let (tx2, rx2) = tokio::io::duplex(buf_size); tracing::info!("constructing remote client (mpsc)"); - let (client, client_driver) = RemoteNetworkingClient::new_from_stream(tx1, rx2, format); + let (client, client_driver) = RemoteNetworkingClient::new_from_async_io(tx1, rx2, format); tracing::info!("spawning driver for remote client"); tokio::task::spawn(client_driver); @@ -61,7 +61,7 @@ async fn setup_pipe( tracing::info!("constructing remote server (mpsc)"); let (server, server_driver) = - RemoteNetworkingServer::new_from_stream(tx2, rx1, format, Box::new(local_networking)); + RemoteNetworkingServer::new_from_async_io(tx2, rx1, format, Box::new(local_networking)); tracing::info!("spawning driver for remote server"); tokio::task::spawn(server_driver); From 83ca434fe45cd5297d54d947a6a078ac8d6b35ba Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 20 Jul 2023 07:58:24 +1000 Subject: [PATCH 20/52] Fixed some compile errors after the virtual-net changes --- lib/wasix/src/fs/inode_guard.rs | 2 +- lib/wasix/src/fs/mod.rs | 5 ++--- lib/wasix/src/net/socket.rs | 2 +- lib/wasix/src/syscalls/wasix/epoll_ctl.rs | 5 +++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/wasix/src/fs/inode_guard.rs b/lib/wasix/src/fs/inode_guard.rs index ddc534c8a39..1f4b3498035 100644 --- a/lib/wasix/src/fs/inode_guard.rs +++ b/lib/wasix/src/fs/inode_guard.rs @@ -12,7 +12,7 @@ use futures::future::BoxFuture; use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite}; use virtual_fs::{FsError, Pipe as VirtualPipe, VirtualFile}; use wasmer_virtual_io::{InterestType, StatefulHandler}; -use wasmer_virtual_net::NetworkError; +use wasmer_virtual_net::net_error_into_io_err; use wasmer_wasix_types::{ types::Eventtype, wasi::{self, EpollType}, diff --git a/lib/wasix/src/fs/mod.rs b/lib/wasix/src/fs/mod.rs index 091d560b4b2..f57f753c95f 100644 --- a/lib/wasix/src/fs/mod.rs +++ b/lib/wasix/src/fs/mod.rs @@ -32,9 +32,8 @@ use wasmer_wasix_types::{ pub use self::fd::{EpollFd, EpollInterest, EpollJoinGuard, Fd, InodeVal, Kind}; pub(crate) use self::inode_guard::{ - net_error_into_io_err, InodeValFilePollGuard, InodeValFilePollGuardJoin, - InodeValFilePollGuardMode, InodeValFileReadGuard, InodeValFileWriteGuard, WasiStateFileGuard, - POLL_GUARD_MAX_RET, + InodeValFilePollGuard, InodeValFilePollGuardJoin, InodeValFilePollGuardMode, + InodeValFileReadGuard, InodeValFileWriteGuard, WasiStateFileGuard, POLL_GUARD_MAX_RET, }; pub use self::notification::NotificationInner; use crate::syscalls::map_io_err; diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index 38c9c7fff90..0dd927f6dc0 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -1259,7 +1259,7 @@ impl InodeSocketProtected { pub fn set_handler( &mut self, handler: Box, - ) -> virtual_net::Result<()> { + ) -> wasmer_virtual_net::Result<()> { match &mut self.kind { InodeSocketKind::TcpListener { socket, .. } => socket.set_handler(handler), InodeSocketKind::TcpStream { socket, .. } => socket.set_handler(handler), diff --git a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs index c0456cfd51e..59169b2210e 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use tokio::sync::{mpsc::UnboundedSender, watch}; use wasmer_virtual_io::{InterestHandler, InterestType}; +use wasmer_virtual_net::net_error_into_io_err; use wasmer_wasix_types::wasi::{ EpollCtl, EpollEvent, EpollType, SubscriptionClock, SubscriptionUnion, Userdata, }; @@ -12,8 +13,8 @@ use futures::Future; use super::*; use crate::{ fs::{ - net_error_into_io_err, EpollFd, EpollInterest, EpollJoinGuard, InodeValFilePollGuard, - InodeValFilePollGuardJoin, InodeValFilePollGuardMode, POLL_GUARD_MAX_RET, + EpollFd, EpollInterest, EpollJoinGuard, InodeValFilePollGuard, InodeValFilePollGuardJoin, + InodeValFilePollGuardMode, POLL_GUARD_MAX_RET, }, state::PollEventSet, syscalls::*, From e53029811283809d0ac1f75221691479c6079c86 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 20 Jul 2023 14:43:31 +1000 Subject: [PATCH 21/52] Added support for hyper web socket with virtual-net --- Cargo.lock | 71 ++++++++- lib/virtual-net/Cargo.toml | 5 +- lib/virtual-net/src/rx_tx.rs | 184 ++++++++++++++++++++++- lib/virtual-net/src/server.rs | 268 +++++++++++++++++----------------- lib/virtual-net/src/tests.rs | 4 +- 5 files changed, 390 insertions(+), 142 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db19008474d..937abaf8315 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1148,6 +1148,12 @@ dependencies = [ "parking_lot_core 0.9.8", ] +[[package]] +name = "data-encoding" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" + [[package]] name = "derivative" version = "2.2.0" @@ -2032,6 +2038,19 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyper-tungstenite" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "226df6fd0aece319a325419d770aa9d947defa60463f142cd82b329121f906a3" +dependencies = [ + "hyper", + "pin-project", + "tokio", + "tokio-tungstenite", + "tungstenite", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -4122,6 +4141,17 @@ dependencies = [ "sha1_smol", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha1_smol" version = "1.0.0" @@ -4324,7 +4354,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha1", + "sha1 0.6.1", "syn 1.0.109", ] @@ -4720,6 +4750,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio-tungstenite" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec509ac96e9a0c43427c74f003127d953a265737636129424288d27cb5c4b12c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.6.10" @@ -4989,6 +5031,25 @@ dependencies = [ "termcolor", ] +[[package]] +name = "tungstenite" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15fba1a6d6bb030745759a9a2a588bfe8490fc8b4751a277db3a0be1c9ebbf67" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1 0.10.5", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.16.0" @@ -5118,6 +5179,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8parse" version = "0.2.1" @@ -6233,6 +6300,8 @@ dependencies = [ "bytes", "derivative", "futures-util", + "hyper", + "hyper-tungstenite", "libc", "mio", "pin-project-lite", diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index bd759e39890..87d0270990b 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -28,18 +28,21 @@ futures-util = { version = "0.3" } anyhow = "1.0" tokio-serde = { version = "0.8", features = [ "bincode" ], optional = true } tokio-util = { version = "0.6", features = ["codec"], optional = true } +hyper-tungstenite = { version = "0.10", optional = true } +hyper = { version = "0.14", optional = true } [dev-dependencies] tokio = { version = "1", default_features = false, features = [ "macros" ] } tracing-test = { version = "0.2" } [features] -default = [ "host-net", "remote", "json", "messagepack", "cbor" ] +default = [ "host-net", "remote", "json", "messagepack", "cbor", "hyper" ] host-net = [ "tokio", "libc", "tokio/io-util", "wasmer-virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] remote = [ "tokio", "libc", "tokio/io-util", "tokio/sync", "tokio-serde", "tokio-util" ] json = [ "tokio-serde/json" ] messagepack = [ "tokio-serde/messagepack" ] cbor = [ "tokio-serde/cbor" ] +hyper = [ "hyper-tungstenite", "dep:hyper" ] [package.metadata.docs.rs] features = ["host-net", "remote"] diff --git a/lib/virtual-net/src/rx_tx.rs b/lib/virtual-net/src/rx_tx.rs index 9145192e3a2..afa2bfb6312 100644 --- a/lib/virtual-net/src/rx_tx.rs +++ b/lib/virtual-net/src/rx_tx.rs @@ -5,7 +5,7 @@ use std::{ task::{Context, Poll, Waker}, }; -use crate::Result; +use crate::{meta::FrameSerializationFormat, Result}; use futures_util::{future::BoxFuture, Future, Sink, SinkExt, Stream}; use serde::Serialize; use tokio::{ @@ -70,6 +70,19 @@ where work: mpsc::UnboundedSender>, wakers: RemoteTxWakers, }, + WebSocket { + tx: Arc< + tokio::sync::Mutex< + futures_util::stream::SplitSink< + hyper_tungstenite::WebSocketStream, + hyper_tungstenite::tungstenite::Message, + >, + >, + >, + work: mpsc::UnboundedSender>, + wakers: RemoteTxWakers, + format: FrameSerializationFormat, + }, } impl RemoteTx where @@ -97,6 +110,24 @@ where .await .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) } + RemoteTx::WebSocket { tx, format, .. } => { + let data = match format { + FrameSerializationFormat::Bincode => { + bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {err}"); + NetworkError::IOError + })? + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + return Err(NetworkError::IOError); + } + }; + let mut tx = tx.lock().await; + tx.send(hyper_tungstenite::tungstenite::Message::Binary(data)) + .await + .map_err(|_| NetworkError::ConnectionAborted) + } } } @@ -134,6 +165,64 @@ where return Poll::Ready(Ok(())); } + // Otherwise we push it to the driver which will block all future send + // operations until it finishes + work.send(job).map_err(|err| { + tracing::error!("failed to send remaining bytes for request - {}", err); + NetworkError::ConnectionAborted + })?; + Poll::Ready(Ok(())) + } + RemoteTx::WebSocket { + tx, + format, + work, + wakers, + .. + } => { + let mut tx_guard = match tx.clone().try_lock_owned() { + Ok(lock) => lock, + Err(_) => { + wakers.add(cx.waker()); + return Poll::Pending; + } + }; + match tx_guard.poll_ready_unpin(cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => { + tracing::warn!("failed to poll web socket for readiness - {err}"); + return Poll::Ready(Err(NetworkError::IOError)); + } + Poll::Pending => return Poll::Pending, + } + + let data = match format { + FrameSerializationFormat::Bincode => { + bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {err}"); + NetworkError::IOError + })? + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + return Poll::Ready(Err(NetworkError::IOError)); + } + }; + + let mut job = Box::pin(async move { + if let Err(err) = tx_guard + .send(hyper_tungstenite::tungstenite::Message::Binary(data)) + .await + { + tracing::error!("failed to send remaining bytes for request - {}", err); + } + }); + + // First we try to finish it synchronously + if job.as_mut().poll(cx).is_ready() { + return Poll::Ready(Ok(())); + } + // Otherwise we push it to the driver which will block all future send // operations until it finishes work.send(job).map_err(|err| { @@ -188,6 +277,64 @@ where return Ok(()); } + // Otherwise we push it to the driver which will block all future send + // operations until it finishes + work.send(job).map_err(|err| { + tracing::error!("failed to send remaining bytes for request - {}", err); + NetworkError::ConnectionAborted + })?; + Ok(()) + } + RemoteTx::WebSocket { + tx, format, work, .. + } => { + let data = match format { + FrameSerializationFormat::Bincode => { + bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {err}"); + NetworkError::IOError + })? + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + return Err(NetworkError::IOError); + } + }; + + let mut tx_guard = match tx.clone().try_lock_owned() { + Ok(lock) => lock, + Err(_) => { + let tx = tx.clone(); + work.send(Box::pin(async move { + let mut tx_guard = tx.lock().await; + tx_guard + .send(hyper_tungstenite::tungstenite::Message::Binary(data)) + .await + .ok(); + })) + .ok(); + return Ok(()); + } + }; + + let inline_waker = InlineWaker::new(); + let waker = inline_waker.as_waker(); + let mut cx = Context::from_waker(&waker); + + let mut job = Box::pin(async move { + if let Err(err) = tx_guard + .send(hyper_tungstenite::tungstenite::Message::Binary(data)) + .await + { + tracing::error!("failed to send remaining bytes for request - {}", err); + } + }); + + // First we try to finish it synchronously + if job.as_mut().poll(&mut cx).is_ready() { + return Ok(()); + } + // Otherwise we push it to the driver which will block all future send // operations until it finishes work.send(job).map_err(|err| { @@ -211,6 +358,12 @@ where Stream { rx: Pin> + Send + 'static>>, }, + WebSocket { + rx: futures_util::stream::SplitStream< + hyper_tungstenite::WebSocketStream, + >, + format: FrameSerializationFormat, + }, } impl RemoteRx where @@ -235,6 +388,35 @@ where Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, }, + RemoteRx::WebSocket { rx, format } => match Pin::new(rx).poll_next(cx) { + Poll::Ready(Some(Ok(hyper_tungstenite::tungstenite::Message::Binary(msg)))) => { + match format { + FrameSerializationFormat::Bincode => { + return match bincode::deserialize(&msg) { + Ok(msg) => Poll::Ready(Some(msg)), + Err(err) => { + tracing::warn!("failed to deserialize message - {}", err); + continue; + } + } + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + continue; + } + } + } + Poll::Ready(Some(Ok(msg))) => { + tracing::warn!("unsupported message from channel - {}", msg); + continue; + } + Poll::Ready(Some(Err(err))) => { + tracing::warn!("failed to read from channel - {}", err); + Poll::Ready(None) + } + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => Poll::Pending, + }, }; } } diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index d8f530d97c2..7290bbbb491 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -6,9 +6,11 @@ use crate::{ }; use crate::{IpCidr, IpRoute, NetworkError, StreamSecurity, VirtualIcmpSocket}; use derivative::Derivative; -use futures_util::stream::FuturesOrdered; +use futures_util::stream::{FuturesOrdered, SplitSink, SplitStream}; use futures_util::{future::BoxFuture, StreamExt}; use futures_util::{Sink, Stream}; +#[cfg(feature = "hyper")] +use hyper_tungstenite::WebSocketStream; use std::collections::HashSet; use std::mem::MaybeUninit; use std::net::IpAddr; @@ -22,7 +24,6 @@ use std::{ sync::{Arc, Mutex}, task::{Context, Poll}, }; -use tokio::sync::OwnedMutexGuard; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, @@ -44,30 +45,19 @@ type BackgroundTask = Option>; pub struct RemoteNetworkingServer { #[allow(dead_code)] common: Arc, - inner: Arc>>, + inner: Arc, } impl RemoteNetworkingServer { - /// Creates a new interface on the remote location using - /// a unique interface ID and a pair of channels - pub fn new_from_mpsc( - tx: mpsc::Sender, - rx: mpsc::Receiver, - inner: Box, + fn new( + tx: RemoteTx, + rx: RemoteRx, + work: mpsc::UnboundedReceiver>, + inner: Arc, ) -> (Self, RemoteNetworkingServerDriver) { - let (tx_work, rx_work) = mpsc::unbounded_channel(); - let tx_wakers = RemoteTxWakers::default(); - let common = RemoteAdapterCommon { - tx: RemoteTx::Mpsc { - tx, - work: tx_work, - wakers: tx_wakers.clone(), - }, - rx: Mutex::new(RemoteRx::Mpsc { - rx, - wakers: tx_wakers, - }), + tx, + rx: Mutex::new(rx), sockets: Default::default(), socket_accept: Default::default(), handler: Default::default(), @@ -75,9 +65,8 @@ impl RemoteNetworkingServer { }; let common = Arc::new(common); - let inner = Arc::new(tokio::sync::Mutex::new(inner)); let driver = RemoteNetworkingServerDriver { - more_work: rx_work, + more_work: work, tasks: Default::default(), common: common.clone(), inner: inner.clone(), @@ -86,6 +75,27 @@ impl RemoteNetworkingServer { (networking, driver) } + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_mpsc( + tx: mpsc::Sender, + rx: mpsc::Receiver, + inner: Arc, + ) -> (Self, RemoteNetworkingServerDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + let tx_wakers = RemoteTxWakers::default(); + + let tx = RemoteTx::Mpsc { + tx, + work: tx_work, + wakers: tx_wakers.clone(), + }; + let rx = RemoteRx::Mpsc { + rx, + wakers: tx_wakers, + }; + Self::new(tx, rx, rx_work, inner) + } /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels @@ -93,7 +103,7 @@ impl RemoteNetworkingServer { tx: TX, rx: RX, format: FrameSerializationFormat, - inner: Box, + inner: Arc, ) -> (Self, RemoteNetworkingServerDriver) where TX: AsyncWrite + Send + 'static, @@ -142,33 +152,38 @@ impl RemoteNetworkingServer { }; let (tx_work, rx_work) = mpsc::unbounded_channel(); - let tx_wakers = RemoteTxWakers::default(); - let handler = RemoteAdapterHandler::default(); - let common = RemoteAdapterCommon { - tx: RemoteTx::Stream { - tx: Arc::new(tokio::sync::Mutex::new(tx)), - work: tx_work, - wakers: tx_wakers.clone(), - }, - rx: Mutex::new(RemoteRx::Stream { rx }), - sockets: Default::default(), - socket_accept: Default::default(), - handler, - stall_rx: Default::default(), + let tx = RemoteTx::Stream { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + wakers: RemoteTxWakers::default(), }; - let common = Arc::new(common); + let rx = RemoteRx::Stream { rx }; + Self::new(tx, rx, rx_work, inner) + } - let inner = Arc::new(tokio::sync::Mutex::new(inner)); - let driver = RemoteNetworkingServerDriver { - more_work: rx_work, - tasks: Default::default(), - common: common.clone(), - inner: inner.clone(), - }; - let networking = Self { common, inner }; + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + #[cfg(feature = "hyper")] + pub fn new_from_ws_io( + tx: SplitSink< + WebSocketStream, + hyper_tungstenite::tungstenite::Message, + >, + rx: SplitStream>, + format: FrameSerializationFormat, + inner: Arc, + ) -> (Self, RemoteNetworkingServerDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); - (networking, driver) + let tx = RemoteTx::WebSocket { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + wakers: RemoteTxWakers::default(), + format, + }; + let rx = RemoteRx::WebSocket { rx, format }; + Self::new(tx, rx, rx_work, inner) } } @@ -180,48 +195,39 @@ impl VirtualNetworking for RemoteNetworkingServer { access_token: &str, security: StreamSecurity, ) -> Result<(), NetworkError> { - let inner = self.inner.lock().await; - inner.bridge(network, access_token, security).await + self.inner.bridge(network, access_token, security).await } async fn unbridge(&self) -> Result<(), NetworkError> { - let inner = self.inner.lock().await; - inner.unbridge().await + self.inner.unbridge().await } async fn dhcp_acquire(&self) -> Result, NetworkError> { - let inner = self.inner.lock().await; - inner.dhcp_acquire().await + self.inner.dhcp_acquire().await } fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<(), NetworkError> { - let inner = self.inner.blocking_lock(); - inner.ip_add(ip, prefix) + self.inner.ip_add(ip, prefix) } fn ip_remove(&self, ip: IpAddr) -> Result<(), NetworkError> { - let inner = self.inner.blocking_lock(); - inner.ip_remove(ip) + self.inner.ip_remove(ip) } fn ip_clear(&self) -> Result<(), NetworkError> { - let inner = self.inner.blocking_lock(); - inner.ip_clear() + self.inner.ip_clear() } fn ip_list(&self) -> Result, NetworkError> { - let inner = self.inner.blocking_lock(); - inner.ip_list() + self.inner.ip_list() } fn mac(&self) -> Result<[u8; 6], NetworkError> { - let inner = self.inner.blocking_lock(); - inner.mac() + self.inner.mac() } fn gateway_set(&self, ip: IpAddr) -> Result<(), NetworkError> { - let inner = self.inner.blocking_lock(); - inner.gateway_set(ip) + self.inner.gateway_set(ip) } fn route_add( @@ -231,28 +237,24 @@ impl VirtualNetworking for RemoteNetworkingServer { preferred_until: Option, expires_at: Option, ) -> Result<(), NetworkError> { - let inner = self.inner.blocking_lock(); - inner.route_add(cidr, via_router, preferred_until, expires_at) + self.inner + .route_add(cidr, via_router, preferred_until, expires_at) } fn route_remove(&self, cidr: IpAddr) -> Result<(), NetworkError> { - let inner = self.inner.blocking_lock(); - inner.route_remove(cidr) + self.inner.route_remove(cidr) } fn route_clear(&self) -> Result<(), NetworkError> { - let inner = self.inner.blocking_lock(); - inner.route_clear() + self.inner.route_clear() } fn route_list(&self) -> Result, NetworkError> { - let inner = self.inner.blocking_lock(); - inner.route_list() + self.inner.route_list() } async fn bind_raw(&self) -> Result, NetworkError> { - let inner = self.inner.lock().await; - inner.bind_raw().await + self.inner.bind_raw().await } async fn listen_tcp( @@ -262,8 +264,7 @@ impl VirtualNetworking for RemoteNetworkingServer { reuse_port: bool, reuse_addr: bool, ) -> Result, NetworkError> { - let inner = self.inner.lock().await; - inner + self.inner .listen_tcp(addr, only_v6, reuse_port, reuse_addr) .await } @@ -274,16 +275,14 @@ impl VirtualNetworking for RemoteNetworkingServer { reuse_port: bool, reuse_addr: bool, ) -> Result, NetworkError> { - let inner = self.inner.lock().await; - inner.bind_udp(addr, reuse_port, reuse_addr).await + self.inner.bind_udp(addr, reuse_port, reuse_addr).await } async fn bind_icmp( &self, addr: IpAddr, ) -> Result, NetworkError> { - let inner = self.inner.lock().await; - inner.bind_icmp(addr).await + self.inner.bind_icmp(addr).await } async fn connect_tcp( @@ -291,8 +290,7 @@ impl VirtualNetworking for RemoteNetworkingServer { addr: SocketAddr, peer: SocketAddr, ) -> Result, NetworkError> { - let inner = self.inner.lock().await; - inner.connect_tcp(addr, peer).await + self.inner.connect_tcp(addr, peer).await } async fn resolve( @@ -301,8 +299,7 @@ impl VirtualNetworking for RemoteNetworkingServer { port: Option, dns_server: Option, ) -> Result, NetworkError> { - let inner = self.inner.lock().await; - inner.resolve(host, port, dns_server).await + self.inner.resolve(host, port, dns_server).await } } @@ -312,7 +309,7 @@ pin_project_lite::pin_project! { more_work: mpsc::UnboundedReceiver>, #[pin] tasks: FuturesOrdered>, - inner: Arc>>, + inner: Arc, } } @@ -476,16 +473,13 @@ impl RemoteNetworkingServerDriver { req_id: Option, ) -> BackgroundTask where - F: FnOnce(OwnedMutexGuard>) -> Fut - + Send - + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, Fut: Future + Send + 'static, T: FnOnce(Fut::Output) -> ResponseType + Send + 'static, { let inner = self.inner.clone(); let common = self.common.clone(); Self::process_async(async move { - let inner = inner.lock_owned().await; let future = work(inner); let ret = future.await; req_id @@ -501,9 +495,7 @@ impl RemoteNetworkingServerDriver { fn process_async_noop(&self, work: F, req_id: Option) -> BackgroundTask where - F: FnOnce(OwnedMutexGuard>) -> Fut - + Send - + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, Fut: Future> + Send + 'static, { self.process_async_inner( @@ -523,9 +515,7 @@ impl RemoteNetworkingServerDriver { req_id: Option, ) -> BackgroundTask where - F: FnOnce(OwnedMutexGuard>) -> Fut - + Send - + 'static, + F: FnOnce(Arc) -> Fut + Send + 'static, Fut: Future> + Send + 'static, { let common = self.common.clone(); @@ -681,37 +671,39 @@ impl RemoteNetworkingServerDriver { req_id, ), RequestType::IpAdd { ip, prefix } => self.process_async_noop( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { inner.ip_add(ip, prefix) }, req_id, ), RequestType::IpRemove(ip) => self.process_async_noop( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { inner.ip_remove(ip) }, req_id, ), - RequestType::IpClear => self.process_async_noop( - move |inner: OwnedMutexGuard>| async move { - inner.ip_clear() - }, - req_id, - ), - RequestType::GetIpList => self.process_async_inner( - move |inner: OwnedMutexGuard>| async move { - inner.ip_list() - }, - |ret| match ret { - Ok(cidr) => ResponseType::CidrList(cidr), - Err(err) => ResponseType::Err(err), - }, - req_id, - ), + RequestType::IpClear => { + self.process_async_noop( + move |inner: Arc| async move { + inner.ip_clear() + }, + req_id, + ) + } + RequestType::GetIpList => { + self.process_async_inner( + move |inner: Arc| async move { + inner.ip_list() + }, + |ret| match ret { + Ok(cidr) => ResponseType::CidrList(cidr), + Err(err) => ResponseType::Err(err), + }, + req_id, + ) + } RequestType::GetMac => self.process_async_inner( - move |inner: OwnedMutexGuard>| async move { - inner.mac() - }, + move |inner: Arc| async move { inner.mac() }, |ret| match ret { Ok(mac) => ResponseType::Mac(mac), Err(err) => ResponseType::Err(err), @@ -719,7 +711,7 @@ impl RemoteNetworkingServerDriver { req_id, ), RequestType::GatewaySet(ip) => self.process_async_noop( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { inner.gateway_set(ip) }, req_id, @@ -730,35 +722,37 @@ impl RemoteNetworkingServerDriver { preferred_until, expires_at, } => self.process_async_noop( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { inner.route_add(cidr, via_router, preferred_until, expires_at) }, req_id, ), RequestType::RouteRemove(ip) => self.process_async_noop( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { inner.route_remove(ip) }, req_id, ), RequestType::RouteClear => self.process_async_noop( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { inner.route_clear() }, req_id, ), - RequestType::GetRouteList => self.process_async_inner( - move |inner: OwnedMutexGuard>| async move { - inner.route_list() - }, - |ret| match ret { - Ok(routes) => ResponseType::RouteList(routes), - Err(err) => ResponseType::Err(err), - }, - req_id, - ), + RequestType::GetRouteList => { + self.process_async_inner( + move |inner: Arc| async move { + inner.route_list() + }, + |ret| match ret { + Ok(routes) => ResponseType::RouteList(routes), + Err(err) => ResponseType::Err(err), + }, + req_id, + ) + } RequestType::BindRaw(socket_id) => self.process_async_new_socket( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { Ok(RemoteAdapterSocket::RawSocket(inner.bind_raw().await?)) }, socket_id, @@ -771,7 +765,7 @@ impl RemoteNetworkingServerDriver { reuse_port, reuse_addr, } => self.process_async_new_socket( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { Ok(RemoteAdapterSocket::TcpListener { socket: inner .listen_tcp(addr, only_v6, reuse_port, reuse_addr) @@ -788,7 +782,7 @@ impl RemoteNetworkingServerDriver { reuse_port, reuse_addr, } => self.process_async_new_socket( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { Ok(RemoteAdapterSocket::UdpSocket( inner.bind_udp(addr, reuse_port, reuse_addr).await?, )) @@ -797,7 +791,7 @@ impl RemoteNetworkingServerDriver { req_id, ), RequestType::BindIcmp { socket_id, addr } => self.process_async_new_socket( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { Ok(RemoteAdapterSocket::IcmpSocket( inner.bind_icmp(addr).await?, )) @@ -810,7 +804,7 @@ impl RemoteNetworkingServerDriver { addr, peer, } => self.process_async_new_socket( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { Ok(RemoteAdapterSocket::TcpSocket( inner.connect_tcp(addr, peer).await?, )) @@ -823,7 +817,7 @@ impl RemoteNetworkingServerDriver { port, dns_server, } => self.process_async_inner( - move |inner: OwnedMutexGuard>| async move { + move |inner: Arc| async move { inner.resolve(&host, port, dns_server).await }, |ret| match ret { diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs index 4af10f7d134..dec3f97904e 100644 --- a/lib/virtual-net/src/tests.rs +++ b/lib/virtual-net/src/tests.rs @@ -33,7 +33,7 @@ async fn setup_mpsc() -> (RemoteNetworkingClient, RemoteNetworkingServer) { tracing::info!("constructing remote server (mpsc)"); let (server, server_driver) = - RemoteNetworkingServer::new_from_mpsc(tx2, rx1, Box::new(local_networking)); + RemoteNetworkingServer::new_from_mpsc(tx2, rx1, Arc::new(local_networking)); tracing::info!("spawning driver for remote server"); tokio::task::spawn(server_driver); @@ -61,7 +61,7 @@ async fn setup_pipe( tracing::info!("constructing remote server (mpsc)"); let (server, server_driver) = - RemoteNetworkingServer::new_from_async_io(tx2, rx1, format, Box::new(local_networking)); + RemoteNetworkingServer::new_from_async_io(tx2, rx1, format, Arc::new(local_networking)); tracing::info!("spawning driver for remote server"); tokio::task::spawn(server_driver); From a4f5b4240ef583f89ed7cd91c99af7d97b76c71d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 20 Jul 2023 23:12:49 +1000 Subject: [PATCH 22/52] Bit more refactoring of the packages and implemented more channels --- Cargo.lock | 142 +++++++++------ lib/cli/Cargo.toml | 4 +- lib/cli/src/commands/run.rs | 2 +- lib/cli/src/commands/run/wasi.rs | 6 +- lib/virtual-io/Cargo.toml | 2 +- lib/virtual-net/Cargo.toml | 10 +- lib/virtual-net/src/client.rs | 135 ++++++++++----- lib/virtual-net/src/host.rs | 4 +- lib/virtual-net/src/lib.rs | 4 +- lib/virtual-net/src/rx_tx.rs | 199 +++++++++++++++++++++- lib/virtual-net/src/server.rs | 20 +-- lib/wasi-web/Cargo.lock | 10 +- lib/wasi-web/src/runtime.rs | 4 +- lib/wasix/Cargo.toml | 8 +- lib/wasix/src/fs/inode_guard.rs | 4 +- lib/wasix/src/lib.rs | 6 +- lib/wasix/src/net/mod.rs | 2 +- lib/wasix/src/net/socket.rs | 14 +- lib/wasix/src/runtime/mod.rs | 6 +- lib/wasix/src/runtime/task_manager/mod.rs | 2 +- lib/wasix/src/state/env.rs | 2 +- lib/wasix/src/syscalls/mod.rs | 2 +- lib/wasix/src/syscalls/wasix/epoll_ctl.rs | 4 +- 23 files changed, 429 insertions(+), 163 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 937abaf8315..a60d09c7a07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,6 +275,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.2" @@ -2047,8 +2053,8 @@ dependencies = [ "hyper", "pin-project", "tokio", - "tokio-tungstenite", - "tungstenite", + "tokio-tungstenite 0.19.0", + "tungstenite 0.19.0", ] [[package]] @@ -3517,7 +3523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "async-compression", - "base64", + "base64 0.21.2", "bytes", "encoding_rs", "futures-core", @@ -3747,7 +3753,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64", + "base64 0.21.2", ] [[package]] @@ -4750,6 +4756,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio-tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.18.0", +] + [[package]] name = "tokio-tungstenite" version = "0.19.0" @@ -4759,7 +4777,7 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite", + "tungstenite 0.19.0", ] [[package]] @@ -5031,6 +5049,25 @@ dependencies = [ "termcolor", ] +[[package]] +name = "tungstenite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" +dependencies = [ + "base64 0.13.1", + "byteorder", + "bytes", + "http", + "httparse", + "log", + "rand", + "sha1 0.10.5", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "tungstenite" version = "0.19.0" @@ -5267,6 +5304,49 @@ dependencies = [ "webc", ] +[[package]] +name = "virtual-io" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "derivative", + "futures", + "mio", + "serde", + "socket2", + "thiserror", + "tracing", +] + +[[package]] +name = "virtual-net" +version = "0.3.0" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.21.2", + "bincode", + "bytes", + "derivative", + "futures-util", + "hyper", + "hyper-tungstenite", + "libc", + "mio", + "pin-project-lite", + "serde", + "socket2", + "thiserror", + "tokio", + "tokio-serde", + "tokio-tungstenite 0.18.0", + "tokio-util 0.6.10", + "tracing", + "tracing-test", + "virtual-io", +] + [[package]] name = "void" version = "1.0.2" @@ -5418,7 +5498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d816eaf89496a0eefecb79f43463b07205701165c17790fa6f0eab5ae72bc39" dependencies = [ "anyhow", - "base64", + "base64 0.21.2", "flate2", "indexmap 1.9.3", "json5", @@ -5844,6 +5924,7 @@ dependencies = [ "unix_mode", "url", "virtual-fs", + "virtual-net", "walkdir", "wapm-targz-to-pirita", "wasm-coredump-builder", @@ -5859,7 +5940,6 @@ dependencies = [ "wasmer-registry 5.2.0", "wasmer-toml", "wasmer-types", - "wasmer-virtual-net", "wasmer-vm", "wasmer-wasix", "wasmer-wasix-experimental-io-devices", @@ -6274,48 +6354,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "wasmer-virtual-io" -version = "0.1.0" -dependencies = [ - "async-trait", - "bytes", - "derivative", - "futures", - "mio", - "serde", - "socket2", - "thiserror", - "tracing", -] - -[[package]] -name = "wasmer-virtual-net" -version = "0.3.0" -dependencies = [ - "anyhow", - "async-trait", - "base64", - "bincode", - "bytes", - "derivative", - "futures-util", - "hyper", - "hyper-tungstenite", - "libc", - "mio", - "pin-project-lite", - "serde", - "socket2", - "thiserror", - "tokio", - "tokio-serde", - "tokio-util 0.6.10", - "tracing", - "tracing-test", - "wasmer-virtual-io", -] - [[package]] name = "wasmer-vm" version = "4.0.0" @@ -6393,6 +6431,8 @@ dependencies = [ "url", "urlencoding", "virtual-fs", + "virtual-io", + "virtual-net", "wai-bindgen-wasmer", "waker-fn", "wapm-targz-to-pirita", @@ -6401,8 +6441,6 @@ dependencies = [ "wasmer", "wasmer-emscripten", "wasmer-types", - "wasmer-virtual-io", - "wasmer-virtual-net", "wasmer-wasix-types", "wcgi", "wcgi-host", @@ -6720,7 +6758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02ba1b6e7ad252e691a86d727aa422fbc5ed95e9bca4ec8547869e9b5779f8f3" dependencies = [ "anyhow", - "base64", + "base64 0.21.2", "byteorder", "bytes", "indexmap 1.9.3", diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index cf37ad400f5..f13cb2ffd1a 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -60,7 +60,7 @@ wasmer-object = { version = "=4.0.0", path = "../object", optional = true } virtual-fs = { version = "0.6.0", path = "../virtual-fs", default-features = false, features = [ "host-fs", ] } -wasmer-virtual-net = { version = "0.3.0", path = "../virtual-net" } +virtual-net = { version = "0.3.0", path = "../virtual-net" } # Wasmer-owned dependencies. webc = { workspace = true } @@ -170,7 +170,7 @@ coredump = ["wasm-coredump-builder"] sys = ["compiler", "wasmer-vm"] jsc = ["backend", "wasmer/jsc", "wasmer/std"] wast = ["wasmer-wast"] -host-net = ["wasmer-virtual-net/host-net"] +host-net = ["virtual-net/host-net"] wat = ["wasmer/wat"] compiler = [ "backend", diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index e16273fc383..c3fc5d37d46 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -803,7 +803,7 @@ impl MonitoringRuntime { } impl wasmer_wasix::Runtime for MonitoringRuntime { - fn networking(&self) -> &wasmer_virtual_net::DynVirtualNetworking { + fn networking(&self) -> &virtual_net::DynVirtualNetworking { self.runtime.networking() } diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 23fe11ede95..3a887b584d1 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -256,11 +256,9 @@ impl Wasi { let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(handle))); if self.networking { - rt.set_networking_implementation(wasmer_virtual_net::host::LocalNetworking::default()); + rt.set_networking_implementation(virtual_net::host::LocalNetworking::default()); } else { - rt.set_networking_implementation( - wasmer_virtual_net::UnsupportedVirtualNetworking::default(), - ); + rt.set_networking_implementation(virtual_net::UnsupportedVirtualNetworking::default()); } if !self.no_tty { diff --git a/lib/virtual-io/Cargo.toml b/lib/virtual-io/Cargo.toml index 52b8c2f3a0a..48cdf577209 100644 --- a/lib/virtual-io/Cargo.toml +++ b/lib/virtual-io/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "wasmer-virtual-io" +name = "virtual-io" version = "0.1.0" description = "Wasmer Virtual IO Engine" authors.workspace = true diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index 87d0270990b..6700ba28e77 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "wasmer-virtual-net" +name = "virtual-net" version = "0.3.0" description = "Wasmer Virtual Networking" authors.workspace = true @@ -19,7 +19,7 @@ libc = { version = "0.2.139", optional = true } mio = { version = "0.8", optional = true } socket2 = { version = "0.4", optional = true } derivative = { version = "^2" } -wasmer-virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } +virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } base64 = "0.21" bincode = { version = "1.3" } serde = { version = "1.0", default-features = false, features = ["derive"] } @@ -30,19 +30,21 @@ tokio-serde = { version = "0.8", features = [ "bincode" ], optional = true } tokio-util = { version = "0.6", features = ["codec"], optional = true } hyper-tungstenite = { version = "0.10", optional = true } hyper = { version = "0.14", optional = true } +tokio-tungstenite = { version = "0.18", optional = true } [dev-dependencies] tokio = { version = "1", default_features = false, features = [ "macros" ] } tracing-test = { version = "0.2" } [features] -default = [ "host-net", "remote", "json", "messagepack", "cbor", "hyper" ] -host-net = [ "tokio", "libc", "tokio/io-util", "wasmer-virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] +default = [ "host-net", "remote", "json", "messagepack", "cbor", "hyper", "tokio-tungstenite" ] +host-net = [ "tokio", "libc", "tokio/io-util", "virtual-io/sys", "tokio/net", "tokio/rt", "socket2", "mio" ] remote = [ "tokio", "libc", "tokio/io-util", "tokio/sync", "tokio-serde", "tokio-util" ] json = [ "tokio-serde/json" ] messagepack = [ "tokio-serde/messagepack" ] cbor = [ "tokio-serde/cbor" ] hyper = [ "hyper-tungstenite", "dep:hyper" ] +tokio-tungstenite = [ "dep:tokio-tungstenite" ] [package.metadata.docs.rs] features = ["host-net", "remote"] diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index c144888fdda..8b19ad35ea2 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -38,8 +38,8 @@ use tokio_serde::SymmetricallyFramed; use tokio_util::codec::FramedRead; use tokio_util::codec::FramedWrite; use tokio_util::codec::LengthDelimitedCodec; -use wasmer_virtual_io::InlineWaker; -use wasmer_virtual_io::InterestType; +use virtual_io::InlineWaker; +use virtual_io::InterestType; use crate::meta; use crate::meta::FrameSerializationFormat; @@ -73,25 +73,14 @@ pub struct RemoteNetworkingClient { } impl RemoteNetworkingClient { - /// Creates a new interface on the remote location using - /// a unique interface ID and a pair of channels - pub fn new_from_mpsc( - tx: mpsc::Sender, - rx: mpsc::Receiver, + fn new( + tx: RemoteTx, + rx: RemoteRx, + rx_work: mpsc::UnboundedReceiver>, ) -> (Self, RemoteNetworkingClientDriver) { - let (tx_work, rx_work) = mpsc::unbounded_channel(); - let tx_wakers = RemoteTxWakers::default(); - let common = RemoteCommon { - tx: RemoteTx::Mpsc { - tx, - work: tx_work, - wakers: tx_wakers.clone(), - }, - rx: Mutex::new(RemoteRx::Mpsc { - rx, - wakers: tx_wakers, - }), + tx, + rx: Mutex::new(rx), request_seed: AtomicU64::new(1), requests: Default::default(), socket_seed: AtomicU64::new(1), @@ -113,6 +102,28 @@ impl RemoteNetworkingClient { (networking, driver) } + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + pub fn new_from_mpsc( + tx: mpsc::Sender, + rx: mpsc::Receiver, + ) -> (Self, RemoteNetworkingClientDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + let tx_wakers = RemoteTxWakers::default(); + + let tx = RemoteTx::Mpsc { + tx, + work: tx_work, + wakers: tx_wakers.clone(), + }; + let rx = RemoteRx::Mpsc { + rx, + wakers: tx_wakers, + }; + + Self::new(tx, rx, rx_work) + } + /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels /// @@ -172,32 +183,68 @@ impl RemoteNetworkingClient { let (tx_work, rx_work) = mpsc::unbounded_channel(); let tx_wakers = RemoteTxWakers::default(); - let common = RemoteCommon { - tx: RemoteTx::Stream { - tx: Arc::new(tokio::sync::Mutex::new(tx)), - work: tx_work, - wakers: tx_wakers.clone(), - }, - rx: Mutex::new(RemoteRx::Stream { rx }), - request_seed: AtomicU64::new(1), - requests: Default::default(), - socket_seed: AtomicU64::new(1), - recv_tx: Default::default(), - recv_with_addr_tx: Default::default(), - accept_tx: Default::default(), - handlers: Default::default(), - stall: Default::default(), + let tx = RemoteTx::Stream { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + wakers: tx_wakers.clone(), }; - let common = Arc::new(common); + let rx = RemoteRx::Stream { rx }; - let driver = RemoteNetworkingClientDriver { - more_work: rx_work, - tasks: Default::default(), - common: common.clone(), + Self::new(tx, rx, rx_work) + } + + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + #[cfg(feature = "hyper")] + pub fn new_from_hyper_ws_io( + tx: futures_util::stream::SplitSink< + hyper_tungstenite::WebSocketStream, + hyper_tungstenite::tungstenite::Message, + >, + rx: futures_util::stream::SplitStream< + hyper_tungstenite::WebSocketStream, + >, + format: FrameSerializationFormat, + ) -> (Self, RemoteNetworkingClientDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + + let tx = RemoteTx::HyperWebSocket { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + wakers: RemoteTxWakers::default(), + format, }; - let networking = Self { common }; + let rx = RemoteRx::HyperWebSocket { rx, format }; + Self::new(tx, rx, rx_work) + } - (networking, driver) + /// Creates a new interface on the remote location using + /// a unique interface ID and a pair of channels + #[cfg(feature = "tokio-tungstenite")] + pub fn new_from_tokio_ws_io( + tx: futures_util::stream::SplitSink< + tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, + tokio_tungstenite::tungstenite::Message, + >, + rx: futures_util::stream::SplitStream< + tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, + >, + format: FrameSerializationFormat, + ) -> (Self, RemoteNetworkingClientDriver) { + let (tx_work, rx_work) = mpsc::unbounded_channel(); + + let tx = RemoteTx::TokioWebSocket { + tx: Arc::new(tokio::sync::Mutex::new(tx)), + work: tx_work, + wakers: RemoteTxWakers::default(), + format, + }; + let rx = RemoteRx::TokioWebSocket { rx, format }; + Self::new(tx, rx, rx_work) } fn new_socket(&self, id: SocketId) -> RemoteSocket { @@ -471,7 +518,7 @@ struct RemoteCommon { recv_with_addr_tx: Mutex, SocketAddr)>>>, accept_tx: Mutex>>, #[derivative(Debug = "ignore")] - handlers: Mutex>>, + handlers: Mutex>>, // The stall guard will prevent reads while its held and there are background tasks running // (the idea behind this is to create back pressure so that the task list infinitely grow) @@ -906,7 +953,7 @@ impl VirtualSocket for RemoteSocket { fn set_handler( &mut self, - handler: Box, + handler: Box, ) -> Result<()> { self.common .handlers @@ -957,7 +1004,7 @@ impl VirtualTcpListener for RemoteSocket { fn set_handler( &mut self, - handler: Box, + handler: Box, ) -> Result<()> { VirtualSocket::set_handler(self, handler) } diff --git a/lib/virtual-net/src/host.rs b/lib/virtual-net/src/host.rs index 4812210f507..16df4f209e5 100644 --- a/lib/virtual-net/src/host.rs +++ b/lib/virtual-net/src/host.rs @@ -15,7 +15,7 @@ use std::time::Duration; use tokio::runtime::Handle; #[allow(unused_imports, dead_code)] use tracing::{debug, error, info, trace, warn}; -use wasmer_virtual_io::{InterestGuard, InterestHandler, Selector}; +use virtual_io::{InterestGuard, InterestHandler, Selector}; #[derive(Derivative)] #[derivative(Debug)] @@ -313,7 +313,7 @@ impl VirtualSocket for LocalTcpStream { if self.first_handler_writeable { self.first_handler_writeable = false; - handler.interest(wasmer_virtual_io::InterestType::Writable); + handler.interest(virtual_io::InterestType::Writable); } let guard = InterestGuard::new( diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index 0416b49e5d6..d0a975fd8a4 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -39,9 +39,9 @@ use tokio::io::AsyncWrite; pub use bytes::Bytes; pub use bytes::BytesMut; use serde::{Deserialize, Serialize}; -pub use wasmer_virtual_io::{handler_into_waker, InterestHandler}; +pub use virtual_io::{handler_into_waker, InterestHandler}; #[cfg(feature = "host-net")] -pub use wasmer_virtual_io::{InterestGuard, InterestHandlerWaker, InterestType}; +pub use virtual_io::{InterestGuard, InterestHandlerWaker, InterestType}; pub type Result = std::result::Result; diff --git a/lib/virtual-net/src/rx_tx.rs b/lib/virtual-net/src/rx_tx.rs index afa2bfb6312..5f160180742 100644 --- a/lib/virtual-net/src/rx_tx.rs +++ b/lib/virtual-net/src/rx_tx.rs @@ -10,12 +10,13 @@ use futures_util::{future::BoxFuture, Future, Sink, SinkExt, Stream}; use serde::Serialize; use tokio::{ io::AsyncWrite, + net::TcpStream, sync::{ mpsc::{self, error::TrySendError}, oneshot, }, }; -use wasmer_virtual_io::InlineWaker; +use virtual_io::InlineWaker; use crate::{io_err_into_net_error, NetworkError}; @@ -70,7 +71,7 @@ where work: mpsc::UnboundedSender>, wakers: RemoteTxWakers, }, - WebSocket { + HyperWebSocket { tx: Arc< tokio::sync::Mutex< futures_util::stream::SplitSink< @@ -83,6 +84,21 @@ where wakers: RemoteTxWakers, format: FrameSerializationFormat, }, + TokioWebSocket { + tx: Arc< + tokio::sync::Mutex< + futures_util::stream::SplitSink< + tokio_tungstenite::WebSocketStream< + tokio_tungstenite::MaybeTlsStream, + >, + tokio_tungstenite::tungstenite::Message, + >, + >, + >, + work: mpsc::UnboundedSender>, + wakers: RemoteTxWakers, + format: FrameSerializationFormat, + }, } impl RemoteTx where @@ -110,7 +126,7 @@ where .await .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) } - RemoteTx::WebSocket { tx, format, .. } => { + RemoteTx::HyperWebSocket { tx, format, .. } => { let data = match format { FrameSerializationFormat::Bincode => { bincode::serialize(&req).map_err(|err| { @@ -128,6 +144,24 @@ where .await .map_err(|_| NetworkError::ConnectionAborted) } + RemoteTx::TokioWebSocket { tx, format, .. } => { + let data = match format { + FrameSerializationFormat::Bincode => { + bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {err}"); + NetworkError::IOError + })? + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + return Err(NetworkError::IOError); + } + }; + let mut tx = tx.lock().await; + tx.send(tokio_tungstenite::tungstenite::Message::Binary(data)) + .await + .map_err(|_| NetworkError::ConnectionAborted) + } } } @@ -173,7 +207,7 @@ where })?; Poll::Ready(Ok(())) } - RemoteTx::WebSocket { + RemoteTx::HyperWebSocket { tx, format, work, @@ -223,6 +257,64 @@ where return Poll::Ready(Ok(())); } + // Otherwise we push it to the driver which will block all future send + // operations until it finishes + work.send(job).map_err(|err| { + tracing::error!("failed to send remaining bytes for request - {}", err); + NetworkError::ConnectionAborted + })?; + Poll::Ready(Ok(())) + } + RemoteTx::TokioWebSocket { + tx, + format, + work, + wakers, + .. + } => { + let mut tx_guard = match tx.clone().try_lock_owned() { + Ok(lock) => lock, + Err(_) => { + wakers.add(cx.waker()); + return Poll::Pending; + } + }; + match tx_guard.poll_ready_unpin(cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => { + tracing::warn!("failed to poll web socket for readiness - {err}"); + return Poll::Ready(Err(NetworkError::IOError)); + } + Poll::Pending => return Poll::Pending, + } + + let data = match format { + FrameSerializationFormat::Bincode => { + bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {err}"); + NetworkError::IOError + })? + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + return Poll::Ready(Err(NetworkError::IOError)); + } + }; + + let mut job = Box::pin(async move { + if let Err(err) = tx_guard + .send(tokio_tungstenite::tungstenite::Message::Binary(data)) + .await + { + tracing::error!("failed to send remaining bytes for request - {}", err); + } + }); + + // First we try to finish it synchronously + if job.as_mut().poll(cx).is_ready() { + return Poll::Ready(Ok(())); + } + // Otherwise we push it to the driver which will block all future send // operations until it finishes work.send(job).map_err(|err| { @@ -285,7 +377,7 @@ where })?; Ok(()) } - RemoteTx::WebSocket { + RemoteTx::HyperWebSocket { tx, format, work, .. } => { let data = match format { @@ -335,6 +427,64 @@ where return Ok(()); } + // Otherwise we push it to the driver which will block all future send + // operations until it finishes + work.send(job).map_err(|err| { + tracing::error!("failed to send remaining bytes for request - {}", err); + NetworkError::ConnectionAborted + })?; + Ok(()) + } + RemoteTx::TokioWebSocket { + tx, format, work, .. + } => { + let data = match format { + FrameSerializationFormat::Bincode => { + bincode::serialize(&req).map_err(|err| { + tracing::warn!("failed to serialize message - {err}"); + NetworkError::IOError + })? + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + return Err(NetworkError::IOError); + } + }; + + let mut tx_guard = match tx.clone().try_lock_owned() { + Ok(lock) => lock, + Err(_) => { + let tx = tx.clone(); + work.send(Box::pin(async move { + let mut tx_guard = tx.lock().await; + tx_guard + .send(tokio_tungstenite::tungstenite::Message::Binary(data)) + .await + .ok(); + })) + .ok(); + return Ok(()); + } + }; + + let inline_waker = InlineWaker::new(); + let waker = inline_waker.as_waker(); + let mut cx = Context::from_waker(&waker); + + let mut job = Box::pin(async move { + if let Err(err) = tx_guard + .send(tokio_tungstenite::tungstenite::Message::Binary(data)) + .await + { + tracing::error!("failed to send remaining bytes for request - {}", err); + } + }); + + // First we try to finish it synchronously + if job.as_mut().poll(&mut cx).is_ready() { + return Ok(()); + } + // Otherwise we push it to the driver which will block all future send // operations until it finishes work.send(job).map_err(|err| { @@ -358,12 +508,18 @@ where Stream { rx: Pin> + Send + 'static>>, }, - WebSocket { + HyperWebSocket { rx: futures_util::stream::SplitStream< hyper_tungstenite::WebSocketStream, >, format: FrameSerializationFormat, }, + TokioWebSocket { + rx: futures_util::stream::SplitStream< + tokio_tungstenite::WebSocketStream>, + >, + format: FrameSerializationFormat, + }, } impl RemoteRx where @@ -388,7 +544,7 @@ where Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, }, - RemoteRx::WebSocket { rx, format } => match Pin::new(rx).poll_next(cx) { + RemoteRx::HyperWebSocket { rx, format } => match Pin::new(rx).poll_next(cx) { Poll::Ready(Some(Ok(hyper_tungstenite::tungstenite::Message::Binary(msg)))) => { match format { FrameSerializationFormat::Bincode => { @@ -417,6 +573,35 @@ where Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, }, + RemoteRx::TokioWebSocket { rx, format } => match Pin::new(rx).poll_next(cx) { + Poll::Ready(Some(Ok(tokio_tungstenite::tungstenite::Message::Binary(msg)))) => { + match format { + FrameSerializationFormat::Bincode => { + return match bincode::deserialize(&msg) { + Ok(msg) => Poll::Ready(Some(msg)), + Err(err) => { + tracing::warn!("failed to deserialize message - {}", err); + continue; + } + } + } + format => { + tracing::warn!("format not currently supported - {format:?}"); + continue; + } + } + } + Poll::Ready(Some(Ok(msg))) => { + tracing::warn!("unsupported message from channel - {}", msg); + continue; + } + Poll::Ready(Some(Err(err))) => { + tracing::warn!("failed to read from channel - {}", err); + Poll::Ready(None) + } + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => Poll::Pending, + }, }; } } diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index 7290bbbb491..0b15009168e 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -9,8 +9,6 @@ use derivative::Derivative; use futures_util::stream::{FuturesOrdered, SplitSink, SplitStream}; use futures_util::{future::BoxFuture, StreamExt}; use futures_util::{Sink, Stream}; -#[cfg(feature = "hyper")] -use hyper_tungstenite::WebSocketStream; use std::collections::HashSet; use std::mem::MaybeUninit; use std::net::IpAddr; @@ -37,7 +35,7 @@ use tokio_serde::formats::SymmetricalJson; use tokio_serde::formats::SymmetricalMessagePack; use tokio_serde::SymmetricallyFramed; use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec}; -use wasmer_virtual_io::InterestHandler; +use virtual_io::InterestHandler; type BackgroundTask = Option>; @@ -165,24 +163,24 @@ impl RemoteNetworkingServer { /// Creates a new interface on the remote location using /// a unique interface ID and a pair of channels #[cfg(feature = "hyper")] - pub fn new_from_ws_io( + pub fn new_from_hyper_ws_io( tx: SplitSink< - WebSocketStream, + hyper_tungstenite::WebSocketStream, hyper_tungstenite::tungstenite::Message, >, - rx: SplitStream>, + rx: SplitStream>, format: FrameSerializationFormat, inner: Arc, ) -> (Self, RemoteNetworkingServerDriver) { let (tx_work, rx_work) = mpsc::unbounded_channel(); - let tx = RemoteTx::WebSocket { + let tx = RemoteTx::HyperWebSocket { tx: Arc::new(tokio::sync::Mutex::new(tx)), work: tx_work, wakers: RemoteTxWakers::default(), format, }; - let rx = RemoteRx::WebSocket { rx, format }; + let rx = RemoteRx::HyperWebSocket { rx, format }; Self::new(tx, rx, rx_work, inner) } } @@ -624,7 +622,7 @@ impl RemoteNetworkingServerDriver { // Now we attach the handler to the main listening socket let mut handler = Box::new(self.common.handler.clone().for_socket(socket_id)); - handler.interest(wasmer_virtual_io::InterestType::Readable); + handler.interest(virtual_io::InterestType::Readable); self.process_inner_noop( move |socket| match socket { RemoteAdapterSocket::TcpListener { @@ -1524,7 +1522,7 @@ impl RemoteAdapterHandler { } } impl InterestHandler for RemoteAdapterHandler { - fn interest(&mut self, interest: wasmer_virtual_io::InterestType) { + fn interest(&mut self, interest: virtual_io::InterestType) { let mut guard = self.state.lock().unwrap(); guard.driver_wakers.drain(..).for_each(|w| w.wake()); let socket_id = match self.socket_id.clone() { @@ -1532,7 +1530,7 @@ impl InterestHandler for RemoteAdapterHandler { None => return, }; match interest { - wasmer_virtual_io::InterestType::Readable => { + virtual_io::InterestType::Readable => { guard.readable.insert(socket_id); } _ => {} diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 6a71e54a5a9..2e50d9d00af 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -2047,7 +2047,7 @@ dependencies = [ ] [[package]] -name = "wasmer-virtual-io" +name = "virtual-io" version = "0.1.0" dependencies = [ "async-trait", @@ -2060,7 +2060,7 @@ dependencies = [ ] [[package]] -name = "wasmer-virtual-net" +name = "virtual-net" version = "0.3.0" dependencies = [ "async-trait", @@ -2073,7 +2073,7 @@ dependencies = [ "serde", "thiserror", "tracing", - "wasmer-virtual-io", + "virtual-io", ] [[package]] @@ -2462,8 +2462,8 @@ dependencies = [ "url", "urlencoding", "virtual-fs", - "wasmer-virtual-io", - "wasmer-virtual-net", + "virtual-io", + "virtual-net", "wai-bindgen-wasmer", "waker-fn", "wasm-bindgen", diff --git a/lib/wasi-web/src/runtime.rs b/lib/wasi-web/src/runtime.rs index 7dce9200f1f..586633dde7f 100644 --- a/lib/wasi-web/src/runtime.rs +++ b/lib/wasi-web/src/runtime.rs @@ -57,7 +57,7 @@ pub(crate) struct WebRuntime { package_loader: Arc, source: Arc, module_cache: Arc, - net: wasmer_wasix::wasmer_virtual_net::DynVirtualNetworking, + net: wasmer_wasix::virtual_net::DynVirtualNetworking, tasks: Arc, } @@ -426,7 +426,7 @@ impl VirtualFile for TermLog { } impl wasmer_wasix::Runtime for WebRuntime { - fn networking(&self) -> &wasmer_wasix::wasmer_virtual_net::DynVirtualNetworking { + fn networking(&self) -> &wasmer_wasix::virtual_net::DynVirtualNetworking { &self.net } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index fcc1d35cd86..bcd9544bf28 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -20,9 +20,9 @@ getrandom = "0.2" wasmer-wasix-types = { path = "../wasi-types", version = "0.9.0", features = [ "enable-serde" ] } wasmer-types = { path = "../types", version = "=4.0.0", default-features = false } wasmer = { path = "../api", version = "=4.0.0", default-features = false, features = ["wat", "js-serializable-module"] } -wasmer-virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } +virtual-io = { path = "../virtual-io", version = "0.1.0", default-features = false } virtual-fs = { path = "../virtual-fs", version = "0.6.0", default-features = false, features = ["webc-fs"] } -wasmer-virtual-net = { path = "../virtual-net", version = "0.3.0", default-features = false } +virtual-net = { path = "../virtual-net", version = "0.3.0", default-features = false } wasmer-emscripten = { path = "../emscripten", version = "=4.0.0", optional = true } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"] } @@ -128,11 +128,11 @@ js = ["virtual-fs/no-time", "getrandom/js", "chrono"] js-default = ["js"] test-js = ["js", "wasmer/wat"] -host-vnet = ["wasmer-virtual-net/host-net"] +host-vnet = ["virtual-net/host-net"] host-threads = [] host-reqwest = ["reqwest"] host-fs = ["virtual-fs/host-fs"] -remote-vnet = ["wasmer-virtual-net/remote"] +remote-vnet = ["virtual-net/remote"] logging = ["tracing/log"] disable-all-logging = ["tracing/release_max_level_off", "tracing/max_level_off"] diff --git a/lib/wasix/src/fs/inode_guard.rs b/lib/wasix/src/fs/inode_guard.rs index 1f4b3498035..e8e7cfd1243 100644 --- a/lib/wasix/src/fs/inode_guard.rs +++ b/lib/wasix/src/fs/inode_guard.rs @@ -11,8 +11,8 @@ use std::{ use futures::future::BoxFuture; use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite}; use virtual_fs::{FsError, Pipe as VirtualPipe, VirtualFile}; -use wasmer_virtual_io::{InterestType, StatefulHandler}; -use wasmer_virtual_net::net_error_into_io_err; +use virtual_io::{InterestType, StatefulHandler}; +use virtual_net::net_error_into_io_err; use wasmer_wasix_types::{ types::Eventtype, wasi::{self, EpollType}, diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index beef7f1869a..f3db61da7b3 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -70,11 +70,11 @@ use wasmer::{ pub use virtual_fs; pub use virtual_fs::{DuplexPipe, FsError, Pipe, VirtualFile, WasiBidirectionalSharedPipePair}; -pub use wasmer_virtual_net; -pub use wasmer_virtual_net::{UnsupportedVirtualNetworking, VirtualNetworking}; +pub use virtual_net; +pub use virtual_net::{UnsupportedVirtualNetworking, VirtualNetworking}; #[cfg(feature = "host-vnet")] -pub use wasmer_virtual_net::{ +pub use virtual_net::{ host::{LocalNetworking, LocalTcpListener, LocalTcpStream, LocalUdpSocket}, io_err_into_net_error, }; diff --git a/lib/wasix/src/net/mod.rs b/lib/wasix/src/net/mod.rs index 7845d22f2de..a59d71508cf 100644 --- a/lib/wasix/src/net/mod.rs +++ b/lib/wasix/src/net/mod.rs @@ -4,9 +4,9 @@ use std::{ time::Duration, }; +use virtual_net::{IpCidr, IpRoute, NetworkError}; use wasmer::{MemoryView, WasmPtr}; use wasmer_types::MemorySize; -use wasmer_virtual_net::{IpCidr, IpRoute, NetworkError}; use wasmer_wasix_types::{ types::{ OptionTag, OptionTimestamp, Route, __wasi_addr_ip4_t, __wasi_addr_ip6_t, diff --git a/lib/wasix/src/net/socket.rs b/lib/wasix/src/net/socket.rs index 0dd927f6dc0..fc6872d1bf5 100644 --- a/lib/wasix/src/net/socket.rs +++ b/lib/wasix/src/net/socket.rs @@ -10,15 +10,15 @@ use std::{ #[cfg(feature = "enable-serde")] use serde_derive::{Deserialize, Serialize}; -use wasmer_types::MemorySize; -use wasmer_virtual_io::{ +use virtual_io::{ FilteredHandler, FilteredHandlerSubscriptions, InterestHandler, InterestType, StatefulHandlerState, }; -use wasmer_virtual_net::{ +use virtual_net::{ NetworkError, VirtualIcmpSocket, VirtualNetworking, VirtualRawSocket, VirtualTcpListener, VirtualTcpSocket, VirtualUdpSocket, }; +use wasmer_types::MemorySize; use wasmer_wasix_types::wasi::{Addressfamily, Errno, Rights, SockProto, Sockoption, Socktype}; use crate::{net::net_error_into_wasi_err, VirtualTaskManager}; @@ -1259,16 +1259,14 @@ impl InodeSocketProtected { pub fn set_handler( &mut self, handler: Box, - ) -> wasmer_virtual_net::Result<()> { + ) -> virtual_net::Result<()> { match &mut self.kind { InodeSocketKind::TcpListener { socket, .. } => socket.set_handler(handler), InodeSocketKind::TcpStream { socket, .. } => socket.set_handler(handler), InodeSocketKind::UdpSocket { socket, .. } => socket.set_handler(handler), InodeSocketKind::Raw(socket) => socket.set_handler(handler), InodeSocketKind::Icmp(socket) => socket.set_handler(handler), - InodeSocketKind::PreSocket { .. } => { - Err(wasmer_virtual_net::NetworkError::NotConnected) - } + InodeSocketKind::PreSocket { .. } => Err(virtual_net::NetworkError::NotConnected), } } @@ -1276,7 +1274,7 @@ impl InodeSocketProtected { &mut self, handler: Box, interest: InterestType, - ) -> wasmer_virtual_net::Result<()> { + ) -> virtual_net::Result<()> { if self.aggregate_handler.is_none() { let upper = FilteredHandler::new(); let subs = upper.subscriptions().clone(); diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 4f8268fad7b..3cb1b552ffc 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -16,8 +16,8 @@ use std::{ use derivative::Derivative; use futures::future::BoxFuture; +use virtual_net::{DynVirtualNetworking, VirtualNetworking}; use wasmer::Module; -use wasmer_virtual_net::{DynVirtualNetworking, VirtualNetworking}; use crate::{ http::{DynHttpClient, HttpClient}, @@ -187,9 +187,9 @@ impl PluggableRuntime { // TODO: the cfg flags below should instead be handled by separate implementations. cfg_if::cfg_if! { if #[cfg(feature = "host-vnet")] { - let networking = Arc::new(wasmer_virtual_net::host::LocalNetworking::default()); + let networking = Arc::new(virtual_net::host::LocalNetworking::default()); } else { - let networking = Arc::new(wasmer_virtual_net::UnsupportedVirtualNetworking::default()); + let networking = Arc::new(virtual_net::UnsupportedVirtualNetworking::default()); } } let http_client = diff --git a/lib/wasix/src/runtime/task_manager/mod.rs b/lib/wasix/src/runtime/task_manager/mod.rs index 40fd1d5dbad..c69c43dab81 100644 --- a/lib/wasix/src/runtime/task_manager/mod.rs +++ b/lib/wasix/src/runtime/task_manager/mod.rs @@ -16,7 +16,7 @@ use crate::os::task::thread::WasiThreadError; use crate::syscalls::AsyncifyFuture; use crate::{capture_snapshot, InstanceSnapshot, WasiEnv, WasiFunctionEnv, WasiThread}; -pub use wasmer_virtual_io::waker::*; +pub use virtual_io::waker::*; #[derive(Debug)] pub enum SpawnMemoryType<'a> { diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index c1a4314c924..b8a895302fb 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -11,11 +11,11 @@ use futures::future::BoxFuture; use rand::Rng; use tracing::{trace, warn}; use virtual_fs::{FileSystem, FsError, StaticFile, VirtualFile}; +use virtual_net::DynVirtualNetworking; use wasmer::{ AsStoreMut, AsStoreRef, FunctionEnvMut, Global, Instance, Memory, MemoryType, MemoryView, Module, TypedFunction, }; -use wasmer_virtual_net::DynVirtualNetworking; use wasmer_wasix_types::{ types::Signal, wasi::{Errno, ExitCode, Snapshot0Clockid}, diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 62659aa259d..ef24583bafd 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -69,12 +69,12 @@ pub use wasm::*; pub(crate) use virtual_fs::{ AsyncSeekExt, AsyncWriteExt, DuplexPipe, FileSystem, FsError, VirtualFile, }; +pub(crate) use virtual_net::StreamSecurity; pub(crate) use wasmer::{ AsStoreMut, AsStoreRef, Extern, Function, FunctionEnv, FunctionEnvMut, Global, Instance, Memory, Memory32, Memory64, MemoryAccessError, MemoryError, MemorySize, MemoryView, Module, OnCalledAction, Pages, RuntimeError, Store, TypedFunction, Value, WasmPtr, WasmSlice, }; -pub(crate) use wasmer_virtual_net::StreamSecurity; pub(crate) use wasmer_wasix_types::{asyncify::__wasi_asyncify_t, wasi::EventUnion}; #[cfg(any(target_os = "windows"))] pub use windows::*; diff --git a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs index 59169b2210e..093697fa344 100644 --- a/lib/wasix/src/syscalls/wasix/epoll_ctl.rs +++ b/lib/wasix/src/syscalls/wasix/epoll_ctl.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use tokio::sync::{mpsc::UnboundedSender, watch}; -use wasmer_virtual_io::{InterestHandler, InterestType}; -use wasmer_virtual_net::net_error_into_io_err; +use virtual_io::{InterestHandler, InterestType}; +use virtual_net::net_error_into_io_err; use wasmer_wasix_types::wasi::{ EpollCtl, EpollEvent, EpollType, SubscriptionClock, SubscriptionUnion, Userdata, }; From 57d7ab91804618c4e986874ceedd5f676f7dc88d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 24 Jul 2023 15:26:02 +1000 Subject: [PATCH 23/52] Converted some sync operations to async operations --- Cargo.lock | 53 ++------ lib/virtual-net/Cargo.toml | 2 +- lib/virtual-net/src/client.rs | 26 ++-- lib/virtual-net/src/lib.rs | 21 ++-- lib/virtual-net/src/server.rs | 113 +++++++++--------- lib/wasix/src/syscalls/wasix/port_addr_add.rs | 1 + .../src/syscalls/wasix/port_addr_clear.rs | 2 +- .../src/syscalls/wasix/port_addr_list.rs | 2 +- .../src/syscalls/wasix/port_addr_remove.rs | 2 +- .../src/syscalls/wasix/port_gateway_set.rs | 2 +- lib/wasix/src/syscalls/wasix/port_mac.rs | 2 +- .../src/syscalls/wasix/port_route_add.rs | 1 + .../src/syscalls/wasix/port_route_clear.rs | 2 +- .../src/syscalls/wasix/port_route_list.rs | 2 +- .../src/syscalls/wasix/port_route_remove.rs | 2 +- 15 files changed, 100 insertions(+), 133 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be4a9f06656..bf056078726 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,12 +275,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.2" @@ -2053,8 +2047,8 @@ dependencies = [ "hyper", "pin-project", "tokio", - "tokio-tungstenite 0.19.0", - "tungstenite 0.19.0", + "tokio-tungstenite", + "tungstenite", ] [[package]] @@ -3472,7 +3466,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "async-compression", - "base64 0.21.2", + "base64", "bytes", "encoding_rs", "futures-core", @@ -3702,7 +3696,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64", ] [[package]] @@ -4705,18 +4699,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "tokio-tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.18.0", -] - [[package]] name = "tokio-tungstenite" version = "0.19.0" @@ -4726,7 +4708,7 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.19.0", + "tungstenite", ] [[package]] @@ -4998,25 +4980,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "tungstenite" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788" -dependencies = [ - "base64 0.13.1", - "byteorder", - "bytes", - "http", - "httparse", - "log", - "rand", - "sha1 0.10.5", - "thiserror", - "url", - "utf-8", -] - [[package]] name = "tungstenite" version = "0.19.0" @@ -5259,7 +5222,7 @@ version = "0.3.0" dependencies = [ "anyhow", "async-trait", - "base64 0.21.2", + "base64", "bincode", "bytes", "derivative", @@ -5274,7 +5237,7 @@ dependencies = [ "thiserror", "tokio", "tokio-serde", - "tokio-tungstenite 0.18.0", + "tokio-tungstenite", "tokio-util 0.6.10", "tracing", "tracing-test", @@ -6664,7 +6627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d8b985cecc5a364f746c7fcd6e5396986360a58550072f2f9147a07532f525c" dependencies = [ "anyhow", - "base64 0.21.2", + "base64", "byteorder", "bytes", "flate2", diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index 6700ba28e77..8f45a6ca706 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -30,7 +30,7 @@ tokio-serde = { version = "0.8", features = [ "bincode" ], optional = true } tokio-util = { version = "0.6", features = ["codec"], optional = true } hyper-tungstenite = { version = "0.10", optional = true } hyper = { version = "0.14", optional = true } -tokio-tungstenite = { version = "0.18", optional = true } +tokio-tungstenite = { version = "0.19", optional = true } [dev-dependencies] tokio = { version = "1", default_features = false, features = [ "macros" ] } diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index 8b19ad35ea2..af5c0634cac 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -601,22 +601,22 @@ impl VirtualNetworking for RemoteNetworkingClient { } } - fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<()> { + async fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<()> { self.common .io_iface_fire_and_forget(RequestType::IpAdd { ip, prefix }) } - fn ip_remove(&self, ip: IpAddr) -> Result<()> { + async fn ip_remove(&self, ip: IpAddr) -> Result<()> { self.common .io_iface_fire_and_forget(RequestType::IpRemove(ip)) } - fn ip_clear(&self) -> Result<()> { + async fn ip_clear(&self) -> Result<()> { self.common.io_iface_fire_and_forget(RequestType::IpClear) } - fn ip_list(&self) -> Result> { - match InlineWaker::block_on(self.common.io_iface(RequestType::Unbridge)) { + async fn ip_list(&self) -> Result> { + match self.common.io_iface(RequestType::GetIpList).await { ResponseType::Err(err) => Err(err), ResponseType::CidrList(routes) => Ok(routes), res => { @@ -626,8 +626,8 @@ impl VirtualNetworking for RemoteNetworkingClient { } } - fn mac(&self) -> Result<[u8; 6]> { - match InlineWaker::block_on(self.common.io_iface(RequestType::GetMac)) { + async fn mac(&self) -> Result<[u8; 6]> { + match self.common.io_iface(RequestType::GetMac).await { ResponseType::Err(err) => Err(err), ResponseType::Mac(mac) => Ok(mac), res => { @@ -637,12 +637,12 @@ impl VirtualNetworking for RemoteNetworkingClient { } } - fn gateway_set(&self, ip: IpAddr) -> Result<()> { + async fn gateway_set(&self, ip: IpAddr) -> Result<()> { self.common .io_iface_fire_and_forget(RequestType::GatewaySet(ip)) } - fn route_add( + async fn route_add( &self, cidr: IpCidr, via_router: IpAddr, @@ -657,18 +657,18 @@ impl VirtualNetworking for RemoteNetworkingClient { }) } - fn route_remove(&self, cidr: IpAddr) -> Result<()> { + async fn route_remove(&self, cidr: IpAddr) -> Result<()> { self.common .io_iface_fire_and_forget(RequestType::RouteRemove(cidr)) } - fn route_clear(&self) -> Result<()> { + async fn route_clear(&self) -> Result<()> { self.common .io_iface_fire_and_forget(RequestType::RouteClear) } - fn route_list(&self) -> Result> { - match InlineWaker::block_on(self.common.io_iface(RequestType::GetRouteList)) { + async fn route_list(&self) -> Result> { + match self.common.io_iface(RequestType::GetRouteList).await { ResponseType::Err(err) => Err(err), ResponseType::RouteList(routes) => Ok(routes), res => { diff --git a/lib/virtual-net/src/lib.rs b/lib/virtual-net/src/lib.rs index d0a975fd8a4..4e2527bc769 100644 --- a/lib/virtual-net/src/lib.rs +++ b/lib/virtual-net/src/lib.rs @@ -93,37 +93,37 @@ pub trait VirtualNetworking: fmt::Debug + Send + Sync + 'static { } /// Adds a static IP address to the interface with a netmask prefix - fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<()> { + async fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<()> { Err(NetworkError::Unsupported) } /// Removes a static (or dynamic) IP address from the interface - fn ip_remove(&self, ip: IpAddr) -> Result<()> { + async fn ip_remove(&self, ip: IpAddr) -> Result<()> { Err(NetworkError::Unsupported) } /// Clears all the assigned IP addresses for this interface - fn ip_clear(&self) -> Result<()> { + async fn ip_clear(&self) -> Result<()> { Err(NetworkError::Unsupported) } /// Lists all the IP addresses currently assigned to this interface - fn ip_list(&self) -> Result> { + async fn ip_list(&self) -> Result> { Err(NetworkError::Unsupported) } /// Returns the hardware MAC address for this interface - fn mac(&self) -> Result<[u8; 6]> { + async fn mac(&self) -> Result<[u8; 6]> { Err(NetworkError::Unsupported) } /// Adds a default gateway to the routing table - fn gateway_set(&self, ip: IpAddr) -> Result<()> { + async fn gateway_set(&self, ip: IpAddr) -> Result<()> { Err(NetworkError::Unsupported) } /// Adds a specific route to the routing table - fn route_add( + async fn route_add( &self, cidr: IpCidr, via_router: IpAddr, @@ -134,17 +134,17 @@ pub trait VirtualNetworking: fmt::Debug + Send + Sync + 'static { } /// Removes a routing rule from the routing table - fn route_remove(&self, cidr: IpAddr) -> Result<()> { + async fn route_remove(&self, cidr: IpAddr) -> Result<()> { Err(NetworkError::Unsupported) } /// Clears the routing table for this interface - fn route_clear(&self) -> Result<()> { + async fn route_clear(&self) -> Result<()> { Err(NetworkError::Unsupported) } /// Lists all the routes defined in the routing table for this interface - fn route_list(&self) -> Result> { + async fn route_list(&self) -> Result> { Err(NetworkError::Unsupported) } @@ -522,6 +522,7 @@ pub trait VirtualIcmpSocket: { } +#[async_trait::async_trait] pub trait VirtualRawSocket: VirtualSocket + fmt::Debug + Send + Sync + 'static { /// Sends out a datagram or stream of bytes on this socket fn try_send(&mut self, data: &[u8]) -> Result; diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index 0b15009168e..b1b2fcefaee 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -204,31 +204,31 @@ impl VirtualNetworking for RemoteNetworkingServer { self.inner.dhcp_acquire().await } - fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<(), NetworkError> { - self.inner.ip_add(ip, prefix) + async fn ip_add(&self, ip: IpAddr, prefix: u8) -> Result<(), NetworkError> { + self.inner.ip_add(ip, prefix).await } - fn ip_remove(&self, ip: IpAddr) -> Result<(), NetworkError> { - self.inner.ip_remove(ip) + async fn ip_remove(&self, ip: IpAddr) -> Result<(), NetworkError> { + self.inner.ip_remove(ip).await } - fn ip_clear(&self) -> Result<(), NetworkError> { - self.inner.ip_clear() + async fn ip_clear(&self) -> Result<(), NetworkError> { + self.inner.ip_clear().await } - fn ip_list(&self) -> Result, NetworkError> { - self.inner.ip_list() + async fn ip_list(&self) -> Result, NetworkError> { + self.inner.ip_list().await } - fn mac(&self) -> Result<[u8; 6], NetworkError> { - self.inner.mac() + async fn mac(&self) -> Result<[u8; 6], NetworkError> { + self.inner.mac().await } - fn gateway_set(&self, ip: IpAddr) -> Result<(), NetworkError> { - self.inner.gateway_set(ip) + async fn gateway_set(&self, ip: IpAddr) -> Result<(), NetworkError> { + self.inner.gateway_set(ip).await } - fn route_add( + async fn route_add( &self, cidr: IpCidr, via_router: IpAddr, @@ -237,18 +237,19 @@ impl VirtualNetworking for RemoteNetworkingServer { ) -> Result<(), NetworkError> { self.inner .route_add(cidr, via_router, preferred_until, expires_at) + .await } - fn route_remove(&self, cidr: IpAddr) -> Result<(), NetworkError> { - self.inner.route_remove(cidr) + async fn route_remove(&self, cidr: IpAddr) -> Result<(), NetworkError> { + self.inner.route_remove(cidr).await } - fn route_clear(&self) -> Result<(), NetworkError> { - self.inner.route_clear() + async fn route_clear(&self) -> Result<(), NetworkError> { + self.inner.route_clear().await } - fn route_list(&self) -> Result, NetworkError> { - self.inner.route_list() + async fn route_list(&self) -> Result, NetworkError> { + self.inner.route_list().await } async fn bind_raw(&self) -> Result, NetworkError> { @@ -670,47 +671,47 @@ impl RemoteNetworkingServerDriver { ), RequestType::IpAdd { ip, prefix } => self.process_async_noop( move |inner: Arc| async move { - inner.ip_add(ip, prefix) + inner.ip_add(ip, prefix).await }, req_id, ), RequestType::IpRemove(ip) => self.process_async_noop( move |inner: Arc| async move { - inner.ip_remove(ip) + inner.ip_remove(ip).await }, req_id, ), - RequestType::IpClear => { - self.process_async_noop( - move |inner: Arc| async move { - inner.ip_clear() - }, - req_id, - ) - } - RequestType::GetIpList => { + RequestType::IpClear => self.process_async_noop( + move |inner: Arc| async move { + inner.ip_clear().await + }, + req_id, + ), + RequestType::GetIpList => self.process_async_inner( + move |inner: Arc| async move { + inner.ip_list().await + }, + |ret| match ret { + Ok(cidr) => ResponseType::CidrList(cidr), + Err(err) => ResponseType::Err(err), + }, + req_id, + ), + RequestType::GetMac => { self.process_async_inner( move |inner: Arc| async move { - inner.ip_list() + inner.mac().await }, |ret| match ret { - Ok(cidr) => ResponseType::CidrList(cidr), + Ok(mac) => ResponseType::Mac(mac), Err(err) => ResponseType::Err(err), }, req_id, ) } - RequestType::GetMac => self.process_async_inner( - move |inner: Arc| async move { inner.mac() }, - |ret| match ret { - Ok(mac) => ResponseType::Mac(mac), - Err(err) => ResponseType::Err(err), - }, - req_id, - ), RequestType::GatewaySet(ip) => self.process_async_noop( move |inner: Arc| async move { - inner.gateway_set(ip) + inner.gateway_set(ip).await }, req_id, ), @@ -721,34 +722,34 @@ impl RemoteNetworkingServerDriver { expires_at, } => self.process_async_noop( move |inner: Arc| async move { - inner.route_add(cidr, via_router, preferred_until, expires_at) + inner + .route_add(cidr, via_router, preferred_until, expires_at) + .await }, req_id, ), RequestType::RouteRemove(ip) => self.process_async_noop( move |inner: Arc| async move { - inner.route_remove(ip) + inner.route_remove(ip).await }, req_id, ), RequestType::RouteClear => self.process_async_noop( move |inner: Arc| async move { - inner.route_clear() + inner.route_clear().await + }, + req_id, + ), + RequestType::GetRouteList => self.process_async_inner( + move |inner: Arc| async move { + inner.route_list().await + }, + |ret| match ret { + Ok(routes) => ResponseType::RouteList(routes), + Err(err) => ResponseType::Err(err), }, req_id, ), - RequestType::GetRouteList => { - self.process_async_inner( - move |inner: Arc| async move { - inner.route_list() - }, - |ret| match ret { - Ok(routes) => ResponseType::RouteList(routes), - Err(err) => ResponseType::Err(err), - }, - req_id, - ) - } RequestType::BindRaw(socket_id) => self.process_async_new_socket( move |inner: Arc| async move { Ok(RemoteAdapterSocket::RawSocket(inner.bind_raw().await?)) diff --git a/lib/wasix/src/syscalls/wasix/port_addr_add.rs b/lib/wasix/src/syscalls/wasix/port_addr_add.rs index 1bfa433f88d..ce4633957d5 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_add.rs @@ -21,6 +21,7 @@ pub fn port_addr_add( let net = env.net().clone(); wasi_try_ok!(__asyncify(&mut ctx, None, async { net.ip_add(cidr.ip, cidr.prefix) + .await .map_err(net_error_into_wasi_err) })?); Ok(Errno::Success) diff --git a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs index 3129d62c09d..73afbe7577e 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_clear.rs @@ -8,7 +8,7 @@ pub fn port_addr_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result( let net = env.net().clone(); let addrs = wasi_try_ok!(__asyncify(&mut ctx, None, async { - net.ip_list().map_err(net_error_into_wasi_err) + net.ip_list().await.map_err(net_error_into_wasi_err) })?); let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; diff --git a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs index daab635c614..ef48d647953 100644 --- a/lib/wasix/src/syscalls/wasix/port_addr_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_addr_remove.rs @@ -20,7 +20,7 @@ pub fn port_addr_remove( let net = env.net().clone(); wasi_try_ok!(__asyncify(&mut ctx, None, async { - net.ip_remove(ip).map_err(net_error_into_wasi_err) + net.ip_remove(ip).await.map_err(net_error_into_wasi_err) })?); Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs index ecb9a8c8a28..5a4cfdc4198 100644 --- a/lib/wasix/src/syscalls/wasix/port_gateway_set.rs +++ b/lib/wasix/src/syscalls/wasix/port_gateway_set.rs @@ -20,7 +20,7 @@ pub fn port_gateway_set( let net = env.net().clone(); wasi_try_ok!(__asyncify(&mut ctx, None, async { - net.gateway_set(ip).map_err(net_error_into_wasi_err) + net.gateway_set(ip).await.map_err(net_error_into_wasi_err) })?); Ok(Errno::Success) } diff --git a/lib/wasix/src/syscalls/wasix/port_mac.rs b/lib/wasix/src/syscalls/wasix/port_mac.rs index 2bd39294e1b..15de5612b19 100644 --- a/lib/wasix/src/syscalls/wasix/port_mac.rs +++ b/lib/wasix/src/syscalls/wasix/port_mac.rs @@ -13,7 +13,7 @@ pub fn port_mac( let net = env.net().clone(); let mac = wasi_try_ok!(__asyncify(&mut ctx, None, async { - net.mac().map_err(net_error_into_wasi_err) + net.mac().await.map_err(net_error_into_wasi_err) })?); let env = ctx.data(); let memory = unsafe { env.memory_view(&ctx) }; diff --git a/lib/wasix/src/syscalls/wasix/port_route_add.rs b/lib/wasix/src/syscalls/wasix/port_route_add.rs index db85e487298..9e3ee6e4c88 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_add.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_add.rs @@ -36,6 +36,7 @@ pub fn port_route_add( let net = env.net().clone(); wasi_try_ok!(__asyncify(&mut ctx, None, async { net.route_add(cidr, via_router, preferred_until, expires_at) + .await .map_err(net_error_into_wasi_err) })?); Ok(Errno::Success) diff --git a/lib/wasix/src/syscalls/wasix/port_route_clear.rs b/lib/wasix/src/syscalls/wasix/port_route_clear.rs index f006b07b99a..cd853f929de 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_clear.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_clear.rs @@ -8,7 +8,7 @@ pub fn port_route_clear(mut ctx: FunctionEnvMut<'_, WasiEnv>) -> Result( let net = env.net().clone(); let routes = wasi_try_ok!(__asyncify(&mut ctx, None, async { - net.route_list().map_err(net_error_into_wasi_err) + net.route_list().await.map_err(net_error_into_wasi_err) })?); Span::current().record("nroutes", routes.len()); diff --git a/lib/wasix/src/syscalls/wasix/port_route_remove.rs b/lib/wasix/src/syscalls/wasix/port_route_remove.rs index 52068c032e6..3ab127e9e4d 100644 --- a/lib/wasix/src/syscalls/wasix/port_route_remove.rs +++ b/lib/wasix/src/syscalls/wasix/port_route_remove.rs @@ -16,7 +16,7 @@ pub fn port_route_remove( let net = env.net().clone(); wasi_try_ok!(__asyncify(&mut ctx, None, async { - net.route_remove(ip).map_err(net_error_into_wasi_err) + net.route_remove(ip).await.map_err(net_error_into_wasi_err) })?); Ok(Errno::Success) From cab8716e32687f0604c359283442762650ca5fb8 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 25 Jul 2023 04:06:50 +1000 Subject: [PATCH 24/52] Fixed multiple bugs with the remote networking packet send and recv paths --- lib/virtual-net/src/client.rs | 2 +- lib/virtual-net/src/server.rs | 104 ++++++++++++++++++++++------------ 2 files changed, 68 insertions(+), 38 deletions(-) diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index af5c0634cac..1b04ad86778 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -1074,7 +1074,7 @@ impl VirtualRawSocket for RemoteSocket { if self.rx_buffer.len() > 0 { let amt = self.rx_buffer.len().min(buf.len()); let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; - buf.copy_from_slice(&self.rx_buffer[..amt]); + buf[..amt].copy_from_slice(&self.rx_buffer[..amt]); self.rx_buffer.advance(amt); return Ok(amt); } diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index b1b2fcefaee..07d92111151 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -427,12 +427,11 @@ impl RemoteNetworkingServerDriver { let mut guard = self.common.sockets.lock().unwrap(); guard .get_mut(&socket_id) - .map(|s| { - req_id - .map(|req_id| s.send(&self.common, socket_id, data, req_id)) - .flatten() + .map(|s| s.send(&self.common, socket_id, data, req_id)) + .unwrap_or_else(|| { + tracing::debug!("orphaned socket {:?}", socket_id); + None }) - .unwrap_or(None) } fn process_send_to( @@ -1207,15 +1206,21 @@ impl RemoteAdapterSocket { common: &Arc, socket_id: SocketId, data: Vec, - req_id: u64, + req_id: Option, ) -> BackgroundTask { match self { Self::TcpSocket(this) => match this.try_send(&data) { - Ok(amount) => common.send(MessageResponse::Sent { - socket_id, - req_id, - amount: amount as u64, - }), + Ok(amount) => { + if let Some(req_id) = req_id { + common.send(MessageResponse::Sent { + socket_id, + req_id, + amount: amount as u64, + }) + } else { + None + } + } Err(NetworkError::WouldBlock) => { let common = common.clone(); Some(Box::pin(async move { @@ -1228,7 +1233,7 @@ impl RemoteAdapterSocket { common: Arc, socket_id: SocketId, data: Vec, - req_id: u64, + req_id: Option, } impl Future for Poller { type Output = BackgroundTask; @@ -1251,23 +1256,31 @@ impl RemoteAdapterSocket { if let RemoteAdapterSocket::TcpSocket(socket) = socket { match socket.try_send(&self.data) { Ok(amount) => { - return Poll::Ready(self.common.send( - MessageResponse::Sent { - socket_id: self.socket_id, - req_id: self.req_id, - amount: amount as u64, - }, - )) + if let Some(req_id) = self.req_id { + return Poll::Ready(self.common.send( + MessageResponse::Sent { + socket_id: self.socket_id, + req_id, + amount: amount as u64, + }, + )); + } else { + return Poll::Ready(None); + } } Err(NetworkError::WouldBlock) => return Poll::Pending, Err(error) => { - return Poll::Ready(self.common.send( - MessageResponse::SendError { - socket_id: self.socket_id, - req_id: self.req_id, - error, - }, - )) + if let Some(req_id) = self.req_id { + return Poll::Ready(self.common.send( + MessageResponse::SendError { + socket_id: self.socket_id, + req_id, + error, + }, + )); + } else { + return Poll::Ready(None); + } } } } @@ -1291,11 +1304,17 @@ impl RemoteAdapterSocket { } })) } - Err(error) => common.send(MessageResponse::SendError { - socket_id, - req_id, - error, - }), + Err(error) => { + if let Some(req_id) = req_id { + common.send(MessageResponse::SendError { + socket_id, + req_id, + error, + }) + } else { + None + } + } }, Self::RawSocket(this) => { // when the RAW socket is overloaded we just silently drop the packet @@ -1303,14 +1322,22 @@ impl RemoteAdapterSocket { // not lossless. In reality most socket drivers under this remote socket // will always succeed on `try_send` with RawSockets as they are always // processed. - this.try_send(&data).ok(); + if let Err(err) = this.try_send(&data) { + tracing::debug!("failed to send raw packet - {}", err); + } None } - _ => common.send(MessageResponse::SendError { - socket_id, - req_id, - error: NetworkError::Unsupported, - }), + _ => { + if let Some(req_id) = req_id { + common.send(MessageResponse::SendError { + socket_id, + req_id, + error: NetworkError::Unsupported, + }) + } else { + None + } + } } } pub fn send_to( @@ -1430,6 +1457,7 @@ impl RemoteAdapterSocket { }) { ret.push_back(task); } + continue; } Err(_) => {} } @@ -1450,6 +1478,7 @@ impl RemoteAdapterSocket { }) { ret.push_back(task); } + continue; } Err(_) => {} } @@ -1469,6 +1498,7 @@ impl RemoteAdapterSocket { }) { ret.push_back(task); } + continue; } Err(_) => {} } From 30acca8a49d45024987701ff58ac1f6d475c8866 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 25 Jul 2023 16:38:32 +1000 Subject: [PATCH 25/52] Reduced the warning level on channel closing for VPN sessions --- lib/virtual-net/src/rx_tx.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/virtual-net/src/rx_tx.rs b/lib/virtual-net/src/rx_tx.rs index 5f160180742..1dd982c08de 100644 --- a/lib/virtual-net/src/rx_tx.rs +++ b/lib/virtual-net/src/rx_tx.rs @@ -538,7 +538,7 @@ where RemoteRx::Stream { rx } => match rx.as_mut().poll_next(cx) { Poll::Ready(Some(Ok(msg))) => Poll::Ready(Some(msg)), Poll::Ready(Some(Err(err))) => { - tracing::warn!("failed to read from channel - {}", err); + tracing::debug!("failed to read from channel - {}", err); Poll::Ready(None) } Poll::Ready(None) => Poll::Ready(None), @@ -567,7 +567,7 @@ where continue; } Poll::Ready(Some(Err(err))) => { - tracing::warn!("failed to read from channel - {}", err); + tracing::debug!("failed to read from channel - {}", err); Poll::Ready(None) } Poll::Ready(None) => Poll::Ready(None), @@ -596,7 +596,7 @@ where continue; } Poll::Ready(Some(Err(err))) => { - tracing::warn!("failed to read from channel - {}", err); + tracing::debug!("failed to read from channel - {}", err); Poll::Ready(None) } Poll::Ready(None) => Poll::Ready(None), From deac05ca49186a4a88ee0a8b4b6ceb1f5cce4b32 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 26 Jul 2023 14:20:29 +1000 Subject: [PATCH 26/52] Fixed a linting issue due to some commented code --- lib/cli/src/cli.rs | 5 ----- lib/wasi-web/Cargo.lock | 1 + 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index bf8aa36be06..751f7a2f8df 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -270,11 +270,6 @@ enum Cmd { /// Create a dynamic on the Deploy Edge, and connect to it through SSH. Ssh(wasmer_deploy_cli::cmd::ssh::CmdSsh), - /* - /// Connect to the Edge. - Connect(wasmer_deploy_cli::cmd::connect::CmdConnect), - */ - /// Manage Wasmer namespaces. #[clap(subcommand, alias = "namespaces")] Namespace(wasmer_deploy_cli::cmd::namespace::CmdNamespace), diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 785cb3ca97f..2114e755e30 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -2239,6 +2239,7 @@ dependencies = [ name = "virtual-net" version = "0.4.0" dependencies = [ + "anyhow", "async-trait", "base64 0.21.2", "bincode", From 38b85fa186336ba2ac4cee92cbc4bd038a53f3ab Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 26 Jul 2023 22:27:53 +1000 Subject: [PATCH 27/52] Added a forced shutdown on tokio runtimes as the STDIN blocks the shutdown --- lib/c-api/src/wasm_c_api/wasi/mod.rs | 6 +-- lib/cli/src/commands/run.rs | 6 +-- lib/cli/src/commands/run/wasi.rs | 4 +- lib/wasix/src/os/console/mod.rs | 7 +-- lib/wasix/src/runtime/task_manager/thread.rs | 7 +++ lib/wasix/src/runtime/task_manager/tokio.rs | 56 ++++++++++++++++---- tests/lib/wast/src/wasi_wast.rs | 2 +- 7 files changed, 66 insertions(+), 22 deletions(-) diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index b08bf870fb6..7e8c5008d51 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -275,7 +275,7 @@ fn prepare_webc_env( let handle = runtime.handle().clone(); let _guard = handle.enter(); - let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(handle))); + let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(runtime))); rt.set_engine(Some(store_mut.engine().clone())); let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; @@ -324,7 +324,6 @@ pub struct wasi_env_t { /// cbindgen:ignore pub(super) inner: WasiFunctionEnv, pub(super) store: StoreRef, - pub(super) _runtime: tokio::runtime::Runtime, } /// Create a new WASI environment. @@ -349,7 +348,7 @@ pub unsafe extern "C" fn wasi_env_new( let handle = runtime.handle().clone(); let _guard = handle.enter(); - let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(handle))); + let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(runtime))); rt.set_engine(Some(store_mut.engine().clone())); if !config.inherit_stdout { @@ -370,7 +369,6 @@ pub unsafe extern "C" fn wasi_env_new( Some(Box::new(wasi_env_t { inner: env, store: store.clone(), - _runtime: runtime, })) } diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 355eedf0b95..e1f0e4ec503 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -105,9 +105,9 @@ impl Run { let _guard = handle.enter(); let (store, _) = self.store.get_store()?; - let runtime = - self.wasi - .prepare_runtime(store.engine().clone(), &self.env, handle.clone())?; + let runtime = self + .wasi + .prepare_runtime(store.engine().clone(), &self.env, runtime)?; // This is a slow operation, so let's temporarily wrap the runtime with // something that displays progress diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 3a887b584d1..5f74b6ddaff 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -251,9 +251,9 @@ impl Wasi { &self, engine: Engine, env: &WasmerEnv, - handle: Handle, + runtime: tokio::runtime::Runtime, ) -> Result { - let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(handle))); + let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(runtime))); if self.networking { rt.set_networking_implementation(virtual_net::host::LocalNetworking::default()); diff --git a/lib/wasix/src/os/console/mod.rs b/lib/wasix/src/os/console/mod.rs index 3700fcd71ea..66e19cfa88a 100644 --- a/lib/wasix/src/os/console/mod.rs +++ b/lib/wasix/src/os/console/mod.rs @@ -292,9 +292,10 @@ mod tests { #[cfg_attr(not(feature = "host-reqwest"), ignore = "Requires a HTTP client")] fn test_console_dash_tty_with_args_and_env() { let tokio_rt = tokio::runtime::Runtime::new().unwrap(); - let _guard = tokio_rt.handle().enter(); + let rt_handle = tokio_rt.handle().clone(); + let _guard = rt_handle.enter(); - let tm = TokioTaskManager::new(tokio_rt.handle().clone()); + let tm = TokioTaskManager::new(tokio_rt); let mut rt = PluggableRuntime::new(Arc::new(tm)); rt.set_engine(Some(wasmer::Engine::default())) .set_package_loader(BuiltinPackageLoader::from_env().unwrap()); @@ -316,7 +317,7 @@ mod tests { .run() .unwrap(); - let code = tokio_rt + let code = rt_handle .block_on(async move { virtual_fs::AsyncWriteExt::write_all( &mut stdin_tx, diff --git a/lib/wasix/src/runtime/task_manager/thread.rs b/lib/wasix/src/runtime/task_manager/thread.rs index b330850b9c6..f33575657eb 100644 --- a/lib/wasix/src/runtime/task_manager/thread.rs +++ b/lib/wasix/src/runtime/task_manager/thread.rs @@ -18,6 +18,13 @@ pub struct ThreadTaskManager { pool: Arc, } +impl Drop +for ThreadTaskManager { + fn drop(&mut self) { + self.runtime.shutdown_timeout(Duration::from_secs(0)); + } +} + impl Default for ThreadTaskManager { #[cfg(feature = "sys-thread")] fn default() -> Self { diff --git a/lib/wasix/src/runtime/task_manager/tokio.rs b/lib/wasix/src/runtime/task_manager/tokio.rs index d7cad4789c6..89bae085de5 100644 --- a/lib/wasix/src/runtime/task_manager/tokio.rs +++ b/lib/wasix/src/runtime/task_manager/tokio.rs @@ -1,28 +1,66 @@ use std::{num::NonZeroUsize, pin::Pin, sync::Arc, time::Duration}; +use std::sync::Mutex; use futures::{future::BoxFuture, Future}; -use tokio::runtime::Handle; +use tokio::runtime::{Handle, Runtime}; use crate::{os::task::thread::WasiThreadError, WasiFunctionEnv}; use super::{TaskWasm, TaskWasmRunProperties, VirtualTaskManager}; +#[derive(Debug, Clone)] +pub enum RuntimeOrHandle { + Handle(Handle), + Runtime(Handle, Arc>>), +} +impl From +for RuntimeOrHandle { + fn from(value: Handle) -> Self { + Self::Handle(value) + } +} +impl From +for RuntimeOrHandle { + fn from(value: Runtime) -> Self { + Self::Runtime(value.handle().clone(), Arc::new(Mutex::new(Some(value)))) + } +} + +impl Drop +for RuntimeOrHandle { + fn drop(&mut self) { + if let Self::Runtime(_, runtime) = self { + runtime.lock().unwrap().take().map(|h| h.shutdown_timeout(Duration::from_secs(0))); + } + } +} + +impl RuntimeOrHandle { + pub fn handle(&self) -> &Handle { + match self { + Self::Handle(h) => h, + Self::Runtime(h, _) => h + } + } +} + /// A task manager that uses tokio to spawn tasks. #[derive(Clone, Debug)] pub struct TokioTaskManager { - handle: Handle, + rt: RuntimeOrHandle, pool: Arc, } impl TokioTaskManager { - pub fn new(rt: Handle) -> Self { + pub fn new(rt: I) -> Self + where I: Into { let concurrency = std::thread::available_parallelism() .unwrap_or(NonZeroUsize::new(1).unwrap()) .get(); let max_threads = 200usize.max(concurrency * 100); Self { - handle: rt, + rt: rt.into(), pool: Arc::new( rayon::ThreadPoolBuilder::new() .num_threads(max_threads) @@ -33,13 +71,13 @@ impl TokioTaskManager { } pub fn runtime_handle(&self) -> tokio::runtime::Handle { - self.handle.clone() + self.rt.handle().clone() } } impl Default for TokioTaskManager { fn default() -> Self { - Self::new(Handle::current()) + Self::new(Handle::current().clone()) } } @@ -55,7 +93,7 @@ impl VirtualTaskManager for TokioTaskManager { /// See [`VirtualTaskManager::sleep_now`]. fn sleep_now(&self, time: Duration) -> Pin + Send + Sync>> { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); - self.handle.spawn(async move { + self.rt.handle().spawn(async move { if time == Duration::ZERO { tokio::task::yield_now().await; } else { @@ -73,7 +111,7 @@ impl VirtualTaskManager for TokioTaskManager { &self, task: Box BoxFuture<'static, ()> + Send + 'static>, ) -> Result<(), WasiThreadError> { - self.handle.spawn(async move { + self.rt.handle().spawn(async move { let fut = task(); fut.await }); @@ -99,7 +137,7 @@ impl VirtualTaskManager for TokioTaskManager { let trigger = trigger(); let pool = self.pool.clone(); - self.handle.spawn(async move { + self.rt.handle().spawn(async move { let result = trigger.await; // Build the task that will go on the callback pool.spawn(move || { diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index e5d9d61f1c0..8354ea775da 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -106,7 +106,7 @@ impl<'a> WasiTest<'a> { #[cfg(not(target_arch = "wasm32"))] let _guard = handle.enter(); #[cfg(not(target_arch = "wasm32"))] - let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(handle))); + let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(runtime))); #[cfg(target_arch = "wasm32")] let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::default())); rt.set_engine(Some(store.engine().clone())); From 38d35f859e9537fade7f2b3fdd1d80789a09ff0f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 26 Jul 2023 22:35:54 +1000 Subject: [PATCH 28/52] Linting fixes --- lib/wasix/src/runtime/task_manager/tokio.rs | 23 ++++++++++++--------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/wasix/src/runtime/task_manager/tokio.rs b/lib/wasix/src/runtime/task_manager/tokio.rs index 89bae085de5..aff3e1e07e8 100644 --- a/lib/wasix/src/runtime/task_manager/tokio.rs +++ b/lib/wasix/src/runtime/task_manager/tokio.rs @@ -1,5 +1,5 @@ -use std::{num::NonZeroUsize, pin::Pin, sync::Arc, time::Duration}; use std::sync::Mutex; +use std::{num::NonZeroUsize, pin::Pin, sync::Arc, time::Duration}; use futures::{future::BoxFuture, Future}; use tokio::runtime::{Handle, Runtime}; @@ -13,24 +13,25 @@ pub enum RuntimeOrHandle { Handle(Handle), Runtime(Handle, Arc>>), } -impl From -for RuntimeOrHandle { +impl From for RuntimeOrHandle { fn from(value: Handle) -> Self { Self::Handle(value) } } -impl From -for RuntimeOrHandle { +impl From for RuntimeOrHandle { fn from(value: Runtime) -> Self { Self::Runtime(value.handle().clone(), Arc::new(Mutex::new(Some(value)))) } } -impl Drop -for RuntimeOrHandle { +impl Drop for RuntimeOrHandle { fn drop(&mut self) { if let Self::Runtime(_, runtime) = self { - runtime.lock().unwrap().take().map(|h| h.shutdown_timeout(Duration::from_secs(0))); + runtime + .lock() + .unwrap() + .take() + .map(|h| h.shutdown_timeout(Duration::from_secs(0))); } } } @@ -39,7 +40,7 @@ impl RuntimeOrHandle { pub fn handle(&self) -> &Handle { match self { Self::Handle(h) => h, - Self::Runtime(h, _) => h + Self::Runtime(h, _) => h, } } } @@ -53,7 +54,9 @@ pub struct TokioTaskManager { impl TokioTaskManager { pub fn new(rt: I) -> Self - where I: Into { + where + I: Into, + { let concurrency = std::thread::available_parallelism() .unwrap_or(NonZeroUsize::new(1).unwrap()) .get(); From 8acb19498d6c5c4d1112f348df1073218f0850ca Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 26 Jul 2023 22:46:23 +1000 Subject: [PATCH 29/52] More lint fixes and a compilation fix for c_api --- lib/c-api/src/wasm_c_api/wasi/mod.rs | 7 +++---- lib/wasix/src/runtime/task_manager/tokio.rs | 10 ++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index 7e8c5008d51..508293dcb93 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -233,7 +233,7 @@ unsafe fn wasi_env_with_filesystem_inner( let module = &module.as_ref()?.inner; let imports = imports?; - let (wasi_env, import_object, runtime) = prepare_webc_env( + let (wasi_env, import_object) = prepare_webc_env( config, &mut store.store_mut(), module, @@ -247,7 +247,6 @@ unsafe fn wasi_env_with_filesystem_inner( Some(Box::new(wasi_env_t { inner: wasi_env, store: store.clone(), - _runtime: runtime, })) } @@ -259,7 +258,7 @@ fn prepare_webc_env( bytes: &'static u8, len: usize, package_name: &str, -) -> Option<(WasiFunctionEnv, Imports, tokio::runtime::Runtime)> { +) -> Option<(WasiFunctionEnv, Imports)> { use virtual_fs::static_fs::StaticFileSystem; use webc::v1::{FsEntryType, WebC}; @@ -316,7 +315,7 @@ fn prepare_webc_env( let env = builder.finalize(store).ok()?; let import_object = env.import_object(store, module).ok()?; - Some((env, import_object, runtime)) + Some((env, import_object)) } #[allow(non_camel_case_types)] diff --git a/lib/wasix/src/runtime/task_manager/tokio.rs b/lib/wasix/src/runtime/task_manager/tokio.rs index aff3e1e07e8..f7b0dc1d5d5 100644 --- a/lib/wasix/src/runtime/task_manager/tokio.rs +++ b/lib/wasix/src/runtime/task_manager/tokio.rs @@ -27,11 +27,9 @@ impl From for RuntimeOrHandle { impl Drop for RuntimeOrHandle { fn drop(&mut self) { if let Self::Runtime(_, runtime) = self { - runtime - .lock() - .unwrap() - .take() - .map(|h| h.shutdown_timeout(Duration::from_secs(0))); + if let Some(h) = runtime.lock().unwrap().take() { + h.shutdown_timeout(Duration::from_secs(0)) + } } } } @@ -80,7 +78,7 @@ impl TokioTaskManager { impl Default for TokioTaskManager { fn default() -> Self { - Self::new(Handle::current().clone()) + Self::new(Handle::current()) } } From 7941eb9a6e7f6862937936acbd063d57f5b4354f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 26 Jul 2023 23:02:40 +1000 Subject: [PATCH 30/52] Made prepare_runtime backwards compatible --- lib/cli/src/commands/run/wasi.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 5f74b6ddaff..dc8c13e7e02 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -28,7 +28,10 @@ use wasmer_wasix::{ FileSystemSource, InMemorySource, MultiSource, PackageSpecifier, Source, WapmSource, WebSource, }, - task_manager::{tokio::TokioTaskManager, VirtualTaskManagerExt}, + task_manager::{ + tokio::{RuntimeOrHandle, TokioTaskManager}, + VirtualTaskManagerExt, + }, }, types::__WASI_STDIN_FILENO, wasmer_wasix_types::wasi::Errno, @@ -247,13 +250,16 @@ impl Wasi { caps } - pub fn prepare_runtime( + pub fn prepare_runtime( &self, engine: Engine, env: &WasmerEnv, - runtime: tokio::runtime::Runtime, - ) -> Result { - let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(runtime))); + rt_or_handle: I, + ) -> Result + where + I: Into, + { + let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(rt_or_handle.into()))); if self.networking { rt.set_networking_implementation(virtual_net::host::LocalNetworking::default()); From 9fdb2c17213fe5a5352221d0ef1b59c9dd189598 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 27 Jul 2023 13:19:08 +1000 Subject: [PATCH 31/52] wasmer-sh now takes a configuration file and auto redirects http to https --- lib/wasi-web/Cargo.lock | 1 + lib/wasi-web/Cargo.toml | 1 + lib/wasi-web/js/index.js | 16 +++++- lib/wasi-web/js/init.js | 11 ++++ lib/wasi-web/public/index.html | 1 + lib/wasi-web/src/glue.rs | 99 +++++++++++++++++++-------------- lib/wasi-web/webpack.config.mjs | 1 + 7 files changed, 86 insertions(+), 44 deletions(-) create mode 100644 lib/wasi-web/js/init.js diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index b5dde71bdd6..c1b3e48dba3 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -2698,6 +2698,7 @@ dependencies = [ "derivative", "dummy-waker", "fastrand 1.9.0", + "form_urlencoded", "futures", "http", "js-sys", diff --git a/lib/wasi-web/Cargo.toml b/lib/wasi-web/Cargo.toml index 757ee2c5359..89fb8f4133a 100644 --- a/lib/wasi-web/Cargo.toml +++ b/lib/wasi-web/Cargo.toml @@ -22,6 +22,7 @@ wasmer-wasix = { path = "../wasix", version = "0.10.0", default-features = fals wasm-bindgen = { version = "0.2", features = [ "serde-serialize" ] } wasm-bindgen-futures = "0.4" console_error_panic_hook = "^0.1" +form_urlencoded = "1.2" js-sys = "0.3" tracing = { version = "^0.1", features = [ "log", "release_max_level_info" ] } #tracing = { version = "^0.1", features = [ "log" ] } diff --git a/lib/wasi-web/js/index.js b/lib/wasi-web/js/index.js index abf6eafd7a4..6f316887759 100644 --- a/lib/wasi-web/js/index.js +++ b/lib/wasi-web/js/index.js @@ -1,11 +1,25 @@ import init, { start } from '../pkg/index.js'; +import { init_cfg } from '../js/init.js'; import 'regenerator-runtime/runtime.js' import './workers-polyfill.js' +if (location.protocol !== 'https:') { + if (location.hostname !== 'localhost') { + location.replace(`https:${location.href.substring(location.protocol.length)}`); + } +} + +export function init_encoded() { + return 'init=' + encodeURIComponent(init_cfg.init) + '&' + + 'uses=' + encodeURIComponent(init_cfg.uses) + '&' + + 'prompt=' + encodeURIComponent(init_cfg.prompt) + '&' + + 'no_welcome=' + encodeURIComponent(init_cfg.no_welcome); +} + async function run() { Error.stackTraceLimit = 20; await init(); - await start(); + await start(init_encoded()); } run(); diff --git a/lib/wasi-web/js/init.js b/lib/wasi-web/js/init.js new file mode 100644 index 00000000000..0e56cd27816 --- /dev/null +++ b/lib/wasi-web/js/init.js @@ -0,0 +1,11 @@ +export const init_cfg = { + init: 'sharrattj/bash', + uses: [ + 'sharrattj/coreutils', + 'curl/curl', + 'john-sharratt/catsay', + 'python/python' + ], + prompt: 'wasmer.sh', + no_welcome: false +}; diff --git a/lib/wasi-web/public/index.html b/lib/wasi-web/public/index.html index a0a9c5f4b2a..c0064e3dbba 100644 --- a/lib/wasi-web/public/index.html +++ b/lib/wasi-web/public/index.html @@ -19,6 +19,7 @@ wasmer.sh + diff --git a/lib/wasi-web/src/glue.rs b/lib/wasi-web/src/glue.rs index 8b38c581896..a2a7d59052c 100644 --- a/lib/wasi-web/src/glue.rs +++ b/lib/wasi-web/src/glue.rs @@ -37,11 +37,43 @@ pub fn main() { set_panic_hook(); } -pub const DEFAULT_BOOT_WEBC: &str = "sharrattj/bash"; -pub const DEFAULT_BOOT_USES: [&str; 1] = ["sharrattj/coreutils"]; +#[derive(Debug, Clone, Default)] +pub struct StartArgs { + init: Option, + uses: Vec, + prompt: Option, + no_welcome: bool, + token: Option, +} +impl StartArgs { + pub fn parse(mut self, args: &str) -> Self { + let query_pairs = || form_urlencoded::parse(args.as_bytes()); + + if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "init").next() { + self.init = Some(val.to_string()); + } + if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "uses").next() { + self.uses = val.split(",").map(|v| v.to_string()).collect(); + } + if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "prompt").next() { + self.prompt = Some(val.to_string()); + } + if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "no_welcome").next() { + match val.as_ref() { + "true" | "yes" | "" => self.no_welcome = true, + "false" | "no" => self.no_welcome = false, + _ => {} + } + } + if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "token").next() { + self.token = Some(val.to_string()); + } + self + } +} #[wasm_bindgen] -pub fn start() -> Result<(), JsValue> { +pub fn start(encoded_args: String) -> Result<(), JsValue> { #[wasm_bindgen] extern "C" { #[wasm_bindgen(js_namespace = navigator, js_name = userAgent)] @@ -82,6 +114,13 @@ pub fn start() -> Result<(), JsValue> { let is_mobile = wasmer_wasix::os::common::is_mobile(&user_agent); debug!("user_agent: {}", user_agent); + // Compute the configuration + let mut args = StartArgs::default().parse(&encoded_args); + let location = url::Url::parse(location.as_str()).unwrap(); + if let Some(query) = location.query() { + args = args.parse(query); + } + let elem = window .document() .unwrap() @@ -137,21 +176,20 @@ pub fn start() -> Result<(), JsValue> { tty_options, ); - let location = url::Url::parse(location.as_str()).unwrap(); - let mut console = if let Some(init) = location - .query_pairs() - .filter(|(key, _)| key == "init") - .next() - .map(|(_, val)| val.to_string()) - { - let mut console = Console::new(init.as_str(), runtime.clone()); - console = console.with_no_welcome(true); - console - } else { - let mut console = Console::new(DEFAULT_BOOT_WEBC, runtime.clone()); - console = console.with_uses(DEFAULT_BOOT_USES.iter().map(|a| a.to_string()).collect()); - console - }; + let init = args.init.ok_or(JsValue::from_str( + "no initialization package has been specified", + ))?; + let prompt = args + .prompt + .ok_or(JsValue::from_str("no prompt has been specified"))?; + + let mut console = Console::new(init.as_str(), runtime.clone()) + .with_no_welcome(args.no_welcome) + .with_prompt(prompt); + + if let Some(token) = args.token { + console = console.with_token(token); + } let mut env = HashMap::new(); if let Some(origin) = location.domain().clone() { @@ -159,31 +197,6 @@ pub fn start() -> Result<(), JsValue> { } env.insert("LOCATION".to_string(), location.to_string()); - if let Some(prompt) = location - .query_pairs() - .filter(|(key, _)| key == "prompt") - .next() - .map(|(_, val)| val.to_string()) - { - console = console.with_prompt(prompt); - } - - if location - .query_pairs() - .any(|(key, _)| key == "no_welcome" || key == "no-welcome") - { - console = console.with_no_welcome(true); - } - - if let Some(token) = location - .query_pairs() - .filter(|(key, _)| key == "token") - .next() - .map(|(_, val)| val.to_string()) - { - console = console.with_token(token); - } - console = console .with_user_agent(user_agent.as_str()) .with_stdin(Box::new(stdin_rx)) diff --git a/lib/wasi-web/webpack.config.mjs b/lib/wasi-web/webpack.config.mjs index 5a661059240..83d344f8b32 100644 --- a/lib/wasi-web/webpack.config.mjs +++ b/lib/wasi-web/webpack.config.mjs @@ -17,6 +17,7 @@ export default { { from: resolve(__dirname, "public/wasmer.css") }, { from: resolve(__dirname, "public/worker.js") }, { from: resolve(__dirname, "public/favicon.png") }, + { from: resolve(__dirname, "js/init.js") }, ], }), new WasmPackPlugin({ From ba9b5c6a7f6ac98bc40b45df78f44ccef242cd32 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 27 Jul 2023 13:56:17 +1000 Subject: [PATCH 32/52] Optimized the build tests and build sizes --- lib/wasi-web/Cargo.toml | 23 +++++++++-------------- lib/wasi-web/package.json | 4 ++-- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/wasi-web/Cargo.toml b/lib/wasi-web/Cargo.toml index 89fb8f4133a..b077b7fdb81 100644 --- a/lib/wasi-web/Cargo.toml +++ b/lib/wasi-web/Cargo.toml @@ -140,16 +140,15 @@ build-deps = "^0.1" [lib] crate-type = ["cdylib"] +[profile.dev] +opt-level = 1 + +[profile.release] +lto = true +opt-level = 'z' + [package.metadata.wasm-pack.profile.dev] -# Should `wasm-opt` be used to further optimize the wasm binary generated after -# the Rust compiler has finished? Using `wasm-opt` can often further decrease -# binary size or do clever tricks that haven't made their way into LLVM yet. -# -# Configuration is set to `false` by default for the dev profile, but it can -# be set to an array of strings which are explicit arguments to pass to -# `wasm-opt`. For example `['-Os']` would optimize for size while `['-O4']` -# would execute very expensive optimizations passes -wasm-opt = ['--strip-debug --enable-reference-types'] +wasm-opt = false [package.metadata.wasm-pack.profile.dev.wasm-bindgen] debug-js-glue = true @@ -157,11 +156,7 @@ demangle-name-section = true dwarf-debug-info = true [package.metadata.wasm-pack.profile.release] -# The version of wasm-opt that wasm-pack bundles crashes on current wasm-bindgen -# .wasm files. Current wasm-opt (version 93) crashes on the DWARF info that -# wasm-bindgen produces. So, we'll just disable wasm-opt for now. -wasm-opt = false #["-O4"] -#wasm-opt = ['--strip-debug --enable-reference-types'] +wasm-opt = ['-Oz'] [package.metadata.wasm-pack.profile.release.wasm-bindgen] debug-js-glue = false diff --git a/lib/wasi-web/package.json b/lib/wasi-web/package.json index 0b5c38d7e22..560c5be75f2 100644 --- a/lib/wasi-web/package.json +++ b/lib/wasi-web/package.json @@ -7,7 +7,7 @@ "type": "module", "scripts": { "build": "webpack", - "dev": "webpack serve" + "dev": "webpack serve --mode development" }, "dependencies": { "comlink": "^4", @@ -27,4 +27,4 @@ "webpack-cli": "^4.8", "webpack-dev-server": "^4.9.0" } -} +} \ No newline at end of file From b383a74b9f967a8df38337ba5f4af7077dae6199 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 27 Jul 2023 14:03:10 +1000 Subject: [PATCH 33/52] Development mode no longer attaches all the debug info --- lib/wasi-web/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/wasi-web/Cargo.toml b/lib/wasi-web/Cargo.toml index b077b7fdb81..34a3b67b8b6 100644 --- a/lib/wasi-web/Cargo.toml +++ b/lib/wasi-web/Cargo.toml @@ -142,6 +142,7 @@ crate-type = ["cdylib"] [profile.dev] opt-level = 1 +debug = "line-tables-only" [profile.release] lto = true From 2713944a31f3aacb91d301231c9625e6f52dd199 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 27 Jul 2023 18:09:26 +1000 Subject: [PATCH 34/52] Preparing another release --- lib/wasi-web/public/index.html | 2 +- lib/wasi-web/wasmer.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/wasi-web/public/index.html b/lib/wasi-web/public/index.html index c0064e3dbba..be66add0cab 100644 --- a/lib/wasi-web/public/index.html +++ b/lib/wasi-web/public/index.html @@ -19,7 +19,7 @@ wasmer.sh - + diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index e684ff8a0f6..8a46cf6755d 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] -name = 'wasmer/wasmer-sh-async' -version = '1.0.3' +name = 'wasmer/wasmer-sh' +version = '0.2.8' description = 'Container that holds the wasmer.sh website.' [dependencies] From 7b3ccb353516393c1c785af21781fb9e6f6df47b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Thu, 27 Jul 2023 18:55:41 +1000 Subject: [PATCH 35/52] Released new version of wasmer.sh --- lib/wasi-web/wasmer.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index 8a46cf6755d..ae0e1f14326 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.8' +version = '0.2.9' description = 'Container that holds the wasmer.sh website.' [dependencies] From 46623a81ec805b5ce24b44bb9bc08684d794146d Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Fri, 28 Jul 2023 08:48:08 +1000 Subject: [PATCH 36/52] Added a cors configuration for the CDN --- lib/wasi-web/cors.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 lib/wasi-web/cors.json diff --git a/lib/wasi-web/cors.json b/lib/wasi-web/cors.json new file mode 100644 index 00000000000..277d9827af4 --- /dev/null +++ b/lib/wasi-web/cors.json @@ -0,0 +1,8 @@ +[ + { + "origin": ["*"], + "method": ["GET", "OPTIONS"], + "responseHeader": ["Content-Type", "user-agent"], + "maxAgeSeconds": 3600 + } +] From bff2d8c0deaa91765bcd4b1dda04fed444c5aadc Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 29 Jul 2023 05:23:37 +1000 Subject: [PATCH 37/52] Fixed up a couple of problems with the init.js config bootstrap --- lib/wasi-web/cors.json | 2 +- lib/wasi-web/src/glue.rs | 3 ++- lib/wasi-web/wasmer.toml | 2 +- lib/wasix/src/os/console/mod.rs | 29 +++++++++++++++++++++++++++-- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/lib/wasi-web/cors.json b/lib/wasi-web/cors.json index 277d9827af4..70ae5fb74b3 100644 --- a/lib/wasi-web/cors.json +++ b/lib/wasi-web/cors.json @@ -2,7 +2,7 @@ { "origin": ["*"], "method": ["GET", "OPTIONS"], - "responseHeader": ["Content-Type", "user-agent"], + "responseHeader": ["*"], "maxAgeSeconds": 3600 } ] diff --git a/lib/wasi-web/src/glue.rs b/lib/wasi-web/src/glue.rs index a2a7d59052c..26a4cb23103 100644 --- a/lib/wasi-web/src/glue.rs +++ b/lib/wasi-web/src/glue.rs @@ -61,7 +61,6 @@ impl StartArgs { if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "no_welcome").next() { match val.as_ref() { "true" | "yes" | "" => self.no_welcome = true, - "false" | "no" => self.no_welcome = false, _ => {} } } @@ -187,6 +186,8 @@ pub fn start(encoded_args: String) -> Result<(), JsValue> { .with_no_welcome(args.no_welcome) .with_prompt(prompt); + console = console.with_uses(args.uses); + if let Some(token) = args.token { console = console.with_token(token); } diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index ae0e1f14326..e10572e719e 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.9' +version = '0.2.10' description = 'Container that holds the wasmer.sh website.' [dependencies] diff --git a/lib/wasix/src/os/console/mod.rs b/lib/wasix/src/os/console/mod.rs index 66e19cfa88a..886b44ab2e3 100644 --- a/lib/wasix/src/os/console/mod.rs +++ b/lib/wasix/src/os/console/mod.rs @@ -4,10 +4,11 @@ pub mod cconst; use std::{ + borrow::Cow, collections::HashMap, io::Write, ops::{Deref, DerefMut}, - path::Path, + path::{Path, PathBuf}, sync::{atomic::AtomicBool, Arc, Mutex}, }; @@ -18,7 +19,7 @@ use tokio::sync::{mpsc, RwLock}; use tracing::{debug, error, info, trace, warn}; use virtual_fs::{ ArcBoxFile, ArcFile, AsyncWriteExt, CombineFile, DeviceFile, DuplexPipe, FileSystem, Pipe, - PipeRx, PipeTx, RootFileSystemBuilder, VirtualFile, + PipeRx, PipeTx, RootFileSystemBuilder, StaticFile, VirtualFile, }; #[cfg(feature = "sys")] use wasmer::Engine; @@ -51,6 +52,7 @@ pub struct Console { stdout: ArcBoxFile, stderr: ArcBoxFile, capabilities: Capabilities, + ro_files: HashMap>, memfs_memory_limiter: Option, } @@ -73,6 +75,7 @@ impl Console { stderr: ArcBoxFile::new(Box::new(Pipe::channel().0)), capabilities: Default::default(), memfs_memory_limiter: None, + ro_files: Default::default(), } } @@ -135,6 +138,11 @@ impl Console { self } + pub fn with_ro_files(mut self, ro_files: HashMap>) -> Self { + self.ro_files = ro_files; + self + } + pub fn with_mem_fs_memory_limiter( mut self, limiter: virtual_fs::limiter::DynFsMemoryLimiter, @@ -245,6 +253,23 @@ impl Console { return Err(SpawnError::BadRequest); } + // The custom readonly files have to be added after the uses packages + // otherwise they will be overriden by their attached file systems + for (path, data) in self.ro_files.clone() { + let path = PathBuf::from(path); + env.fs_root().remove_file(&path).ok(); + let mut file = env + .fs_root() + .new_open_options() + .create(true) + .truncate(true) + .write(true) + .open(&path) + .map_err(|err| SpawnError::Other(err.into()))?; + InlineWaker::block_on(file.copy_reference(Box::new(StaticFile::new(data)))) + .map_err(|err| SpawnError::Other(err.into()))?; + } + // Build the config // Run the binary let store = self.runtime.new_store(); From efa6a80fd85b97b55b9f884915a4e872f6dda57b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 29 Jul 2023 05:34:28 +1000 Subject: [PATCH 38/52] Disabled asynchronous threading --- lib/wasi-web/src/glue.rs | 2 +- lib/wasi-web/wasmer.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasi-web/src/glue.rs b/lib/wasi-web/src/glue.rs index 26a4cb23103..e725351e592 100644 --- a/lib/wasi-web/src/glue.rs +++ b/lib/wasi-web/src/glue.rs @@ -207,7 +207,7 @@ pub fn start(encoded_args: String) -> Result<(), JsValue> { let mut capabilities = Capabilities::default(); capabilities.threading.max_threads = Some(50); - capabilities.threading.enable_asynchronous_threading = true; + capabilities.threading.enable_asynchronous_threading = false; console = console.with_capabilities(capabilities); let (tx, mut rx) = mpsc::unbounded_channel(); diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index e10572e719e..daa189c163b 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.10' +version = '0.2.12' description = 'Container that holds the wasmer.sh website.' [dependencies] From e7ac25b0e50878748db15d6d71349704476bf92b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 29 Jul 2023 05:37:28 +1000 Subject: [PATCH 39/52] Switched to the old http server for now --- lib/wasi-web/wasmer.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index daa189c163b..ed182312577 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,10 +1,10 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.12' +version = '0.2.13' description = 'Container that holds the wasmer.sh website.' [dependencies] -"wasmer/static-web-server-async" = '1' +"wasmer/static-web-server" = '1' [fs] public = 'dist' From f72fd33864541ab17c693cbe8d2888f58c68dfac Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 29 Jul 2023 05:40:32 +1000 Subject: [PATCH 40/52] Removed additional dependencies --- lib/wasi-web/js/init.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/wasi-web/js/init.js b/lib/wasi-web/js/init.js index 0e56cd27816..c1eb9c78598 100644 --- a/lib/wasi-web/js/init.js +++ b/lib/wasi-web/js/init.js @@ -1,10 +1,7 @@ export const init_cfg = { init: 'sharrattj/bash', uses: [ - 'sharrattj/coreutils', - 'curl/curl', 'john-sharratt/catsay', - 'python/python' ], prompt: 'wasmer.sh', no_welcome: false From c67720c418ef2064a17feaf1803a984594bd0af0 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sat, 29 Jul 2023 05:40:50 +1000 Subject: [PATCH 41/52] Publishing a new version --- lib/wasi-web/wasmer.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index ed182312577..33031fc6205 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.13' +version = '0.2.14' description = 'Container that holds the wasmer.sh website.' [dependencies] From 62e4758b7a8bf1752204d15dc7d9dfc46541148e Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 30 Jul 2023 16:56:22 +1000 Subject: [PATCH 42/52] Fixed the init.json file --- lib/wasi-web/js/index.js | 6 +++--- lib/wasi-web/js/init.js | 8 -------- lib/wasi-web/public/index.html | 1 - lib/wasi-web/public/init.json | 9 +++++++++ lib/wasi-web/webpack.config.mjs | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) delete mode 100644 lib/wasi-web/js/init.js create mode 100644 lib/wasi-web/public/init.json diff --git a/lib/wasi-web/js/index.js b/lib/wasi-web/js/index.js index 6f316887759..a5d625e6d8c 100644 --- a/lib/wasi-web/js/index.js +++ b/lib/wasi-web/js/index.js @@ -1,5 +1,4 @@ import init, { start } from '../pkg/index.js'; -import { init_cfg } from '../js/init.js'; import 'regenerator-runtime/runtime.js' import './workers-polyfill.js' @@ -9,7 +8,8 @@ if (location.protocol !== 'https:') { } } -export function init_encoded() { +async function init_encoded() { + let init_cfg = await (await fetch("/init.json")).json(); return 'init=' + encodeURIComponent(init_cfg.init) + '&' + 'uses=' + encodeURIComponent(init_cfg.uses) + '&' + 'prompt=' + encodeURIComponent(init_cfg.prompt) + '&' + @@ -19,7 +19,7 @@ export function init_encoded() { async function run() { Error.stackTraceLimit = 20; await init(); - await start(init_encoded()); + await start(await init_encoded()); } run(); diff --git a/lib/wasi-web/js/init.js b/lib/wasi-web/js/init.js deleted file mode 100644 index c1eb9c78598..00000000000 --- a/lib/wasi-web/js/init.js +++ /dev/null @@ -1,8 +0,0 @@ -export const init_cfg = { - init: 'sharrattj/bash', - uses: [ - 'john-sharratt/catsay', - ], - prompt: 'wasmer.sh', - no_welcome: false -}; diff --git a/lib/wasi-web/public/index.html b/lib/wasi-web/public/index.html index be66add0cab..a0a9c5f4b2a 100644 --- a/lib/wasi-web/public/index.html +++ b/lib/wasi-web/public/index.html @@ -19,7 +19,6 @@ wasmer.sh - diff --git a/lib/wasi-web/public/init.json b/lib/wasi-web/public/init.json new file mode 100644 index 00000000000..3de504ac1ca --- /dev/null +++ b/lib/wasi-web/public/init.json @@ -0,0 +1,9 @@ +{ + "init": "sharrattj/bash", + "uses": [ + "john-sharratt/catsay", + "python/python" + ], + "prompt": "wasmer.sh", + "no_welcome": false +} \ No newline at end of file diff --git a/lib/wasi-web/webpack.config.mjs b/lib/wasi-web/webpack.config.mjs index 83d344f8b32..e426e9cc7c3 100644 --- a/lib/wasi-web/webpack.config.mjs +++ b/lib/wasi-web/webpack.config.mjs @@ -17,7 +17,7 @@ export default { { from: resolve(__dirname, "public/wasmer.css") }, { from: resolve(__dirname, "public/worker.js") }, { from: resolve(__dirname, "public/favicon.png") }, - { from: resolve(__dirname, "js/init.js") }, + { from: resolve(__dirname, "public/init.json") }, ], }), new WasmPackPlugin({ From fbc495877c74e00c35edb348285e292ea9cee188 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 30 Jul 2023 16:56:40 +1000 Subject: [PATCH 43/52] Another release of wasmer.sh --- lib/wasi-web/wasmer.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index 33031fc6205..647433599e4 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.14' +version = '0.2.15' description = 'Container that holds the wasmer.sh website.' [dependencies] From 8b52814f4b9de25bbcf85fdd41220c453a863736 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Sun, 30 Jul 2023 17:19:39 +1000 Subject: [PATCH 44/52] Releasing another version --- lib/wasi-web/wasmer.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index 647433599e4..7edbc4df643 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.15' +version = '0.2.16' description = 'Container that holds the wasmer.sh website.' [dependencies] From 7e1d386d7121c47a7d8938a596a90a652e88702f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 31 Jul 2023 06:40:16 +1000 Subject: [PATCH 45/52] Virtual net is now a dependency for wasmer-sh --- lib/virtual-net/src/rx_tx.rs | 71 +++++++++++++++++-------------- lib/virtual-net/src/server.rs | 4 +- lib/wasi-web/Cargo.lock | 80 +++++++++++++++++++++++++++++++++++ lib/wasi-web/Cargo.toml | 1 + lib/wasi-web/src/ws.rs | 7 +-- 5 files changed, 125 insertions(+), 38 deletions(-) diff --git a/lib/virtual-net/src/rx_tx.rs b/lib/virtual-net/src/rx_tx.rs index 5215cd6a2ef..c7a5d27022f 100644 --- a/lib/virtual-net/src/rx_tx.rs +++ b/lib/virtual-net/src/rx_tx.rs @@ -5,12 +5,13 @@ use std::{ task::{Context, Poll, Waker}, }; -use crate::{meta::FrameSerializationFormat, Result}; +use crate::Result; use futures_util::{future::BoxFuture, Future, Sink, SinkExt, Stream}; use serde::Serialize; +#[cfg(feature = "tokio-tungstenite")] +use tokio::net::TcpStream; use tokio::{ io::AsyncWrite, - net::TcpStream, sync::{ mpsc::{self, error::TrySendError}, oneshot, @@ -71,6 +72,7 @@ where work: mpsc::UnboundedSender>, wakers: RemoteTxWakers, }, + #[cfg(feature = "hyper")] HyperWebSocket { tx: Arc< tokio::sync::Mutex< @@ -82,8 +84,9 @@ where >, work: mpsc::UnboundedSender>, wakers: RemoteTxWakers, - format: FrameSerializationFormat, + format: crate::meta::FrameSerializationFormat, }, + #[cfg(feature = "tokio-tungstenite")] TokioWebSocket { tx: Arc< tokio::sync::Mutex< @@ -97,7 +100,7 @@ where >, work: mpsc::UnboundedSender>, wakers: RemoteTxWakers, - format: FrameSerializationFormat, + format: crate::meta::FrameSerializationFormat, }, } impl RemoteTx @@ -126,14 +129,14 @@ where .await .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) } + #[cfg(feature = "hyper")] RemoteTx::HyperWebSocket { tx, format, .. } => { let data = match format { - FrameSerializationFormat::Bincode => { - bincode::serialize(&req).map_err(|err| { + crate::meta::FrameSerializationFormat::Bincode => bincode::serialize(&req) + .map_err(|err| { tracing::warn!("failed to serialize message - {err}"); NetworkError::IOError - })? - } + })?, format => { tracing::warn!("format not currently supported - {format:?}"); return Err(NetworkError::IOError); @@ -144,14 +147,14 @@ where .await .map_err(|_| NetworkError::ConnectionAborted) } + #[cfg(feature = "tokio-tungstenite")] RemoteTx::TokioWebSocket { tx, format, .. } => { let data = match format { - FrameSerializationFormat::Bincode => { - bincode::serialize(&req).map_err(|err| { + crate::meta::FrameSerializationFormat::Bincode => bincode::serialize(&req) + .map_err(|err| { tracing::warn!("failed to serialize message - {err}"); NetworkError::IOError - })? - } + })?, format => { tracing::warn!("format not currently supported - {format:?}"); return Err(NetworkError::IOError); @@ -207,6 +210,7 @@ where })?; Poll::Ready(Ok(())) } + #[cfg(feature = "hyper")] RemoteTx::HyperWebSocket { tx, format, @@ -231,12 +235,11 @@ where } let data = match format { - FrameSerializationFormat::Bincode => { - bincode::serialize(&req).map_err(|err| { + crate::meta::FrameSerializationFormat::Bincode => bincode::serialize(&req) + .map_err(|err| { tracing::warn!("failed to serialize message - {err}"); NetworkError::IOError - })? - } + })?, format => { tracing::warn!("format not currently supported - {format:?}"); return Poll::Ready(Err(NetworkError::IOError)); @@ -265,6 +268,7 @@ where })?; Poll::Ready(Ok(())) } + #[cfg(feature = "tokio-tungstenite")] RemoteTx::TokioWebSocket { tx, format, @@ -289,12 +293,11 @@ where } let data = match format { - FrameSerializationFormat::Bincode => { - bincode::serialize(&req).map_err(|err| { + crate::meta::FrameSerializationFormat::Bincode => bincode::serialize(&req) + .map_err(|err| { tracing::warn!("failed to serialize message - {err}"); NetworkError::IOError - })? - } + })?, format => { tracing::warn!("format not currently supported - {format:?}"); return Poll::Ready(Err(NetworkError::IOError)); @@ -377,16 +380,16 @@ where })?; Ok(()) } + #[cfg(feature = "hyper")] RemoteTx::HyperWebSocket { tx, format, work, .. } => { let data = match format { - FrameSerializationFormat::Bincode => { - bincode::serialize(&req).map_err(|err| { + crate::meta::FrameSerializationFormat::Bincode => bincode::serialize(&req) + .map_err(|err| { tracing::warn!("failed to serialize message - {err}"); NetworkError::IOError - })? - } + })?, format => { tracing::warn!("format not currently supported - {format:?}"); return Err(NetworkError::IOError); @@ -435,16 +438,16 @@ where })?; Ok(()) } + #[cfg(feature = "tokio-tungstenite")] RemoteTx::TokioWebSocket { tx, format, work, .. } => { let data = match format { - FrameSerializationFormat::Bincode => { - bincode::serialize(&req).map_err(|err| { + crate::meta::FrameSerializationFormat::Bincode => bincode::serialize(&req) + .map_err(|err| { tracing::warn!("failed to serialize message - {err}"); NetworkError::IOError - })? - } + })?, format => { tracing::warn!("format not currently supported - {format:?}"); return Err(NetworkError::IOError); @@ -508,17 +511,19 @@ where Stream { rx: Pin> + Send + 'static>>, }, + #[cfg(feature = "hyper")] HyperWebSocket { rx: futures_util::stream::SplitStream< hyper_tungstenite::WebSocketStream, >, - format: FrameSerializationFormat, + format: crate::meta::FrameSerializationFormat, }, + #[cfg(feature = "tokio-tungstenite")] TokioWebSocket { rx: futures_util::stream::SplitStream< tokio_tungstenite::WebSocketStream>, >, - format: FrameSerializationFormat, + format: crate::meta::FrameSerializationFormat, }, } impl RemoteRx @@ -544,10 +549,11 @@ where Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, }, + #[cfg(feature = "hyper")] RemoteRx::HyperWebSocket { rx, format } => match Pin::new(rx).poll_next(cx) { Poll::Ready(Some(Ok(hyper_tungstenite::tungstenite::Message::Binary(msg)))) => { match format { - FrameSerializationFormat::Bincode => { + crate::meta::FrameSerializationFormat::Bincode => { return match bincode::deserialize(&msg) { Ok(msg) => Poll::Ready(Some(msg)), Err(err) => { @@ -573,10 +579,11 @@ where Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, }, + #[cfg(feature = "tokio-tungstenite")] RemoteRx::TokioWebSocket { rx, format } => match Pin::new(rx).poll_next(cx) { Poll::Ready(Some(Ok(tokio_tungstenite::tungstenite::Message::Binary(msg)))) => { match format { - FrameSerializationFormat::Bincode => { + crate::meta::FrameSerializationFormat::Bincode => { return match bincode::deserialize(&msg) { Ok(msg) => Poll::Ready(Some(msg)), Err(err) => { diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index aadb7672773..f0db46e945f 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -6,7 +6,9 @@ use crate::{ }; use crate::{IpCidr, IpRoute, NetworkError, StreamSecurity, VirtualIcmpSocket}; use derivative::Derivative; -use futures_util::stream::{FuturesOrdered, SplitSink, SplitStream}; +use futures_util::stream::FuturesOrdered; +#[cfg(any(feature = "hyper", feature = "tokio-tungstenite"))] +use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{future::BoxFuture, StreamExt}; use futures_util::{Sink, Stream}; use std::collections::HashSet; diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index a0fe26aa589..0f106002fdc 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -491,6 +491,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea6672d73216c05740850c789368d371ca226dc8104d5f2e30c74252d5d6e5e" +[[package]] +name = "educe" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "079044df30bb07de7d846d41a184c4b00e66ebdac93ee459253474f3a47e50ae" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "enum-iterator" version = "0.7.0" @@ -511,6 +523,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "enum-ordinalize" +version = "3.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4f76552f53cefc9a7f64987c3701b99d982f7690606fd67de1d09712fbf52f1" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.27", +] + [[package]] name = "enumset" version = "1.1.2" @@ -1039,6 +1064,27 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7843ec2de400bcbc6a6328c958dc38e5359da6e93e72e37bc5246bf1ae776389" +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.16" @@ -1953,6 +1999,35 @@ dependencies = [ "syn 2.0.27", ] +[[package]] +name = "tokio-serde" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911a61637386b789af998ee23f50aa30d5fd7edcec8d6d3dedae5e5815205466" +dependencies = [ + "bincode", + "bytes", + "educe", + "futures-core", + "futures-sink", + "pin-project", + "serde", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.5.11" @@ -2246,9 +2321,13 @@ dependencies = [ "bytes", "derivative", "futures-util", + "libc", "pin-project-lite", "serde", "thiserror", + "tokio", + "tokio-serde", + "tokio-util", "tracing", "virtual-mio", ] @@ -2723,6 +2802,7 @@ dependencies = [ "tracing-subscriber 0.2.25", "tracing-wasm", "url", + "virtual-net", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", diff --git a/lib/wasi-web/Cargo.toml b/lib/wasi-web/Cargo.toml index 34a3b67b8b6..97b8ef9a03b 100644 --- a/lib/wasi-web/Cargo.toml +++ b/lib/wasi-web/Cargo.toml @@ -18,6 +18,7 @@ rust-version = "1.67" [dependencies] wasmer = { path = "../api", default_features = false, features = [ "js-default" ] } wasmer-wasix = { path = "../wasix", version = "0.10.0", default-features = false, features = [ "js-default" ] } +virtual-net = { path = "../virtual-net", default-features = false, features = [ "remote" ] } #wasm-bindgen = { version = "0.2", features = [ "nightly", "serde-serialize" ] } wasm-bindgen = { version = "0.2", features = [ "serde-serialize" ] } wasm-bindgen-futures = "0.4" diff --git a/lib/wasi-web/src/ws.rs b/lib/wasi-web/src/ws.rs index 138aef8367d..ee49074d573 100644 --- a/lib/wasi-web/src/ws.rs +++ b/lib/wasi-web/src/ws.rs @@ -12,7 +12,7 @@ pub struct WebSocket { } impl WebSocket { - pub fn new(url: &str) -> Result { + pub fn new(url: &str) -> Result, String> { // Open the web socket let ws_sys = WebSocketSys::new(url).map_err(|err| format!("{:?}", err))?; @@ -38,10 +38,7 @@ impl WebSocket { callback.forget(); } - fn set_onmessage( - &mut self, - callback: Box) + Send + 'static>, - ) { + fn set_onmessage(&mut self, callback: Box) + Send + 'static>) { let callback = Arc::new(callback); let fr = web_sys::FileReader::new().unwrap(); From 05151a96aba0ecd951223d692483a45c3615539b Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 31 Jul 2023 08:59:06 +1000 Subject: [PATCH 46/52] Added virtual networking to the browser --- lib/virtual-net/src/client.rs | 4 +- lib/virtual-net/src/meta.rs | 1 + lib/virtual-net/src/server.rs | 1 + lib/wasi-web/src/glue.rs | 23 ++++++- lib/wasi-web/src/lib.rs | 2 + lib/wasi-web/src/net.rs | 124 ++++++++++++++++++++++++++++++++++ lib/wasi-web/src/runtime.rs | 10 +-- lib/wasi-web/src/ws.rs | 12 ++-- lib/wasi-web/wasmer.toml | 2 +- 9 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 lib/wasi-web/src/net.rs diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index d75d54c9378..b69a50c4685 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -1129,7 +1129,7 @@ impl VirtualConnectionlessSocket for RemoteSocket { Ok((data, addr)) => { let amt = buf.len().min(data.len()); let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; - buf.copy_from_slice(&data[..amt]); + buf[..amt].copy_from_slice(&data[..amt]); Ok((amt, addr)) } Err(TryRecvError::Disconnected) => Err(NetworkError::ConnectionAborted), @@ -1297,7 +1297,7 @@ impl VirtualConnectedSocket for RemoteSocket { if self.rx_buffer.len() > 0 { let amt = self.rx_buffer.len().min(buf.len()); let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; - buf.copy_from_slice(&self.rx_buffer[..amt]); + buf[..amt].copy_from_slice(&self.rx_buffer[..amt]); self.rx_buffer.advance(amt); return Ok(amt); } diff --git a/lib/virtual-net/src/meta.rs b/lib/virtual-net/src/meta.rs index 4b5c99afcb1..c275254eeb8 100644 --- a/lib/virtual-net/src/meta.rs +++ b/lib/virtual-net/src/meta.rs @@ -283,6 +283,7 @@ pub enum MessageRequest { addr: SocketAddr, req_id: Option, }, + Reconnect, } /// Message sent by the server back to a client diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index f0db46e945f..cbd7f08c0f2 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -417,6 +417,7 @@ impl RemoteNetworkingServerDriver { req, req_id, } => self.process_socket(socket, req, req_id), + MessageRequest::Reconnect => None, } } diff --git a/lib/wasi-web/src/glue.rs b/lib/wasi-web/src/glue.rs index e725351e592..f690bbebf17 100644 --- a/lib/wasi-web/src/glue.rs +++ b/lib/wasi-web/src/glue.rs @@ -7,6 +7,7 @@ use std::{collections::HashMap, sync::Arc}; use tokio::sync::mpsc; #[allow(unused_imports, dead_code)] use tracing::{debug, error, info, trace, warn}; +use virtual_net::{UnsupportedVirtualNetworking, VirtualNetworking}; use wasm_bindgen::{prelude::*, JsCast}; use wasmer_wasix::{ capabilities::Capabilities, @@ -23,7 +24,10 @@ use xterm_js_rs::addons::webgl::WebglAddon; use xterm_js_rs::{LogLevel, OnKeyEvent, Terminal, TerminalOptions, Theme}; use super::{common::*, pool::*}; -use crate::runtime::{TermStdout, TerminalCommandRx, WebRuntime}; +use crate::{ + net::connect_networking, + runtime::{TermStdout, TerminalCommandRx, WebRuntime}, +}; #[macro_export] #[doc(hidden)] @@ -43,6 +47,7 @@ pub struct StartArgs { uses: Vec, prompt: Option, no_welcome: bool, + connect: Option, token: Option, } impl StartArgs { @@ -58,6 +63,9 @@ impl StartArgs { if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "prompt").next() { self.prompt = Some(val.to_string()); } + if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "connect").next() { + self.connect = Some(val.to_string()); + } if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "no_welcome").next() { match val.as_ref() { "true" | "yes" | "" => self.no_welcome = true, @@ -167,7 +175,18 @@ pub fn start(encoded_args: String) -> Result<(), JsValue> { let stdout = TermStdout::new(term_tx, tty_options.clone()); let stderr = stdout.clone(); - let runtime = Arc::new(WebRuntime::new(pool.clone(), tty_options.clone(), webgl2)); + let mut net: Arc = + Arc::new(UnsupportedVirtualNetworking::default()); + if let Some(connect) = args.connect { + net = Arc::new(connect_networking(connect)) + } + + let runtime = Arc::new(WebRuntime::new( + pool.clone(), + tty_options.clone(), + webgl2, + net, + )); let mut tty = Tty::new( Box::new(stdin_tx), Box::new(stdout.clone()), diff --git a/lib/wasi-web/src/lib.rs b/lib/wasi-web/src/lib.rs index 15eb0b235ed..f7407187173 100644 --- a/lib/wasi-web/src/lib.rs +++ b/lib/wasi-web/src/lib.rs @@ -2,7 +2,9 @@ mod common; mod glue; mod interval; mod module_cache; +mod net; mod pool; mod runtime; +mod ws; pub use glue::start; diff --git a/lib/wasi-web/src/net.rs b/lib/wasi-web/src/net.rs new file mode 100644 index 00000000000..32224daa9ce --- /dev/null +++ b/lib/wasi-web/src/net.rs @@ -0,0 +1,124 @@ +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use tokio::sync::mpsc; +use virtual_net::{meta::MessageRequest, RemoteNetworkingClient}; +use wasm_bindgen_futures::JsFuture; + +use crate::{runtime::bindgen_sleep, ws::WebSocket}; + +pub fn connect_networking(connect: String) -> RemoteNetworkingClient { + let (recv_tx, recv_rx) = mpsc::channel(100); + let (send_tx, send_rx) = mpsc::channel(100); + let send_tx2 = send_tx.clone(); + + let (client, driver) = virtual_net::RemoteNetworkingClient::new_from_mpsc(send_tx, recv_rx); + wasm_bindgen_futures::spawn_local(driver); + + let send_rx = Arc::new(tokio::sync::Mutex::new(send_rx)); + + wasm_bindgen_futures::spawn_local(async move { + let backoff = Arc::new(AtomicUsize::new(0)); + loop { + // Exponential backoff prevents thrashing of the connection + let backoff_ms = backoff.load(Ordering::SeqCst); + if backoff_ms > 0 { + let promise = bindgen_sleep(backoff_ms as i32); + JsFuture::from(promise).await.ok(); + } + let new_backoff = 8000usize.min((backoff_ms * 2) + 100); + backoff.store(new_backoff, Ordering::SeqCst); + + // Establish a websocket connection to the edge network + let mut ws = match WebSocket::new(connect.as_str()) { + Ok(ws) => ws, + Err(err) => { + tracing::error!("failed to establish web socket connection - {}", err); + continue; + } + }; + + // Wire up the events + let (relay_tx, mut relay_rx) = mpsc::unbounded_channel(); + let (connected_tx, mut connected_rx) = mpsc::unbounded_channel(); + ws.set_onopen({ + let connect = connect.clone(); + let connected_tx = connected_tx.clone(); + Box::new(move || { + tracing::debug!(url = connect, "networking web-socket opened"); + connected_tx.send(true).ok(); + }) + }); + ws.set_onclose({ + let connect = connect.clone(); + + let connected_tx = connected_tx.clone(); + let relay_tx = relay_tx.clone(); + Box::new(move || { + tracing::debug!(url = connect, "networking web-socket closed"); + relay_tx.send(Vec::new()).ok(); + connected_tx.send(false).ok(); + }) + }); + ws.set_onmessage({ + Box::new(move |data| { + relay_tx.send(data).unwrap(); + }) + }); + + // Wait for it to connect and setup the rest of the callbacks + if !connected_rx.recv().await.unwrap_or_default() { + continue; + } + backoff.store(100, Ordering::SeqCst); + + // We process any backends + wasm_bindgen_futures::spawn_local({ + let send_tx2 = send_tx2.clone(); + let recv_tx = recv_tx.clone(); + async move { + while let Some(data) = relay_rx.recv().await { + if data.is_empty() { + break; + } + let data = match bincode::deserialize(&data) { + Ok(d) => d, + Err(err) => { + tracing::error!( + "failed to deserialize networking message - {}", + err + ); + break; + } + }; + if recv_tx.send(data).await.is_err() { + break; + } + } + send_tx2.try_send(MessageRequest::Reconnect).ok(); + } + }); + + while let Some(data) = send_rx.lock().await.recv().await { + if let MessageRequest::Reconnect = &data { + tracing::info!("websocket will reconnect"); + break; + } + let data = match bincode::serialize(&data) { + Ok(d) => d, + Err(err) => { + tracing::error!("failed to serialize networking message - {}", err); + break; + } + }; + if let Err(err) = ws.send(data) { + tracing::error!("websocket has failed - {}", err); + break; + } + } + } + }); + client +} diff --git a/lib/wasi-web/src/runtime.rs b/lib/wasi-web/src/runtime.rs index 586633dde7f..202068dddd0 100644 --- a/lib/wasi-web/src/runtime.rs +++ b/lib/wasi-web/src/runtime.rs @@ -26,7 +26,7 @@ use wasmer_wasix::{ resolver::{Source, WapmSource}, task_manager::TaskWasm, }, - VirtualFile, VirtualNetworking, VirtualTaskManager, WasiThreadError, WasiTtyState, + VirtualFile, VirtualTaskManager, WasiThreadError, WasiTtyState, }; use web_sys::WebGl2RenderingContext; @@ -67,6 +67,7 @@ impl WebRuntime { pool: WebThreadPool, tty_options: TtyOptions, webgl2: WebGl2RenderingContext, + net: wasmer_wasix::virtual_net::DynVirtualNetworking, ) -> WebRuntime { #[cfg(feature = "webgl")] let webgl_tx = GlContext::init(webgl2); @@ -95,7 +96,7 @@ impl WebRuntime { #[cfg(feature = "webgl")] webgl_tx, http_client, - net: Arc::new(WebVirtualNetworking), + net, module_cache: Arc::new(module_cache), package_loader: Arc::new(package_loader), source: Arc::new(source), @@ -103,11 +104,6 @@ impl WebRuntime { } } -#[derive(Clone, Debug)] -struct WebVirtualNetworking; - -impl VirtualNetworking for WebVirtualNetworking {} - #[derive(Debug, Clone)] pub(crate) struct WebTaskManager { pool: WebThreadPool, diff --git a/lib/wasi-web/src/ws.rs b/lib/wasi-web/src/ws.rs index ee49074d573..28ed9c6bcd8 100644 --- a/lib/wasi-web/src/ws.rs +++ b/lib/wasi-web/src/ws.rs @@ -12,16 +12,16 @@ pub struct WebSocket { } impl WebSocket { - pub fn new(url: &str) -> Result, String> { + pub fn new(url: &str) -> Result { // Open the web socket let ws_sys = WebSocketSys::new(url).map_err(|err| format!("{:?}", err))?; - Ok(Box::new(Self { sys: ws_sys })) + Ok(Self { sys: ws_sys }) } } impl WebSocket { - fn set_onopen(&mut self, mut callback: Box) { + pub fn set_onopen(&mut self, mut callback: Box) { let callback = Closure::wrap(Box::new(move |_e: web_sys::ProgressEvent| { callback.deref_mut()(); }) as Box); @@ -29,7 +29,7 @@ impl WebSocket { callback.forget(); } - fn set_onclose(&mut self, callback: Box) { + pub fn set_onclose(&mut self, callback: Box) { let callback = Closure::wrap(Box::new(move |_e: web_sys::ProgressEvent| { callback.deref()(); }) as Box); @@ -38,7 +38,7 @@ impl WebSocket { callback.forget(); } - fn set_onmessage(&mut self, callback: Box) + Send + 'static>) { + pub fn set_onmessage(&mut self, callback: Box) + Send + 'static>) { let callback = Arc::new(callback); let fr = web_sys::FileReader::new().unwrap(); @@ -76,7 +76,7 @@ impl WebSocket { onmessage_callback.forget(); } - fn send(&mut self, data: Vec) -> Result<(), String> { + pub fn send(&mut self, data: Vec) -> Result<(), String> { let data_len = data.len(); let array = js_sys::Uint8Array::new_with_length(data_len as u32); array.copy_from(&data[..]); diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index 7edbc4df643..41ebf8ebad4 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = 'wasmer/wasmer-sh' -version = '0.2.16' +version = '0.2.17' description = 'Container that holds the wasmer.sh website.' [dependencies] From abdb3c0e2636310abe3924c32f308f5b0ec571c5 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 31 Jul 2023 12:49:17 +1000 Subject: [PATCH 47/52] Fixed a series of lint issues --- lib/virtual-net/src/client.rs | 154 ++++++++++++++++++---------------- lib/virtual-net/src/rx_tx.rs | 8 +- lib/virtual-net/src/server.rs | 151 ++++++++++++++------------------- 3 files changed, 153 insertions(+), 160 deletions(-) diff --git a/lib/virtual-net/src/client.rs b/lib/virtual-net/src/client.rs index b69a50c4685..375985f1c54 100644 --- a/lib/virtual-net/src/client.rs +++ b/lib/virtual-net/src/client.rs @@ -186,7 +186,7 @@ impl RemoteNetworkingClient { let tx = RemoteTx::Stream { tx: Arc::new(tokio::sync::Mutex::new(tx)), work: tx_work, - wakers: tx_wakers.clone(), + wakers: tx_wakers, }; let rx = RemoteRx::Stream { rx }; @@ -333,12 +333,10 @@ impl Future for RemoteNetworkingClientDriver { self.tasks.push_back(Box::pin(async move { tx.send(data).await.ok(); - common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Readable)); + if let Some(h) = common.handlers.lock().unwrap().get_mut(&socket_id) + { + h.interest(InterestType::Readable) + } })); } MessageResponse::RecvWithAddr { @@ -355,23 +353,20 @@ impl Future for RemoteNetworkingClientDriver { }; let common = self.common.clone(); self.tasks.push_back(Box::pin(async move { - tx.send((data, addr)).await.ok(); - - common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Readable)); + tx.send(DataWithAddr { data, addr }).await.ok(); + + if let Some(h) = common.handlers.lock().unwrap().get_mut(&socket_id) + { + h.interest(InterestType::Readable) + } })); } MessageResponse::Sent { socket_id, .. } => { - self.common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Writable)); + if let Some(h) = + self.common.handlers.lock().unwrap().get_mut(&socket_id) + { + h.interest(InterestType::Writable) + } } MessageResponse::SendError { socket_id, error, .. @@ -379,20 +374,18 @@ impl Future for RemoteNetworkingClientDriver { NetworkError::ConnectionAborted | NetworkError::ConnectionReset | NetworkError::BrokenPipe => { - self.common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Closed)); + if let Some(h) = + self.common.handlers.lock().unwrap().get_mut(&socket_id) + { + h.interest(InterestType::Closed) + } } _ => { - self.common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Writable)); + if let Some(h) = + self.common.handlers.lock().unwrap().get_mut(&socket_id) + { + h.interest(InterestType::Writable) + } } }, @@ -403,31 +396,28 @@ impl Future for RemoteNetworkingClientDriver { } => { let common = self.common.clone(); self.tasks.push_back(Box::pin(async move { - let tx = common - .accept_tx - .lock() - .unwrap() - .get(&socket_id) - .map(|tx| tx.clone()); + let tx = common.accept_tx.lock().unwrap().get(&socket_id).cloned(); if let Some(tx) = tx { - tx.send((child_id, addr)).await.ok(); + tx.send(SocketWithAddr { + socket: child_id, + addr, + }) + .await + .ok(); } - common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Readable)); + if let Some(h) = common.handlers.lock().unwrap().get_mut(&socket_id) + { + h.interest(InterestType::Readable) + } })); } MessageResponse::Closed { socket_id } => { - self.common - .handlers - .lock() - .unwrap() - .get_mut(&socket_id) - .map(|h| h.interest(InterestType::Closed)); + if let Some(h) = + self.common.handlers.lock().unwrap().get_mut(&socket_id) + { + h.interest(InterestType::Closed) + } } MessageResponse::ResponseToRequest { req_id, res } => { let mut requests = self.common.requests.lock().unwrap(); @@ -504,6 +494,18 @@ impl RequestTx { } } +#[derive(Debug)] +struct DataWithAddr { + pub data: Vec, + pub addr: SocketAddr, +} +#[derive(Debug)] +struct SocketWithAddr { + pub socket: SocketId, + pub addr: SocketAddr, +} +type SocketMap = HashMap; + #[derive(Derivative)] #[derivative(Debug)] struct RemoteCommon { @@ -514,11 +516,11 @@ struct RemoteCommon { request_seed: AtomicU64, requests: Mutex>, socket_seed: AtomicU64, - recv_tx: Mutex>>>, - recv_with_addr_tx: Mutex, SocketAddr)>>>, - accept_tx: Mutex>>, + recv_tx: Mutex>>>, + recv_with_addr_tx: Mutex>>, + accept_tx: Mutex>>, #[derivative(Debug = "ignore")] - handlers: Mutex>>, + handlers: Mutex>>, // The stall guard will prevent reads while its held and there are background tasks running // (the idea behind this is to create back pressure so that the task list infinitely grow) @@ -844,9 +846,9 @@ struct RemoteSocket { common: Arc, rx_buffer: BytesMut, rx_recv: mpsc::Receiver>, - rx_recv_with_addr: mpsc::Receiver<(Vec, SocketAddr)>, + rx_recv_with_addr: mpsc::Receiver, tx_waker: Waker, - rx_accept: mpsc::Receiver<(SocketId, SocketAddr)>, + rx_accept: mpsc::Receiver, pending_accept: Option, } impl Drop for RemoteSocket { @@ -968,7 +970,7 @@ impl VirtualTcpListener for RemoteSocket { fn try_accept(&mut self) -> Result<(Box, SocketAddr)> { self.touch_begin_accept()?; - let (id, addr) = self.rx_accept.try_recv().map_err(|err| match err { + let accepted = self.rx_accept.try_recv().map_err(|err| match err { TryRecvError::Empty => NetworkError::WouldBlock, TryRecvError::Disconnected => NetworkError::ConnectionAborted, })?; @@ -981,16 +983,28 @@ impl VirtualTcpListener for RemoteSocket { // Now we construct the child let (tx, rx_recv) = tokio::sync::mpsc::channel(100); - self.common.recv_tx.lock().unwrap().insert(id, tx); + self.common + .recv_tx + .lock() + .unwrap() + .insert(accepted.socket, tx); let (tx, rx_recv_with_addr) = tokio::sync::mpsc::channel(100); - self.common.recv_with_addr_tx.lock().unwrap().insert(id, tx); + self.common + .recv_with_addr_tx + .lock() + .unwrap() + .insert(accepted.socket, tx); let (tx, rx_accept) = tokio::sync::mpsc::channel(100); - self.common.accept_tx.lock().unwrap().insert(id, tx); + self.common + .accept_tx + .lock() + .unwrap() + .insert(accepted.socket, tx); let socket = RemoteSocket { - socket_id: id, + socket_id: accepted.socket, common: self.common.clone(), rx_buffer: BytesMut::new(), rx_recv, @@ -999,7 +1013,7 @@ impl VirtualTcpListener for RemoteSocket { pending_accept: None, tx_waker: TxWaker::new(&self.common).as_waker(), }; - Ok((Box::new(socket), addr)) + Ok((Box::new(socket), accepted.addr)) } fn set_handler( @@ -1071,7 +1085,7 @@ impl VirtualRawSocket for RemoteSocket { fn try_recv(&mut self, buf: &mut [std::mem::MaybeUninit]) -> Result { loop { - if self.rx_buffer.len() > 0 { + if !self.rx_buffer.is_empty() { let amt = self.rx_buffer.len().min(buf.len()); let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; buf[..amt].copy_from_slice(&self.rx_buffer[..amt]); @@ -1126,11 +1140,11 @@ impl VirtualConnectionlessSocket for RemoteSocket { buf: &mut [std::mem::MaybeUninit], ) -> Result<(usize, SocketAddr)> { match self.rx_recv_with_addr.try_recv() { - Ok((data, addr)) => { - let amt = buf.len().min(data.len()); + Ok(received) => { + let amt = buf.len().min(received.data.len()); let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; - buf[..amt].copy_from_slice(&data[..amt]); - Ok((amt, addr)) + buf[..amt].copy_from_slice(&received.data[..amt]); + Ok((amt, received.addr)) } Err(TryRecvError::Disconnected) => Err(NetworkError::ConnectionAborted), Err(TryRecvError::Empty) => Err(NetworkError::WouldBlock), @@ -1294,7 +1308,7 @@ impl VirtualConnectedSocket for RemoteSocket { fn try_recv(&mut self, buf: &mut [std::mem::MaybeUninit]) -> Result { loop { - if self.rx_buffer.len() > 0 { + if !self.rx_buffer.is_empty() { let amt = self.rx_buffer.len().min(buf.len()); let buf: &mut [u8] = unsafe { std::mem::transmute(buf) }; buf[..amt].copy_from_slice(&self.rx_buffer[..amt]); diff --git a/lib/virtual-net/src/rx_tx.rs b/lib/virtual-net/src/rx_tx.rs index c7a5d27022f..bb052f6de1d 100644 --- a/lib/virtual-net/src/rx_tx.rs +++ b/lib/virtual-net/src/rx_tx.rs @@ -28,7 +28,7 @@ pub(crate) struct RemoteTxWakers { impl RemoteTxWakers { pub fn add(&self, waker: &Waker) { let mut guard = self.wakers.lock().unwrap(); - if guard.iter().any(|w| w.will_wake(waker)) == false { + if !guard.iter().any(|w| w.will_wake(waker)) { guard.push(waker.clone()); } } @@ -58,6 +58,8 @@ impl AsyncWrite for FailOnWrite { } } +pub(crate) type StreamSink = Pin + Send + 'static>>; + pub(crate) enum RemoteTx where T: Serialize, @@ -68,7 +70,7 @@ where wakers: RemoteTxWakers, }, Stream { - tx: Arc + Send + 'static>>>>, + tx: Arc>>, work: mpsc::UnboundedSender>, wakers: RemoteTxWakers, }, @@ -127,7 +129,7 @@ where rx_done .await - .unwrap_or_else(|_| Err(NetworkError::ConnectionAborted)) + .unwrap_or(Err(NetworkError::ConnectionAborted)) } #[cfg(feature = "hyper")] RemoteTx::HyperWebSocket { tx, format, .. } => { diff --git a/lib/virtual-net/src/server.rs b/lib/virtual-net/src/server.rs index cbd7f08c0f2..840addac83a 100644 --- a/lib/virtual-net/src/server.rs +++ b/lib/virtual-net/src/server.rs @@ -322,7 +322,7 @@ impl Future for RemoteNetworkingServerDriver { // that it is woken when something is ready to read or write let readable = { let mut guard = self.common.handler.state.lock().unwrap(); - if guard.driver_wakers.iter().any(|w| w.will_wake(cx.waker())) == false { + if !guard.driver_wakers.iter().any(|w| w.will_wake(cx.waker())) { guard.driver_wakers.push(cx.waker().clone()); } guard.readable.drain().collect() @@ -448,9 +448,7 @@ impl RemoteNetworkingServerDriver { guard .get_mut(&socket_id) .map(|s| { - req_id - .map(|req_id| s.send_to(&self.common, socket_id, data, addr, req_id)) - .flatten() + req_id.and_then(|req_id| s.send_to(&self.common, socket_id, data, addr, req_id)) }) .unwrap_or(None) } @@ -483,14 +481,12 @@ impl RemoteNetworkingServerDriver { Self::process_async(async move { let future = work(inner); let ret = future.await; - req_id - .map(|req_id| { - common.send(MessageResponse::ResponseToRequest { - req_id, - res: transmute(ret), - }) + req_id.and_then(|req_id| { + common.send(MessageResponse::ResponseToRequest { + req_id, + res: transmute(ret), }) - .flatten() + }) }) } @@ -569,26 +565,22 @@ impl RemoteNetworkingServerDriver { let socket = match guard.get_mut(&socket_id) { Some(s) => s, None => { - return req_id - .map(|req_id| { - self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: ResponseType::Err(NetworkError::InvalidFd), - }) + return req_id.and_then(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::InvalidFd), }) - .flatten() + }) } }; work(socket) }; - req_id - .map(|req_id| { - self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: transmute(ret), - }) + req_id.and_then(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: transmute(ret), }) - .flatten() + }) } fn process_inner_noop( @@ -827,14 +819,12 @@ impl RemoteNetworkingServerDriver { }, req_id, ), - _ => req_id - .map(|req_id| { - self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: ResponseType::Err(NetworkError::Unsupported), - }) + _ => req_id.and_then(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::Unsupported), }) - .flatten(), + }), } } @@ -1180,14 +1170,12 @@ impl RemoteNetworkingServerDriver { socket_id, req_id, ), - _ => req_id - .map(|req_id| { - self.common.send(MessageResponse::ResponseToRequest { - req_id, - res: ResponseType::Err(NetworkError::Unsupported), - }) + _ => req_id.and_then(|req_id| { + self.common.send(MessageResponse::ResponseToRequest { + req_id, + res: ResponseType::Err(NetworkError::Unsupported), }) - .flatten(), + }), } } } @@ -1247,43 +1235,41 @@ impl RemoteAdapterSocket { // We make sure the waker is registered with the interest driver which will // wake up this poller when there is writeability let mut guard = self.common.handler.state.lock().unwrap(); - if guard.driver_wakers.iter().any(|w| w.will_wake(cx.waker())) - == false - { + if !guard.driver_wakers.iter().any(|w| w.will_wake(cx.waker())) { guard.driver_wakers.push(cx.waker().clone()); } drop(guard); let mut guard = self.common.sockets.lock().unwrap(); - if let Some(socket) = guard.get_mut(&self.socket_id) { - if let RemoteAdapterSocket::TcpSocket(socket) = socket { - match socket.try_send(&self.data) { - Ok(amount) => { - if let Some(req_id) = self.req_id { - return Poll::Ready(self.common.send( - MessageResponse::Sent { - socket_id: self.socket_id, - req_id, - amount: amount as u64, - }, - )); - } else { - return Poll::Ready(None); - } + if let Some(RemoteAdapterSocket::TcpSocket(socket)) = + guard.get_mut(&self.socket_id) + { + match socket.try_send(&self.data) { + Ok(amount) => { + if let Some(req_id) = self.req_id { + return Poll::Ready(self.common.send( + MessageResponse::Sent { + socket_id: self.socket_id, + req_id, + amount: amount as u64, + }, + )); + } else { + return Poll::Ready(None); } - Err(NetworkError::WouldBlock) => return Poll::Pending, - Err(error) => { - if let Some(req_id) = self.req_id { - return Poll::Ready(self.common.send( - MessageResponse::SendError { - socket_id: self.socket_id, - req_id, - error, - }, - )); - } else { - return Poll::Ready(None); - } + } + Err(NetworkError::WouldBlock) => return Poll::Pending, + Err(error) => { + if let Some(req_id) = self.req_id { + return Poll::Ready(self.common.send( + MessageResponse::SendError { + socket_id: self.socket_id, + req_id, + error, + }, + )); + } else { + return Poll::Ready(None); } } } @@ -1534,19 +1520,11 @@ struct RemoteAdapterHandlerState { driver_wakers: Vec, } -#[derive(Debug, Clone)] +#[derive(Debug, Default, Clone)] struct RemoteAdapterHandler { socket_id: Option, state: Arc>, } -impl Default for RemoteAdapterHandler { - fn default() -> Self { - Self { - socket_id: None, - state: Default::default(), - } - } -} impl RemoteAdapterHandler { pub fn for_socket(self, id: SocketId) -> Self { Self { @@ -1559,19 +1537,18 @@ impl InterestHandler for RemoteAdapterHandler { fn interest(&mut self, interest: virtual_mio::InterestType) { let mut guard = self.state.lock().unwrap(); guard.driver_wakers.drain(..).for_each(|w| w.wake()); - let socket_id = match self.socket_id.clone() { + let socket_id = match self.socket_id { Some(s) => s, None => return, }; - match interest { - virtual_mio::InterestType::Readable => { - guard.readable.insert(socket_id); - } - _ => {} + if interest == virtual_mio::InterestType::Readable { + guard.readable.insert(socket_id); } } } +type SocketMap = HashMap; + #[derive(Derivative)] #[derivative(Debug)] struct RemoteAdapterCommon { @@ -1580,8 +1557,8 @@ struct RemoteAdapterCommon { #[derivative(Debug = "ignore")] rx: Mutex>, #[derivative(Debug = "ignore")] - sockets: Mutex>, - socket_accept: Mutex>, + sockets: Mutex>, + socket_accept: Mutex>, handler: RemoteAdapterHandler, // The stall guard will prevent reads while its held and there are background tasks running From d229abe9b5884d8165265fa36394e8d133cccaed Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 31 Jul 2023 17:28:09 +1000 Subject: [PATCH 48/52] Remove python from the main package --- lib/wasi-web/public/init.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/wasi-web/public/init.json b/lib/wasi-web/public/init.json index 3de504ac1ca..209b2f34782 100644 --- a/lib/wasi-web/public/init.json +++ b/lib/wasi-web/public/init.json @@ -2,8 +2,7 @@ "init": "sharrattj/bash", "uses": [ "john-sharratt/catsay", - "python/python" ], "prompt": "wasmer.sh", "no_welcome": false -} \ No newline at end of file +} From f8a781f9e23321cb2ad80d685e92543b5cff9791 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 31 Jul 2023 18:36:13 +1000 Subject: [PATCH 49/52] Updates to the wasi-web package --- lib/wasi-web/app.yaml | 3 ++- lib/wasi-web/cfg/config.toml | 6 +++--- lib/wasi-web/public/init.json | 2 +- lib/wasi-web/wasmer.toml | 12 ++++++------ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/wasi-web/app.yaml b/lib/wasi-web/app.yaml index 03104bbb4fe..23ec6404a2a 100644 --- a/lib/wasi-web/app.yaml +++ b/lib/wasi-web/app.yaml @@ -1,4 +1,5 @@ --- kind: wasmer.io/App.v0 name: wasmer-sh -package: wasmer/wasmer-sh +app_id: da_AKo6IotJUgY2 +package: wasmer/wasmer-sh@0.2.21 diff --git a/lib/wasi-web/cfg/config.toml b/lib/wasi-web/cfg/config.toml index f922e474006..874f73b900e 100644 --- a/lib/wasi-web/cfg/config.toml +++ b/lib/wasi-web/cfg/config.toml @@ -1,12 +1,12 @@ [general] host = "::" -port = 80 +port = 8080 root = "/public" log-level = "info" cache-control-headers = false compression = true -threads-multiplier = 16 -max-blocking-threads = 32 +threads-multiplier = 8 +max-blocking-threads = 16 [advanced] diff --git a/lib/wasi-web/public/init.json b/lib/wasi-web/public/init.json index 209b2f34782..2a440f4895d 100644 --- a/lib/wasi-web/public/init.json +++ b/lib/wasi-web/public/init.json @@ -1,7 +1,7 @@ { "init": "sharrattj/bash", "uses": [ - "john-sharratt/catsay", + "john-sharratt/catsay" ], "prompt": "wasmer.sh", "no_welcome": false diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index 41ebf8ebad4..fc727480dd1 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,11 +1,11 @@ [package] -name = 'wasmer/wasmer-sh' -version = '0.2.17' -description = 'Container that holds the wasmer.sh website.' +name = "wasmer/wasmer-sh" +version = "0.2.21" +description = "Container that holds the wasmer.sh website." [dependencies] -"wasmer/static-web-server" = '1' +"wasmer/static-web-server-async" = "1" [fs] -public = 'dist' -cfg = 'cfg' +public = "dist" +cfg = "cfg" From a225f478fd9d95649a99d055e3068ca1da00bace Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 1 Aug 2023 03:29:11 +1000 Subject: [PATCH 50/52] Better error message when including broken packages --- lib/wasi-web/wasmer.toml | 4 ++-- lib/wasix/src/state/env.rs | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index fc727480dd1..a89c6a8c325 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,10 +1,10 @@ [package] name = "wasmer/wasmer-sh" -version = "0.2.21" +version = "0.2.24" description = "Container that holds the wasmer.sh website." [dependencies] -"wasmer/static-web-server-async" = "1" +"wasmer/static-web-server" = "1" [fs] public = "dist" diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index b8a895302fb..727a2ee007b 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -947,11 +947,20 @@ impl WasiEnv { let rt = self.runtime(); for package_name in uses { - let specifier = package_name - .parse::() - .map_err(|e| WasiStateCreationError::WasiIncludePackageError(e.to_string()))?; - let pkg = InlineWaker::block_on(BinaryPackage::from_registry(&specifier, rt)) - .map_err(|e| WasiStateCreationError::WasiIncludePackageError(e.to_string()))?; + let specifier = package_name.parse::().map_err(|e| { + WasiStateCreationError::WasiIncludePackageError(format!( + "package_name={package_name}, {}", + e.to_string() + )) + })?; + let pkg = InlineWaker::block_on(BinaryPackage::from_registry(&specifier, rt)).map_err( + |e| { + WasiStateCreationError::WasiIncludePackageError(format!( + "package_name={package_name}, {}", + e.to_string() + )) + }, + )?; self.use_package(&pkg)?; } From fab2a0e4cfce70ef5da5e89a4c1e155f4b0d121f Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Tue, 1 Aug 2023 03:42:37 +1000 Subject: [PATCH 51/52] Fixed an issue where undefined uses field would break wasmer-sh --- lib/wasi-web/src/glue.rs | 20 ++++++++++++++------ lib/wasi-web/wasmer.toml | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/wasi-web/src/glue.rs b/lib/wasi-web/src/glue.rs index f690bbebf17..16b6e2ca7bd 100644 --- a/lib/wasi-web/src/glue.rs +++ b/lib/wasi-web/src/glue.rs @@ -54,25 +54,33 @@ impl StartArgs { pub fn parse(mut self, args: &str) -> Self { let query_pairs = || form_urlencoded::parse(args.as_bytes()); - if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "init").next() { + let find = |key| { + query_pairs() + .filter(|(k, _)| k == key) + .map(|a| a.1) + .filter(|a| a != "undefined") + .next() + }; + + if let Some(val) = find("init") { self.init = Some(val.to_string()); } - if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "uses").next() { + if let Some(val) = find("uses") { self.uses = val.split(",").map(|v| v.to_string()).collect(); } - if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "prompt").next() { + if let Some(val) = find("prompt") { self.prompt = Some(val.to_string()); } - if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "connect").next() { + if let Some(val) = find("connect") { self.connect = Some(val.to_string()); } - if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "no_welcome").next() { + if let Some(val) = find("no_welcome") { match val.as_ref() { "true" | "yes" | "" => self.no_welcome = true, _ => {} } } - if let Some((_, val)) = query_pairs().filter(|(k, _)| k == "token").next() { + if let Some(val) = find("token") { self.token = Some(val.to_string()); } self diff --git a/lib/wasi-web/wasmer.toml b/lib/wasi-web/wasmer.toml index a89c6a8c325..a58e58b7e56 100644 --- a/lib/wasi-web/wasmer.toml +++ b/lib/wasi-web/wasmer.toml @@ -1,6 +1,6 @@ [package] name = "wasmer/wasmer-sh" -version = "0.2.24" +version = "0.2.26" description = "Container that holds the wasmer.sh website." [dependencies] From b7f455756cea713cf8b66167962f4d8ae8abdf73 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Tue, 1 Aug 2023 16:52:12 +0200 Subject: [PATCH 52/52] Fixed linter --- lib/wasix/src/state/env.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 727a2ee007b..d0dbd1652c6 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -950,14 +950,14 @@ impl WasiEnv { let specifier = package_name.parse::().map_err(|e| { WasiStateCreationError::WasiIncludePackageError(format!( "package_name={package_name}, {}", - e.to_string() + e )) })?; let pkg = InlineWaker::block_on(BinaryPackage::from_registry(&specifier, rt)).map_err( |e| { WasiStateCreationError::WasiIncludePackageError(format!( "package_name={package_name}, {}", - e.to_string() + e )) }, )?;