From 518026e79e940a32c5120efa429cc74e80455e0e Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 25 Apr 2024 15:53:14 +0200 Subject: [PATCH 01/55] feat(wasix): Improve SpawnError information Extend SpawnError to add more context and information for some errors that happen during instance spawning. Makes debugging and error reporting a lot better. --- lib/wasix/src/bin_factory/exec.rs | 71 +++++++------------------------ lib/wasix/src/bin_factory/mod.rs | 53 ++++++++++++++++++++++- lib/wasix/src/lib.rs | 61 ++++++++++++++++++++++++-- lib/wasix/src/runtime/mod.rs | 17 +++++--- 4 files changed, 135 insertions(+), 67 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index 6d5db50c289..ee5fa44ab8c 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -1,4 +1,4 @@ -use std::{pin::Pin, sync::Arc}; +use std::sync::Arc; use crate::{ os::task::{ @@ -14,12 +14,11 @@ use crate::{ syscalls::rewind_ext, RewindState, SpawnError, WasiError, WasiRuntimeError, }; -use futures::Future; use tracing::*; -use wasmer::{Function, FunctionEnvMut, Memory32, Memory64, Module, Store}; +use wasmer::{Function, Memory32, Memory64, Module, Store}; use wasmer_wasix_types::wasi::Errno; -use super::{BinFactory, BinaryPackage}; +use super::BinaryPackage; use crate::{Runtime, WasiEnv, WasiFunctionEnv}; #[tracing::instrument(level = "trace", skip_all, fields(%name, package_id=%binary.id))] @@ -59,7 +58,9 @@ pub async fn spawn_load_wasm<'a>( "Unable to spawn a command because its package has no entrypoint", ); env.on_exit(Some(Errno::Noexec.into())).await; - return Err(SpawnError::CompileError); + return Err(SpawnError::MissingEntrypoint { + package_id: binary.id.clone(), + }); }; Ok(wasm) } @@ -70,19 +71,18 @@ pub async fn spawn_load_module( wasm: &[u8], runtime: &Arc, ) -> Result { - let module = match runtime.load_module(wasm).await { - Ok(module) => module, + match runtime.load_module(wasm).await { + Ok(module) => Ok(module), Err(err) => { tracing::error!( command = name, - error = &*err, + error = &err as &dyn std::error::Error, "Failed to compile the module", ); env.on_exit(Some(Errno::Noexec.into())).await; - return Err(SpawnError::CompileError); + Err(err) } - }; - Ok(module) + } } pub async fn spawn_union_fs(env: &WasiEnv, binary: &BinaryPackage) -> Result<(), SpawnError> { @@ -93,7 +93,10 @@ pub async fn spawn_union_fs(env: &WasiEnv, binary: &BinaryPackage) -> Result<(), .await .map_err(|err| { tracing::warn!("failed to union file system - {err}"); - SpawnError::FileSystemError + SpawnError::FileSystemError(crate::ExtendedFsError::with_msg( + err, + "could not union filesystems", + )) })?; tracing::debug!("{:?}", env.state.fs); Ok(()) @@ -326,47 +329,3 @@ fn call_module( debug!("wasi[{pid}]::main() has exited with {code}"); handle.thread.set_status_finished(ret.map(|a| a.into())); } - -impl BinFactory { - pub fn spawn<'a>( - &'a self, - name: String, - store: Store, - env: WasiEnv, - ) -> Pin> + 'a>> { - Box::pin(async move { - // Find the binary (or die trying) and make the spawn type - let binary = self - .get_binary(name.as_str(), Some(env.fs_root())) - .await - .ok_or(SpawnError::NotFound); - if binary.is_err() { - env.on_exit(Some(Errno::Noent.into())).await; - } - let binary = binary?; - - // Execute - spawn_exec(binary, name.as_str(), store, env, &self.runtime).await - }) - } - - pub fn try_built_in( - &self, - name: String, - parent_ctx: Option<&FunctionEnvMut<'_, WasiEnv>>, - store: &mut Option, - builder: &mut Option, - ) -> Result { - // We check for built in commands - if let Some(parent_ctx) = parent_ctx { - if self.commands.exists(name.as_str()) { - return self - .commands - .exec(parent_ctx, name.as_str(), store, builder); - } - } else if self.commands.exists(name.as_str()) { - tracing::warn!("builtin command without a parent ctx - {}", name); - } - Err(SpawnError::NotFound) - } -} diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index 26ee4a8fd23..16311cd4fad 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -1,12 +1,16 @@ use std::{ collections::HashMap, + future::Future, ops::Deref, path::Path, + pin::Pin, sync::{Arc, RwLock}, }; use anyhow::Context; use virtual_fs::{AsyncReadExt, FileSystem}; +use wasmer::FunctionEnvMut; +use wasmer_wasix_types::wasi::Errno; use webc::Container; mod binary_package; @@ -18,7 +22,10 @@ pub use self::{ run_exec, spawn_exec, spawn_exec_module, spawn_load_module, spawn_load_wasm, spawn_union_fs, }, }; -use crate::{os::command::Commands, Runtime}; +use crate::{ + os::{command::Commands, task::TaskJoinHandle}, + Runtime, SpawnError, WasiEnv, +}; #[derive(Debug, Clone)] pub struct BinFactory { @@ -94,6 +101,50 @@ impl BinFactory { cache.insert(name, None); None } + + pub fn spawn<'a>( + &'a self, + name: String, + store: wasmer::Store, + env: WasiEnv, + ) -> Pin> + 'a>> { + Box::pin(async move { + // Find the binary (or die trying) and make the spawn type + let res = self + .get_binary(name.as_str(), Some(env.fs_root())) + .await + .ok_or_else(|| SpawnError::BinaryNotFound { + binary: name.clone(), + }); + if res.is_err() { + env.on_exit(Some(Errno::Noent.into())).await; + } + let binary = res?; + + // Execute + spawn_exec(binary, name.as_str(), store, env, &self.runtime).await + }) + } + + pub fn try_built_in( + &self, + name: String, + parent_ctx: Option<&FunctionEnvMut<'_, WasiEnv>>, + store: &mut Option, + builder: &mut Option, + ) -> Result { + // We check for built in commands + if let Some(parent_ctx) = parent_ctx { + if self.commands.exists(name.as_str()) { + return self + .commands + .exec(parent_ctx, name.as_str(), store, builder); + } + } else if self.commands.exists(name.as_str()) { + tracing::warn!("builtin command without a parent ctx - {}", name); + } + Err(SpawnError::BinaryNotFound { binary: name }) + } } async fn load_package_from_filesystem( diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 258fd19541e..16441e41889 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -137,9 +137,14 @@ pub enum SpawnError { /// Failed to fetch the Wasmer process #[error("fetch failed")] FetchFailed, + #[error(transparent)] + CacheError(crate::runtime::module_cache::CacheError), /// Failed to compile the Wasmer process #[error("compile error")] - CompileError, + CompileError { + module_hash: crate::runtime::module_cache::ModuleHash, + error: wasmer::CompileError, + }, /// Invalid ABI #[error("Wasmer process has an invalid ABI")] InvalidABI, @@ -152,6 +157,16 @@ pub enum SpawnError { /// Not found #[error("not found")] NotFound, + /// Tried to run the specified binary as a new WASI thread/process, but + /// the binary name was not found. + #[error("could not find binary '{binary}'")] + BinaryNotFound { binary: String }, + #[error("could not find an entrypoint in the package '{package_id}'")] + MissingEntrypoint { + package_id: wasmer_config::package::PackageId, + }, + #[error("could not load ")] + ModuleLoad { message: String }, /// Bad request #[error("bad request")] BadRequest, @@ -162,8 +177,8 @@ pub enum SpawnError { #[error("internal error")] InternalError, /// An error occurred while preparing the file system - #[error("file system error")] - FileSystemError, + #[error(transparent)] + FileSystemError(ExtendedFsError), /// Memory allocation failed #[error("memory allocation failed")] MemoryAllocationFailed, @@ -179,6 +194,46 @@ pub enum SpawnError { Other(#[from] Box), } +#[derive(Debug)] +pub struct ExtendedFsError { + pub error: virtual_fs::FsError, + pub message: Option, +} + +impl ExtendedFsError { + pub fn with_msg(error: virtual_fs::FsError, msg: impl Into) -> Self { + Self { + error, + message: Some(msg.into()), + } + } + + pub fn new(error: virtual_fs::FsError) -> Self { + Self { + error, + message: None, + } + } +} + +impl std::fmt::Display for ExtendedFsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + write!(f, "fs error: {}", self.error)?; + + if let Some(msg) = &self.message { + write!(f, " | {}", msg)?; + } + + Ok(()) + } +} + +impl std::error::Error for ExtendedFsError { + fn cause(&self) -> Option<&dyn std::error::Error> { + Some(&self.error) + } +} + impl SpawnError { /// Returns `true` if the spawn error is [`NotFound`]. /// diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 20e7cb8e57e..517b2faa2bd 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -31,7 +31,7 @@ use crate::{ package_loader::{PackageLoader, UnsupportedPackageLoader}, resolver::{MultiSource, Source, WapmSource}, }, - WasiTtyState, + SpawnError, WasiTtyState, }; #[derive(Clone)] @@ -101,7 +101,7 @@ where } /// Load a a Webassembly module, trying to use a pre-compiled version if possible. - fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, anyhow::Result> { + fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, Result> { let engine = self.engine(); let module_cache = self.module_cache(); let hash = ModuleHash::hash(wasm); @@ -114,7 +114,7 @@ where /// Load a a Webassembly module, trying to use a pre-compiled version if possible. /// /// Non-async version of [`Self::load_module`]. - fn load_module_sync(&self, wasm: &[u8]) -> Result { + fn load_module_sync(&self, wasm: &[u8]) -> Result { InlineWaker::block_on(self.load_module(wasm)) } @@ -152,7 +152,7 @@ pub async fn load_module( module_cache: &(dyn ModuleCache + Send + Sync), wasm: &[u8], wasm_hash: ModuleHash, -) -> Result { +) -> Result { let result = module_cache.load(wasm_hash, engine).await; match result { @@ -167,7 +167,10 @@ pub async fn load_module( } } - let module = Module::new(&engine, wasm)?; + let module = Module::new(&engine, wasm).map_err(|err| crate::SpawnError::CompileError { + module_hash: wasm_hash, + error: err, + })?; if let Err(e) = module_cache.save(wasm_hash, engine, &module).await { tracing::warn!( @@ -546,7 +549,7 @@ impl Runtime for OverriddenRuntime { } } - fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, anyhow::Result> { + fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, Result> { if self.engine.is_some() || self.module_cache.is_some() { let engine = self.engine(); let module_cache = self.module_cache(); @@ -559,7 +562,7 @@ impl Runtime for OverriddenRuntime { } } - fn load_module_sync(&self, wasm: &[u8]) -> Result { + fn load_module_sync(&self, wasm: &[u8]) -> Result { if self.engine.is_some() || self.module_cache.is_some() { InlineWaker::block_on(self.load_module(wasm)) } else { From 9585cf5c502baa741e43bc1022b9c6c50f653bc3 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 25 Apr 2024 15:55:19 +0200 Subject: [PATCH 02/55] chore: Bump wasmer-wasix to 0.19.0 --- lib/wasix/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 29d6fcf5545..6efa5a1a971 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wasmer-wasix" -version = "0.18.4" +version = "0.19.0" description = "WASI and WASIX implementation library for Wasmer WebAssembly runtime" categories = ["wasm", "os"] keywords = ["wasm", "webassembly", "wasi", "sandbox", "ABI"] From c267b8046b046e6a1b60ced53a03f9536b48e38c Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 25 Apr 2024 15:59:12 +0200 Subject: [PATCH 03/55] deps: Lift wasmer-wasix to be a workspace dependency --- Cargo.lock | 2 +- Cargo.toml | 6 +++++- lib/c-api/Cargo.toml | 2 +- lib/cli/Cargo.toml | 2 +- lib/sys-utils/Cargo.toml | 2 +- tests/lib/wast/Cargo.toml | 5 +++-- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4dc7c00bc81..9d6e327bed3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6852,7 +6852,7 @@ dependencies = [ [[package]] name = "wasmer-wasix" -version = "0.18.4" +version = "0.19.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index fa699fc4537..961363f30e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ rust-version.workspace = true version.workspace = true [dependencies] +# Repo-local crates wasmer-config = { path = "./lib/config" } wasmer = { version = "=4.3.0-alpha.1", path = "lib/api", default-features = false } wasmer-compiler = { version = "=4.3.0-alpha.1", path = "lib/compiler", features = [ @@ -21,12 +22,14 @@ wasmer-compiler-cranelift = { version = "=4.3.0-alpha.1", path = "lib/compiler-c wasmer-compiler-singlepass = { version = "=4.3.0-alpha.1", path = "lib/compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=4.3.0-alpha.1", path = "lib/compiler-llvm", optional = true } wasmer-emscripten = { version = "=4.3.0-alpha.1", path = "lib/emscripten", optional = true } -wasmer-wasix = { version = "0.18.4", path = "lib/wasix", optional = true } +wasmer-wasix = { path = "lib/wasix", optional = true } wasmer-wast = { version = "=4.3.0-alpha.1", path = "tests/lib/wast", optional = true } wasi-test-generator = { version = "=4.3.0-alpha.1", path = "tests/wasi-wast", optional = true } wasmer-cache = { version = "=4.3.0-alpha.1", path = "lib/cache", optional = true } wasmer-types = { version = "=4.3.0-alpha.1", path = "lib/types" } wasmer-middlewares = { version = "=4.3.0-alpha.1", path = "lib/middlewares", optional = true } + +# Third party dependencies cfg-if = "1.0" tokio = { version = "1", features = [ "rt", @@ -88,6 +91,7 @@ version = "4.3.0-alpha.1" [workspace.dependencies] # Repo-local crates wasmer-config = { path = "./lib/config" } +wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates webc = { version = "6.0.0-alpha3", default-features = false, features = ["package"] } diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index f1b0c9ba0a6..64d7a425358 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -32,7 +32,7 @@ wasmer-compiler-singlepass = { version = "=4.3.0-alpha.1", path = "../compiler-s wasmer-emscripten = { version = "=4.3.0-alpha.1", path = "../emscripten", optional = true } wasmer-middlewares = { version = "=4.3.0-alpha.1", path = "../middlewares", optional = true } wasmer-types = { version = "=4.3.0-alpha.1", path = "../types" } -wasmer-wasix = { version = "0.18.4", path = "../wasix", features = ["host-fs", "host-vnet"], optional = true } +wasmer-wasix = { workspace = true, features = ["host-fs", "host-vnet"], optional = true } webc = { workspace = true, optional = true } virtual-fs = { version = "0.11.3", path = "../virtual-fs", optional = true, default-features = false, features = ["static-fs"] } enumset.workspace = true diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index c2fc0979c09..3f5488eb811 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -76,7 +76,7 @@ wasmer-compiler-singlepass = { version = "=4.3.0-alpha.1", path = "../compiler-s wasmer-compiler-llvm = { version = "=4.3.0-alpha.1", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=4.3.0-alpha.1", path = "../emscripten" } wasmer-vm = { version = "=4.3.0-alpha.1", path = "../vm", optional = true } -wasmer-wasix = { version = "0.18.4", path = "../wasix", features = [ +wasmer-wasix = { workspace = true, features = [ "logging", "webc_runner_rt_wcgi", "webc_runner_rt_dcgi", diff --git a/lib/sys-utils/Cargo.toml b/lib/sys-utils/Cargo.toml index 7f889343d8b..46f5c079cd5 100644 --- a/lib/sys-utils/Cargo.toml +++ b/lib/sys-utils/Cargo.toml @@ -22,7 +22,7 @@ tracing = "0.1.37" libc = { version = "^0.2", default-features = false } [dev-dependencies] -wasmer-wasix = { path = "../wasix", version = "0.18.4" } +wasmer-wasix.workspace = true wasmer = { path = "../api", version = "=4.3.0-alpha.1", default-features = false, features = ["sys", "compiler", "cranelift"] } tracing-subscriber = { version = "0.3.16", features = ["fmt"] } tracing = "0.1.37" diff --git a/tests/lib/wast/Cargo.toml b/tests/lib/wast/Cargo.toml index 05888a01c3f..d3f5d6274a4 100644 --- a/tests/lib/wast/Cargo.toml +++ b/tests/lib/wast/Cargo.toml @@ -11,10 +11,11 @@ readme = "README.md" edition = "2018" [dependencies] -anyhow = "1.0" +wasmer-wasix.workspace = true wasmer = { path = "../../../lib/api", version = "=4.3.0-alpha.1", default-features = false } -wasmer-wasix = { path = "../../../lib/wasix", version = "0.18.4" } virtual-fs = { path = "../../../lib/virtual-fs", version = "0.11.3" } + +anyhow = "1.0" wast = "38.0" serde = "1" tempfile = "3.6.0" From 1aa88f260fbb91656ab0fd0de57ad3757322ab1a Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Thu, 25 Apr 2024 21:39:30 +0330 Subject: [PATCH 04/55] propagate timestamps to virtual file handles --- lib/virtual-fs/src/lib.rs | 6 +++++ lib/virtual-fs/src/mem_fs/file.rs | 24 +++++++++++++++++++ .../syscalls/wasi/fd_filestat_set_times.rs | 23 ++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index 7fb2343490a..7dd966dfe0e 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -336,6 +336,12 @@ pub trait VirtualFile: /// the time at which the file was created in nanoseconds as a UNIX timestamp fn created_time(&self) -> u64; + /// sets accessed time + fn set_accessed(&mut self, _atime: u64) {} + + /// sets modification time + fn set_modified(&mut self, _mtime: u64) {} + /// the size of the file in bytes fn size(&self) -> u64; diff --git a/lib/virtual-fs/src/mem_fs/file.rs b/lib/virtual-fs/src/mem_fs/file.rs index 1a4336553f8..06be1a8ca62 100644 --- a/lib/virtual-fs/src/mem_fs/file.rs +++ b/lib/virtual-fs/src/mem_fs/file.rs @@ -144,6 +144,30 @@ impl VirtualFile for FileHandle { node.metadata().created } + fn set_accessed(&mut self, atime: u64) { + let mut fs = match self.filesystem.inner.write() { + Ok(fs) => fs, + _ => return, + }; + + let inode = fs.storage.get_mut(self.inode); + if let Some(node) = inode { + node.metadata_mut().accessed = atime; + } + } + + fn set_modified(&mut self, mtime: u64) { + let mut fs = match self.filesystem.inner.write() { + Ok(fs) => fs, + _ => return, + }; + + let inode = fs.storage.get_mut(self.inode); + if let Some(node) = inode { + node.metadata_mut().modified = mtime; + } + } + fn size(&self) -> u64 { let fs = match self.filesystem.inner.read() { Ok(fs) => fs, diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs index df7db200bc1..a6ebeddbf98 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs @@ -1,3 +1,5 @@ +use std::borrow::BorrowMut; + use super::*; use crate::syscalls::*; @@ -59,6 +61,9 @@ pub(crate) fn fd_filestat_set_times_internal( let inode = fd_entry.inode; + let mut atime = None; + let mut mtime = None; + if fst_flags.contains(Fstflags::SET_ATIM) || fst_flags.contains(Fstflags::SET_ATIM_NOW) { let time_to_set = if fst_flags.contains(Fstflags::SET_ATIM) { st_atim @@ -66,6 +71,7 @@ pub(crate) fn fd_filestat_set_times_internal( get_current_time_in_nanos()? }; inode.stat.write().unwrap().st_atim = time_to_set; + atime = Some(time_to_set); } if fst_flags.contains(Fstflags::SET_MTIM) || fst_flags.contains(Fstflags::SET_MTIM_NOW) { @@ -75,6 +81,23 @@ pub(crate) fn fd_filestat_set_times_internal( get_current_time_in_nanos()? }; inode.stat.write().unwrap().st_mtim = time_to_set; + mtime = Some(time_to_set); + } + + if let Kind::File { + handle: Some(handle), + .. + } = inode.kind.write().unwrap().deref() + { + let mut handle = handle.write().unwrap(); + + if let Some(time) = atime { + handle.set_accessed(time); + } + + if let Some(time) = mtime { + handle.set_modified(time); + } } Ok(()) From f60ae5da13541e241879263f0e2eaae2496acb2d Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Tue, 16 Apr 2024 05:03:56 +0200 Subject: [PATCH 05/55] feat(backend-api): Add permalink to DeployApp --- lib/backend-api/src/types.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/backend-api/src/types.rs b/lib/backend-api/src/types.rs index 6c44f7a205e..d7097f3a604 100644 --- a/lib/backend-api/src/types.rs +++ b/lib/backend-api/src/types.rs @@ -530,6 +530,7 @@ mod queries { pub admin_url: String, pub owner: Owner, pub url: String, + pub permalink: String, pub deleted: bool, pub aliases: AppAliasConnection, } From fb3f48778b384bce0a1c8636726d9cbceb2fb640 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Thu, 25 Apr 2024 23:57:48 +0200 Subject: [PATCH 06/55] feat(wasix): More improvements to spawn error propagation --- lib/wasix/src/lib.rs | 6 +++--- lib/wasix/src/os/console/mod.rs | 4 +++- lib/wasix/src/syscalls/mod.rs | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 16441e41889..53c61cb5ed4 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -155,8 +155,8 @@ pub enum SpawnError { #[error("unsupported")] Unsupported, /// Not found - #[error("not found")] - NotFound, + #[error("not found: {message}")] + NotFound { message: String }, /// Tried to run the specified binary as a new WASI thread/process, but /// the binary name was not found. #[error("could not find binary '{binary}'")] @@ -240,7 +240,7 @@ impl SpawnError { /// [`NotFound`]: SpawnError::NotFound #[must_use] pub fn is_not_found(&self) -> bool { - matches!(self, Self::NotFound) + matches!(self, Self::NotFound { .. } | Self::MissingEntrypoint { .. }) } } diff --git a/lib/wasix/src/os/console/mod.rs b/lib/wasix/src/os/console/mod.rs index fda705314b1..67b1f167c1a 100644 --- a/lib/wasix/src/os/console/mod.rs +++ b/lib/wasix/src/os/console/mod.rs @@ -202,7 +202,9 @@ impl Console { .ok(); }); tracing::debug!(error=?e, %webc, "failed to get webc dependency"); - return Err(SpawnError::NotFound); + return Err(SpawnError::NotFound { + message: e.to_string(), + }); } }; diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 0374b9d8b8a..2cd424ea9a0 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -1507,8 +1507,8 @@ pub(crate) fn _prepare_wasi(wasi_env: &mut WasiEnv, args: Option>) { pub(crate) fn conv_spawn_err_to_errno(err: &SpawnError) -> Errno { match err { SpawnError::AccessDenied => Errno::Access, - SpawnError::NotFound => Errno::Noent, SpawnError::Unsupported => Errno::Noexec, + _ if err.is_not_found() => Errno::Noent, _ => Errno::Inval, } } From 6049fc725e62b6b9d5584fe704b617946f843e38 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 26 Apr 2024 11:55:36 +0330 Subject: [PATCH 07/55] consolidate two methods into one --- lib/virtual-fs/src/lib.rs | 7 ++----- lib/virtual-fs/src/mem_fs/file.rs | 21 +++++++------------ .../syscalls/wasi/fd_filestat_set_times.rs | 8 +------ 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index 7dd966dfe0e..e2660bea0b4 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -336,11 +336,8 @@ pub trait VirtualFile: /// the time at which the file was created in nanoseconds as a UNIX timestamp fn created_time(&self) -> u64; - /// sets accessed time - fn set_accessed(&mut self, _atime: u64) {} - - /// sets modification time - fn set_modified(&mut self, _mtime: u64) {} + /// sets accessed and modified time + fn set_times(&mut self, _atime: Option, _mtime: Option) {} /// the size of the file in bytes fn size(&self) -> u64; diff --git a/lib/virtual-fs/src/mem_fs/file.rs b/lib/virtual-fs/src/mem_fs/file.rs index 06be1a8ca62..cc8e1fa09dd 100644 --- a/lib/virtual-fs/src/mem_fs/file.rs +++ b/lib/virtual-fs/src/mem_fs/file.rs @@ -144,7 +144,7 @@ impl VirtualFile for FileHandle { node.metadata().created } - fn set_accessed(&mut self, atime: u64) { + fn set_times(&mut self, atime: Option, mtime: Option) { let mut fs = match self.filesystem.inner.write() { Ok(fs) => fs, _ => return, @@ -152,19 +152,12 @@ impl VirtualFile for FileHandle { let inode = fs.storage.get_mut(self.inode); if let Some(node) = inode { - node.metadata_mut().accessed = atime; - } - } - - fn set_modified(&mut self, mtime: u64) { - let mut fs = match self.filesystem.inner.write() { - Ok(fs) => fs, - _ => return, - }; - - let inode = fs.storage.get_mut(self.inode); - if let Some(node) = inode { - node.metadata_mut().modified = mtime; + if let Some(atime) = atime { + node.metadata_mut().accessed = atime; + } + if let Some(mtime) = mtime { + node.metadata_mut().modified = mtime; + } } } diff --git a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs index a6ebeddbf98..9add80b56d3 100644 --- a/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs +++ b/lib/wasix/src/syscalls/wasi/fd_filestat_set_times.rs @@ -91,13 +91,7 @@ pub(crate) fn fd_filestat_set_times_internal( { let mut handle = handle.write().unwrap(); - if let Some(time) = atime { - handle.set_accessed(time); - } - - if let Some(time) = mtime { - handle.set_modified(time); - } + handle.set_times(atime, mtime); } Ok(()) From 047a6103284535c20a88dfc6065ed922d0863a4e Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 26 Apr 2024 12:14:56 +0330 Subject: [PATCH 08/55] implement set_times for more VirtualFile implementors --- lib/virtual-fs/src/arc_box_file.rs | 4 ++++ lib/virtual-fs/src/arc_file.rs | 4 ++++ lib/virtual-fs/src/combine_file.rs | 4 ++++ lib/virtual-fs/src/cow_file.rs | 8 ++++++++ lib/virtual-fs/src/dual_write_file.rs | 4 ++++ lib/virtual-fs/src/host_fs.rs | 5 +++++ lib/virtual-fs/src/lib.rs | 3 ++- lib/virtual-fs/src/trace_fs.rs | 5 +++++ lib/wasix/src/fs/inode_guard.rs | 7 +++++++ 9 files changed, 43 insertions(+), 1 deletion(-) diff --git a/lib/virtual-fs/src/arc_box_file.rs b/lib/virtual-fs/src/arc_box_file.rs index d518c7069ec..f649b54d95d 100644 --- a/lib/virtual-fs/src/arc_box_file.rs +++ b/lib/virtual-fs/src/arc_box_file.rs @@ -102,6 +102,10 @@ impl VirtualFile for ArcBoxFile { let inner = self.inner.lock().unwrap(); inner.created_time() } + fn set_times(&mut self, atime: Option, mtime: Option) { + let mut inner = self.inner.lock().unwrap(); + inner.set_times(atime, mtime); + } fn size(&self) -> u64 { let inner = self.inner.lock().unwrap(); inner.size() diff --git a/lib/virtual-fs/src/arc_file.rs b/lib/virtual-fs/src/arc_file.rs index 4e980e02033..b66d02edecb 100644 --- a/lib/virtual-fs/src/arc_file.rs +++ b/lib/virtual-fs/src/arc_file.rs @@ -118,6 +118,10 @@ where let inner = self.inner.lock().unwrap(); inner.created_time() } + fn set_times(&mut self, atime: Option, mtime: Option) { + let mut inner = self.inner.lock().unwrap(); + inner.set_times(atime, mtime); + } fn size(&self) -> u64 { let inner = self.inner.lock().unwrap(); inner.size() diff --git a/lib/virtual-fs/src/combine_file.rs b/lib/virtual-fs/src/combine_file.rs index f3db42ca58c..65fc4afd8c9 100644 --- a/lib/virtual-fs/src/combine_file.rs +++ b/lib/virtual-fs/src/combine_file.rs @@ -33,6 +33,10 @@ impl VirtualFile for CombineFile { self.tx.created_time() } + fn set_times(&mut self, atime: Option, mtime: Option) { + self.tx.set_times(atime, mtime) + } + fn size(&self) -> u64 { self.rx.size() } diff --git a/lib/virtual-fs/src/cow_file.rs b/lib/virtual-fs/src/cow_file.rs index 4030981854e..4c885ccf4b1 100644 --- a/lib/virtual-fs/src/cow_file.rs +++ b/lib/virtual-fs/src/cow_file.rs @@ -195,6 +195,14 @@ impl VirtualFile for CopyOnWriteFile { fn created_time(&self) -> u64 { self.created_time } + fn set_times(&mut self, atime: Option, mtime: Option) { + if let Some(atime) = atime { + self.last_accessed = atime; + } + if let Some(mtime) = mtime { + self.last_modified = mtime; + } + } fn size(&self) -> u64 { match self.state.as_ref() { Some(inner) => inner.size(), diff --git a/lib/virtual-fs/src/dual_write_file.rs b/lib/virtual-fs/src/dual_write_file.rs index ef27a3f89d1..3e990577433 100644 --- a/lib/virtual-fs/src/dual_write_file.rs +++ b/lib/virtual-fs/src/dual_write_file.rs @@ -41,6 +41,10 @@ impl VirtualFile for DualWriteFile { self.inner.created_time() } + fn set_times(&mut self, atime: Option, mtime: Option) { + self.inner.set_times(atime, mtime) + } + fn size(&self) -> u64 { self.inner.size() } diff --git a/lib/virtual-fs/src/host_fs.rs b/lib/virtual-fs/src/host_fs.rs index bac84f944b0..1b4e71372ae 100644 --- a/lib/virtual-fs/src/host_fs.rs +++ b/lib/virtual-fs/src/host_fs.rs @@ -434,6 +434,11 @@ impl VirtualFile for File { .unwrap_or(0) } + // FIXME: Need to think about this + fn set_times(&mut self, _atime: Option, _mtime: Option) { + todo!() + } + fn size(&self) -> u64 { self.metadata().len() } diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index e2660bea0b4..10c2e79f0ed 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -336,8 +336,9 @@ pub trait VirtualFile: /// the time at which the file was created in nanoseconds as a UNIX timestamp fn created_time(&self) -> u64; + #[allow(unused_variables)] /// sets accessed and modified time - fn set_times(&mut self, _atime: Option, _mtime: Option) {} + fn set_times(&mut self, atime: Option, mtime: Option) {} /// the size of the file in bytes fn size(&self) -> u64; diff --git a/lib/virtual-fs/src/trace_fs.rs b/lib/virtual-fs/src/trace_fs.rs index 461b1cbf4fe..b7fff9096a1 100644 --- a/lib/virtual-fs/src/trace_fs.rs +++ b/lib/virtual-fs/src/trace_fs.rs @@ -119,6 +119,11 @@ impl VirtualFile for TraceFile { self.file.created_time() } + #[tracing::instrument(level = "trace", skip(self), fields(path=%self.path.display()))] + fn set_times(&mut self, atime: Option, mtime: Option) { + self.file.set_times(atime, mtime) + } + #[tracing::instrument(level = "trace", skip(self), fields(path=%self.path.display()))] fn size(&self) -> u64 { self.file.size() diff --git a/lib/wasix/src/fs/inode_guard.rs b/lib/wasix/src/fs/inode_guard.rs index c944b9d0700..15a2d6e5b34 100644 --- a/lib/wasix/src/fs/inode_guard.rs +++ b/lib/wasix/src/fs/inode_guard.rs @@ -510,6 +510,13 @@ impl VirtualFile for WasiStateFileGuard { } } + fn set_times(&mut self, atime: Option, mtime: Option) { + let mut guard = self.lock_write(); + if let Some(file) = guard.as_mut() { + file.set_times(atime, mtime) + } + } + fn size(&self) -> u64 { let guard = self.lock_read(); if let Some(file) = guard.as_ref() { From 0d7ad765cc01b46777c50e9faadf95f53022d658 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Fri, 26 Apr 2024 14:27:33 +0200 Subject: [PATCH 09/55] Improve error message for unknown atoms --- lib/wasix/src/runtime/package_loader/load_package_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 9bcc7598939..7e910c28360 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -150,7 +150,7 @@ fn load_binary_command( let atom = atom.with_context(|| { format!( - "The '{command_name}' command uses the '{atom_name}' atom, but it isn't present in the WEBC file" + "The '{command_name}' command uses the '{atom_name}' atom, but it isn't present in the WEBC file (package: {package_id})" ) })?; From 1ae237ff74cc0e3f727715c71db78dc2b98cfda7 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Fri, 26 Apr 2024 14:31:12 +0200 Subject: [PATCH 10/55] Improve error message Co-authored-by: Syrus Akbary --- lib/wasix/src/runtime/package_loader/load_package_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 7e910c28360..b35c726437f 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -150,7 +150,7 @@ fn load_binary_command( let atom = atom.with_context(|| { format!( - "The '{command_name}' command uses the '{atom_name}' atom, but it isn't present in the WEBC file (package: {package_id})" + "The '{command_name}' command uses the '{atom_name}' atom, but it isn't present in the package {package_id}" ) })?; From c47dd42f335d7c491c041de9aaca28201bb41b95 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 26 Apr 2024 22:03:49 +0330 Subject: [PATCH 11/55] add impl for host_fs --- lib/virtual-fs/src/arc_box_file.rs | 4 ++-- lib/virtual-fs/src/arc_file.rs | 4 ++-- lib/virtual-fs/src/combine_file.rs | 2 +- lib/virtual-fs/src/cow_file.rs | 4 +++- lib/virtual-fs/src/dual_write_file.rs | 2 +- lib/virtual-fs/src/host_fs.rs | 9 ++++++--- lib/virtual-fs/src/lib.rs | 4 +++- lib/virtual-fs/src/mem_fs/file.rs | 8 ++++++-- lib/virtual-fs/src/trace_fs.rs | 2 +- lib/wasix/src/fs/inode_guard.rs | 8 +++++++- 10 files changed, 32 insertions(+), 15 deletions(-) diff --git a/lib/virtual-fs/src/arc_box_file.rs b/lib/virtual-fs/src/arc_box_file.rs index f649b54d95d..b55a8b415e6 100644 --- a/lib/virtual-fs/src/arc_box_file.rs +++ b/lib/virtual-fs/src/arc_box_file.rs @@ -102,9 +102,9 @@ impl VirtualFile for ArcBoxFile { let inner = self.inner.lock().unwrap(); inner.created_time() } - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { let mut inner = self.inner.lock().unwrap(); - inner.set_times(atime, mtime); + inner.set_times(atime, mtime) } fn size(&self) -> u64 { let inner = self.inner.lock().unwrap(); diff --git a/lib/virtual-fs/src/arc_file.rs b/lib/virtual-fs/src/arc_file.rs index b66d02edecb..088f6e5ce02 100644 --- a/lib/virtual-fs/src/arc_file.rs +++ b/lib/virtual-fs/src/arc_file.rs @@ -118,9 +118,9 @@ where let inner = self.inner.lock().unwrap(); inner.created_time() } - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { let mut inner = self.inner.lock().unwrap(); - inner.set_times(atime, mtime); + inner.set_times(atime, mtime) } fn size(&self) -> u64 { let inner = self.inner.lock().unwrap(); diff --git a/lib/virtual-fs/src/combine_file.rs b/lib/virtual-fs/src/combine_file.rs index 65fc4afd8c9..951de2e5a2f 100644 --- a/lib/virtual-fs/src/combine_file.rs +++ b/lib/virtual-fs/src/combine_file.rs @@ -33,7 +33,7 @@ impl VirtualFile for CombineFile { self.tx.created_time() } - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { self.tx.set_times(atime, mtime) } diff --git a/lib/virtual-fs/src/cow_file.rs b/lib/virtual-fs/src/cow_file.rs index 4c885ccf4b1..643dc2a0554 100644 --- a/lib/virtual-fs/src/cow_file.rs +++ b/lib/virtual-fs/src/cow_file.rs @@ -195,13 +195,15 @@ impl VirtualFile for CopyOnWriteFile { fn created_time(&self) -> u64 { self.created_time } - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { if let Some(atime) = atime { self.last_accessed = atime; } if let Some(mtime) = mtime { self.last_modified = mtime; } + + Ok(()) } fn size(&self) -> u64 { match self.state.as_ref() { diff --git a/lib/virtual-fs/src/dual_write_file.rs b/lib/virtual-fs/src/dual_write_file.rs index 3e990577433..bf71ad41e86 100644 --- a/lib/virtual-fs/src/dual_write_file.rs +++ b/lib/virtual-fs/src/dual_write_file.rs @@ -41,7 +41,7 @@ impl VirtualFile for DualWriteFile { self.inner.created_time() } - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { self.inner.set_times(atime, mtime) } diff --git a/lib/virtual-fs/src/host_fs.rs b/lib/virtual-fs/src/host_fs.rs index 1b4e71372ae..43048b2dc21 100644 --- a/lib/virtual-fs/src/host_fs.rs +++ b/lib/virtual-fs/src/host_fs.rs @@ -434,9 +434,12 @@ impl VirtualFile for File { .unwrap_or(0) } - // FIXME: Need to think about this - fn set_times(&mut self, _atime: Option, _mtime: Option) { - todo!() + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { + let atime = atime.map(|t| filetime::FileTime::from_unix_time(t as i64, 0)); + let mtime = mtime.map(|t| filetime::FileTime::from_unix_time(t as i64, 0)); + + filetime::set_file_handle_times(&self.inner_std, atime, mtime) + .map_err(|_| crate::FsError::IOError) } fn size(&self) -> u64 { diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index 10c2e79f0ed..f3a4f66d2a0 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -338,7 +338,9 @@ pub trait VirtualFile: #[allow(unused_variables)] /// sets accessed and modified time - fn set_times(&mut self, atime: Option, mtime: Option) {} + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { + Ok(()) + } /// the size of the file in bytes fn size(&self) -> u64; diff --git a/lib/virtual-fs/src/mem_fs/file.rs b/lib/virtual-fs/src/mem_fs/file.rs index cc8e1fa09dd..41d753ac730 100644 --- a/lib/virtual-fs/src/mem_fs/file.rs +++ b/lib/virtual-fs/src/mem_fs/file.rs @@ -144,10 +144,10 @@ impl VirtualFile for FileHandle { node.metadata().created } - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { let mut fs = match self.filesystem.inner.write() { Ok(fs) => fs, - _ => return, + _ => return Err(crate::FsError::Lock), }; let inode = fs.storage.get_mut(self.inode); @@ -158,7 +158,11 @@ impl VirtualFile for FileHandle { if let Some(mtime) = mtime { node.metadata_mut().modified = mtime; } + + return Ok(()); } + + return Err(crate::FsError::UnknownError); } fn size(&self) -> u64 { diff --git a/lib/virtual-fs/src/trace_fs.rs b/lib/virtual-fs/src/trace_fs.rs index b7fff9096a1..86abc0687bd 100644 --- a/lib/virtual-fs/src/trace_fs.rs +++ b/lib/virtual-fs/src/trace_fs.rs @@ -120,7 +120,7 @@ impl VirtualFile for TraceFile { } #[tracing::instrument(level = "trace", skip(self), fields(path=%self.path.display()))] - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times(&mut self, atime: Option, mtime: Option) -> crate::Result<()> { self.file.set_times(atime, mtime) } diff --git a/lib/wasix/src/fs/inode_guard.rs b/lib/wasix/src/fs/inode_guard.rs index 15a2d6e5b34..aca0e161c24 100644 --- a/lib/wasix/src/fs/inode_guard.rs +++ b/lib/wasix/src/fs/inode_guard.rs @@ -510,10 +510,16 @@ impl VirtualFile for WasiStateFileGuard { } } - fn set_times(&mut self, atime: Option, mtime: Option) { + fn set_times( + &mut self, + atime: Option, + mtime: Option, + ) -> Result<(), virtual_fs::FsError> { let mut guard = self.lock_write(); if let Some(file) = guard.as_mut() { file.set_times(atime, mtime) + } else { + Err(crate::FsError::Lock) } } From f96650b9fde8fc88e0195990795bfb7d0c7eb129 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 26 Apr 2024 22:14:17 +0330 Subject: [PATCH 12/55] fix lint --- lib/virtual-fs/src/mem_fs/file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/virtual-fs/src/mem_fs/file.rs b/lib/virtual-fs/src/mem_fs/file.rs index 41d753ac730..77a9dedf8c9 100644 --- a/lib/virtual-fs/src/mem_fs/file.rs +++ b/lib/virtual-fs/src/mem_fs/file.rs @@ -162,7 +162,7 @@ impl VirtualFile for FileHandle { return Ok(()); } - return Err(crate::FsError::UnknownError); + Err(crate::FsError::UnknownError) } fn size(&self) -> u64 { From 6074551219f2a86a0bb73b384cd7fec431e4146c Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 29 Apr 2024 01:45:17 +0330 Subject: [PATCH 13/55] respect http_proxy env var --- lib/wasix/src/http/reqwest.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/wasix/src/http/reqwest.rs b/lib/wasix/src/http/reqwest.rs index 036c1dda38f..cb84dc4267d 100644 --- a/lib/wasix/src/http/reqwest.rs +++ b/lib/wasix/src/http/reqwest.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{env, time::Duration}; use anyhow::Context; use futures::{future::BoxFuture, TryStreamExt}; @@ -44,10 +44,14 @@ impl ReqwestHttpClient { // TODO: use persistent client? let client = { let _guard = Handle::try_current().map_err(|_| self.handle.enter()); - reqwest::Client::builder() - .connect_timeout(self.connect_timeout) - .build() - .context("Could not create reqwest client")? + let mut builder = reqwest::Client::builder().connect_timeout(self.connect_timeout); + + let proxy = env::var("http_proxy").or_else(|_| env::var("HTTP_PROXY")); + if let Ok(scheme) = proxy { + builder = builder.proxy(reqwest::Proxy::all(scheme)?); + } + + builder.build().context("Could not create reqwest client")? }; let mut builder = client.request(method, request.url.as_str()); From ea29f1ea56c586632f42de91ee1148a81d3b1889 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 29 Apr 2024 01:59:15 +0330 Subject: [PATCH 14/55] expose modified time in webc_volume_fs --- lib/virtual-fs/src/webc_volume_fs.rs | 29 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/lib/virtual-fs/src/webc_volume_fs.rs b/lib/virtual-fs/src/webc_volume_fs.rs index d3681d91b3c..bc7076d4454 100644 --- a/lib/virtual-fs/src/webc_volume_fs.rs +++ b/lib/virtual-fs/src/webc_volume_fs.rs @@ -162,18 +162,21 @@ impl FileOpener for WebcVolumeFileSystem { } } - match self.volume().metadata(path) { - Some(m) if m.is_file() => {} + let timestamps = match self.volume().metadata(path) { + Some(m) if m.is_file() => m.timestamps(), Some(_) => return Err(FsError::NotAFile), None if conf.create() || conf.create_new() => { // The file would normally be created, but we are a readonly fs. return Err(FsError::PermissionDenied); } None => return Err(FsError::EntryNotFound), - } + }; match self.volume().read_file(path) { - Some((bytes, _)) => Ok(Box::new(File(Cursor::new(bytes)))), + Some((bytes, _)) => Ok(Box::new(File { + timestamps, + content: Cursor::new(bytes), + })), None => { // The metadata() call should guarantee this, so something // probably went wrong internally @@ -184,7 +187,10 @@ impl FileOpener for WebcVolumeFileSystem { } #[derive(Debug, Clone, PartialEq)] -struct File(Cursor); +struct File { + timestamps: Option, + content: Cursor, +} impl VirtualFile for File { fn last_accessed(&self) -> u64 { @@ -192,7 +198,7 @@ impl VirtualFile for File { } fn last_modified(&self) -> u64 { - 0 + self.timestamps.map(|t| t.modified()).unwrap_or(0) } fn created_time(&self) -> u64 { @@ -200,7 +206,7 @@ impl VirtualFile for File { } fn size(&self) -> u64 { - self.0.get_ref().len().try_into().unwrap() + self.content.get_ref().len().try_into().unwrap() } fn set_len(&mut self, _new_size: u64) -> crate::Result<()> { @@ -215,7 +221,8 @@ impl VirtualFile for File { self: Pin<&mut Self>, _cx: &mut std::task::Context<'_>, ) -> Poll> { - let bytes_remaining = self.0.get_ref().len() - usize::try_from(self.0.position()).unwrap(); + let bytes_remaining = + self.content.get_ref().len() - usize::try_from(self.content.position()).unwrap(); Poll::Ready(Ok(bytes_remaining)) } @@ -233,20 +240,20 @@ impl AsyncRead for File { cx: &mut std::task::Context<'_>, buf: &mut tokio::io::ReadBuf<'_>, ) -> Poll> { - AsyncRead::poll_read(Pin::new(&mut self.0), cx, buf) + AsyncRead::poll_read(Pin::new(&mut self.content), cx, buf) } } impl AsyncSeek for File { fn start_seek(mut self: Pin<&mut Self>, position: std::io::SeekFrom) -> std::io::Result<()> { - AsyncSeek::start_seek(Pin::new(&mut self.0), position) + AsyncSeek::start_seek(Pin::new(&mut self.content), position) } fn poll_complete( mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, ) -> Poll> { - AsyncSeek::poll_complete(Pin::new(&mut self.0), cx) + AsyncSeek::poll_complete(Pin::new(&mut self.content), cx) } } From a57cbb50e62ce17ca9628d1d993f47b70abd3239 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Fri, 26 Apr 2024 19:53:06 +0200 Subject: [PATCH 15/55] fix(CLI): Pass the `no_wait` flag to `Deploy` This small patch also changes a couple names of CLI arguments to make them more natural to the end user or reflect the internal names (e.g. `autobump` -> `bump`). --- lib/cli/src/commands/app/create.rs | 6 +++--- lib/cli/src/commands/app/deploy.rs | 8 ++++---- lib/cli/src/commands/publish.rs | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/cli/src/commands/app/create.rs b/lib/cli/src/commands/app/create.rs index 4af09cfa5ac..37a64dac050 100644 --- a/lib/cli/src/commands/app/create.rs +++ b/lib/cli/src/commands/app/create.rs @@ -47,7 +47,7 @@ pub struct CmdAppCreate { /// /// If selected, this might entail the step of publishing the package related to the /// application. By default, the application is not deployed and the package is not published. - #[clap(long)] + #[clap(long = "deploy")] pub deploy_app: bool, /// Skip local schema validation. @@ -371,12 +371,12 @@ impl CmdAppCreate { non_interactive: self.non_interactive, publish_package: true, path: self.app_dir_path.clone(), - no_wait: false, + no_wait: self.no_wait, no_default: false, no_persist_id: false, owner: Some(String::from(owner)), app_name: None, - autobump: false, + bump: false, }; cmd_deploy.run_async().await?; } diff --git a/lib/cli/src/commands/app/deploy.rs b/lib/cli/src/commands/app/deploy.rs index 57a926b33a5..d5e3a422fd4 100644 --- a/lib/cli/src/commands/app/deploy.rs +++ b/lib/cli/src/commands/app/deploy.rs @@ -77,9 +77,9 @@ pub struct CmdAppDeploy { #[clap(long)] pub app_name: Option, - /// Whether or not to autobump the package version if publishing. + /// Whether or not to automatically bump the package version if publishing. #[clap(long)] - pub autobump: bool, + pub bump: bool, } impl CmdAppDeploy { @@ -123,7 +123,7 @@ impl CmdAppDeploy { None => Some(owner), }, non_interactive: self.non_interactive, - autobump: self.autobump, + bump: self.bump, }; match publish_cmd.run_async().await? { @@ -186,7 +186,7 @@ impl CmdAppDeploy { offline: false, owner: None, app_name: None, - no_wait: false, + no_wait: self.no_wait, api: self.api.clone(), fmt: ItemFormatOpts { format: self.fmt.format, diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index a1e746eb375..7c47fafa8a7 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -55,7 +55,7 @@ pub struct Publish { /// Whether or not the patch field of the version of the package - if any - should be bumped. #[clap(long)] - pub autobump: bool, + pub bump: bool, /// Do not prompt for user input. #[clap(long, default_value_t = !std::io::stdin().is_terminal())] @@ -139,7 +139,7 @@ impl AsyncCliCommand for Publish { }; if pkg.version < latest_version { - if self.autobump { + if self.bump { latest_version.patch += 1; version = Some(latest_version); } else if interactive { From b3247b5e66943e05413a5c31e80508f78976d98e Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 29 Apr 2024 15:17:31 +0330 Subject: [PATCH 16/55] add support for multiple hash algos for ModuleHash --- lib/cli/src/commands/run/mod.rs | 6 +- lib/wasix/src/bin_factory/binary_package.rs | 7 +- lib/wasix/src/os/task/control_plane.rs | 8 +- lib/wasix/src/runtime/mod.rs | 6 +- .../src/runtime/module_cache/fallback.rs | 6 +- .../src/runtime/module_cache/filesystem.rs | 10 +- lib/wasix/src/runtime/module_cache/shared.rs | 2 +- .../src/runtime/module_cache/thread_local.rs | 2 +- lib/wasix/src/runtime/module_cache/types.rs | 111 +++++++++++++++--- lib/wasix/src/state/builder.rs | 10 +- 10 files changed, 123 insertions(+), 45 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 684ca900948..db33994ee66 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -705,9 +705,10 @@ impl ExecutableTarget { .load_module_sync(&wasm) .with_context(|| format!("Unable to compile \"{}\"", path.display()))?; + // FIXME: Amin Ok(ExecutableTarget::WebAssembly { module, - module_hash: ModuleHash::hash(&wasm), + module_hash: ModuleHash::xxhash(&wasm), path: path.to_path_buf(), }) } @@ -717,7 +718,8 @@ impl ExecutableTarget { let module = unsafe { Module::deserialize_from_file(&engine, path)? }; let module_hash = { let wasm = std::fs::read(path)?; - ModuleHash::hash(wasm) + // FIXME: Amin + ModuleHash::xxhash(wasm) }; Ok(ExecutableTarget::WebAssembly { diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 80907d665b3..2f7cfc0be91 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -15,6 +15,7 @@ use crate::{ Runtime, }; +// FIXME: Amin #[derive(Derivative, Clone)] #[derivative(Debug)] pub struct BinaryPackageCommand { @@ -52,7 +53,7 @@ impl BinaryPackageCommand { } pub fn hash(&self) -> &ModuleHash { - self.hash.get_or_init(|| ModuleHash::hash(self.atom())) + self.hash.get_or_init(|| ModuleHash::xxhash(self.atom())) } } @@ -146,9 +147,9 @@ impl BinaryPackage { pub fn hash(&self) -> ModuleHash { *self.hash.get_or_init(|| { if let Some(entry) = self.entrypoint_bytes() { - ModuleHash::hash(entry) + ModuleHash::xxhash(entry) } else { - ModuleHash::hash(self.id.to_string()) + ModuleHash::xxhash(self.id.to_string()) } }) } diff --git a/lib/wasix/src/os/task/control_plane.rs b/lib/wasix/src/os/task/control_plane.rs index 8a27069289f..b7826a058b2 100644 --- a/lib/wasix/src/os/task/control_plane.rs +++ b/lib/wasix/src/os/task/control_plane.rs @@ -225,7 +225,7 @@ mod tests { enable_exponential_cpu_backoff: None, }); - let p1 = p.new_process(ModuleHash::random()).unwrap(); + let p1 = p.new_process(ModuleHash::xxhash_random()).unwrap(); let _t1 = p1 .new_thread(WasiMemoryLayout::default(), ThreadStartType::MainThread) .unwrap(); @@ -234,7 +234,7 @@ mod tests { .unwrap(); assert_eq!( - p.new_process(ModuleHash::random()).unwrap_err(), + p.new_process(ModuleHash::xxhash_random()).unwrap_err(), ControlPlaneError::TaskLimitReached { max: 2 } ); } @@ -248,7 +248,7 @@ mod tests { enable_exponential_cpu_backoff: None, }); - let p1 = p.new_process(ModuleHash::random()).unwrap(); + let p1 = p.new_process(ModuleHash::xxhash_random()).unwrap(); for _ in 0..10 { let _thread = p1 @@ -264,7 +264,7 @@ mod tests { .unwrap(); assert_eq!( - p.new_process(ModuleHash::random()).unwrap_err(), + p.new_process(ModuleHash::xxhash_random()).unwrap_err(), ControlPlaneError::TaskLimitReached { max: 2 } ); } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 517b2faa2bd..c532aeb3d6e 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -104,7 +104,8 @@ where fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, Result> { let engine = self.engine(); let module_cache = self.module_cache(); - let hash = ModuleHash::hash(wasm); + // FIXME: Amin + let hash = ModuleHash::xxhash(wasm); let task = async move { load_module(&engine, &module_cache, wasm, hash).await }; @@ -553,7 +554,8 @@ impl Runtime for OverriddenRuntime { if self.engine.is_some() || self.module_cache.is_some() { let engine = self.engine(); let module_cache = self.module_cache(); - let hash = ModuleHash::hash(wasm); + // FIXME: Amin + let hash = ModuleHash::xxhash(wasm); let task = async move { load_module(&engine, &module_cache, wasm, hash).await }; Box::pin(task) diff --git a/lib/wasix/src/runtime/module_cache/fallback.rs b/lib/wasix/src/runtime/module_cache/fallback.rs index e302627b06f..905643ce6c8 100644 --- a/lib/wasix/src/runtime/module_cache/fallback.rs +++ b/lib/wasix/src/runtime/module_cache/fallback.rs @@ -179,7 +179,7 @@ mod tests { async fn load_from_primary() { let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); let primary = SharedCache::default(); let fallback = SharedCache::default(); primary.save(key, &engine, &module).await.unwrap(); @@ -204,7 +204,7 @@ mod tests { async fn loading_from_fallback_also_populates_primary() { let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); let primary = SharedCache::default(); let fallback = SharedCache::default(); fallback.save(key, &engine, &module).await.unwrap(); @@ -230,7 +230,7 @@ mod tests { async fn saving_will_update_both() { let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); let primary = SharedCache::default(); let fallback = SharedCache::default(); let cache = FallbackCache::new(&primary, &fallback); diff --git a/lib/wasix/src/runtime/module_cache/filesystem.rs b/lib/wasix/src/runtime/module_cache/filesystem.rs index 30db6f78451..5528ff2261f 100644 --- a/lib/wasix/src/runtime/module_cache/filesystem.rs +++ b/lib/wasix/src/runtime/module_cache/filesystem.rs @@ -220,7 +220,7 @@ mod tests { let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); let expected_path = cache.path(key, engine.deterministic_id()); cache.save(key, &engine, &module).await.unwrap(); @@ -236,7 +236,7 @@ mod tests { let cache_dir = temp.path().join("this").join("doesn't").join("exist"); assert!(!cache_dir.exists()); let cache = FileSystemCache::new(&cache_dir, create_tokio_task_manager()); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); cache.save(key, &engine, &module).await.unwrap(); @@ -247,7 +247,7 @@ mod tests { async fn missing_file() { let temp = TempDir::new().unwrap(); let engine = Engine::default(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); let err = cache.load(key, &engine).await.unwrap_err(); @@ -260,7 +260,7 @@ mod tests { let temp = TempDir::new().unwrap(); let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); let expected_path = cache.path(key, engine.deterministic_id()); std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap(); @@ -283,7 +283,7 @@ mod tests { let temp = TempDir::new().unwrap(); let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); let expected_path = cache.path(key, engine.deterministic_id()); std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap(); diff --git a/lib/wasix/src/runtime/module_cache/shared.rs b/lib/wasix/src/runtime/module_cache/shared.rs index fa3f138f345..522980c6cb6 100644 --- a/lib/wasix/src/runtime/module_cache/shared.rs +++ b/lib/wasix/src/runtime/module_cache/shared.rs @@ -64,7 +64,7 @@ mod tests { let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); let cache = SharedCache::default(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); cache.save(key, &engine, &module).await.unwrap(); let round_tripped = cache.load(key, &engine).await.unwrap(); diff --git a/lib/wasix/src/runtime/module_cache/thread_local.rs b/lib/wasix/src/runtime/module_cache/thread_local.rs index ef1ad9288f8..bfccc24fa04 100644 --- a/lib/wasix/src/runtime/module_cache/thread_local.rs +++ b/lib/wasix/src/runtime/module_cache/thread_local.rs @@ -70,7 +70,7 @@ mod tests { let engine = Engine::default(); let module = Module::new(&engine, ADD_WAT).unwrap(); let cache = ThreadLocalCache::default(); - let key = ModuleHash::from_bytes([0; 8]); + let key = ModuleHash::xxhash_from_bytes([0; 8]); cache.save(key, &engine, &module).await.unwrap(); let round_tripped = cache.load(key, &engine).await.unwrap(); diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/wasix/src/runtime/module_cache/types.rs index 57324cbd7e9..0e665a4474c 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/wasix/src/runtime/module_cache/types.rs @@ -6,6 +6,7 @@ use std::{ }; use rand::RngCore; +use sha2::Digest; use wasmer::{Engine, Module}; use crate::runtime::module_cache::FallbackCache; @@ -127,50 +128,122 @@ impl CacheError { } } -/// The XXHash hash of a WebAssembly module. +/// The hash of a WebAssembly module. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct ModuleHash([u8; 8]); +pub enum ModuleHash { + XXHash([u8; 8]), + Sha256([u8; 32]), +} impl ModuleHash { - /// Create a new [`ModuleHash`] from the raw XXHash hash. - pub fn from_bytes(key: [u8; 8]) -> Self { - ModuleHash(key) + /// Create a new [`ModuleHash`] from the raw xxhash hash. + pub fn xxhash_from_bytes(key: [u8; 8]) -> Self { + Self::from_bytes(key) + } + + /// Create a new [`ModuleHash`] from the raw sha256 hash. + pub fn sha256_from_bytes(key: [u8; 32]) -> Self { + Self::from_bytes(key) + } + + fn from_bytes(key: [u8; N]) -> Self { + if N == 8 { + let key = key.as_slice().try_into().unwrap(); + ModuleHash::XXHash(key) + } else if N == 32 { + let key = key.as_slice().try_into().unwrap(); + ModuleHash::Sha256(key) + } else { + panic!("Only keys with size 8 or 32 are accepted") + } + } + + /// Creates a random xxhash for the module + pub fn xxhash_random() -> Self { + Self::random::<8>() + } + + /// Creates a random sha256 hash for the module + pub fn sha256_random() -> Self { + Self::random::<32>() } // Creates a random hash for the module - pub fn random() -> Self { + fn random() -> Self { let mut rand = rand::thread_rng(); - let mut key = [0u8; 8]; + let mut key = [0u8; N]; rand.fill_bytes(&mut key); - Self(key) + Self::from_bytes(key) } /// Parse a XXHash hash from a hex-encoded string. - pub fn parse_hex(hex_str: &str) -> Result { - let mut hash = [0_u8; 8]; + pub fn xxhash_parse_hex(hex_str: &str) -> Result { + Self::parse_hex::<8>(hex_str) + } + + /// Parse a Sha256 hash from a hex-encoded string. + pub fn sha256_parse_hex(hex_str: &str) -> Result { + Self::parse_hex::<32>(hex_str) + } + + fn parse_hex(hex_str: &str) -> Result { + let mut hash = [0_u8; N]; hex::decode_to_slice(hex_str, &mut hash)?; - Ok(Self(hash)) + Ok(Self::from_bytes(hash)) } /// Generate a new [`ModuleCache`] based on the XXHash hash of some bytes. - pub fn hash(wasm: impl AsRef<[u8]>) -> Self { + pub fn xxhash(wasm: impl AsRef<[u8]>) -> Self { let wasm = wasm.as_ref(); let hash = xxhash_rust::xxh64::xxh64(wasm, 0); - Self(hash.to_ne_bytes()) + Self::XXHash(hash.to_ne_bytes()) } - /// Get the raw XXHash hash. - pub fn as_bytes(self) -> [u8; 8] { - self.0 + /// Generate a new [`ModuleCache`] based on the Sha256 hash of some bytes. + pub fn sha256(wasm: impl AsRef<[u8]>) -> Self { + let wasm = wasm.as_ref(); + + let hash: [u8; 32] = sha2::Sha256::digest(wasm).into(); + + Self::Sha256(hash) + } + + /// Get the raw hash. + pub fn as_bytes(self) -> [u8; N] { + match self { + ModuleHash::XXHash(bytes) => { + if N == 8 { + bytes.as_slice().try_into().unwrap() + } else { + panic!("Requested for {N} bytes, byt XXHash is 8 bytes"); + } + } + ModuleHash::Sha256(bytes) => { + if N == 32 { + bytes.as_slice().try_into().unwrap() + } else { + panic!("Requested for {N} bytes, but Sha256 is 32 bytes") + } + } + } } } impl Display for ModuleHash { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - for byte in self.0 { - write!(f, "{byte:02X}")?; + fn format(f: &mut Formatter<'_>, bytes: &[u8; N]) -> fmt::Result { + for byte in bytes { + write!(f, "{byte:02X}")?; + } + + Ok(()) + } + + match self { + ModuleHash::XXHash(bytes) => format(f, bytes)?, + ModuleHash::Sha256(bytes) => format(f, bytes)?, } Ok(()) @@ -200,7 +273,7 @@ mod tests { let wasm = b"\0asm..."; let raw = [0x0c, 0xc7, 0x88, 0x60, 0xd4, 0x14, 0x71, 0x4c]; - let hash = ModuleHash::hash(wasm); + let hash = ModuleHash::xxhash(wasm); assert_eq!(hash.as_bytes(), raw); } diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index ad98bc164da..e60971d4b6c 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -910,7 +910,7 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] pub fn build(self) -> Result { - let module_hash = self.module_hash.unwrap_or_else(ModuleHash::random); + let module_hash = self.module_hash.unwrap_or_else(ModuleHash::xxhash_random); let init = self.build_init()?; WasiEnv::from_init(init, module_hash) } @@ -925,7 +925,7 @@ impl WasiEnvBuilder { self, store: &mut impl AsStoreMut, ) -> Result { - let module_hash = self.module_hash.unwrap_or_else(ModuleHash::random); + let module_hash = self.module_hash.unwrap_or_else(ModuleHash::xxhash_random); let init = self.build_init()?; let env = WasiEnv::from_init(init, module_hash)?; let func_env = WasiFunctionEnv::new(store, env); @@ -943,7 +943,7 @@ impl WasiEnvBuilder { module: Module, store: &mut impl AsStoreMut, ) -> Result<(Instance, WasiFunctionEnv), WasiRuntimeError> { - self.instantiate_ext(module, ModuleHash::random(), store) + self.instantiate_ext(module, ModuleHash::xxhash_random(), store) } #[allow(clippy::result_large_err)] @@ -959,7 +959,7 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] pub fn run(self, module: Module) -> Result<(), WasiRuntimeError> { - self.run_ext(module, ModuleHash::random()) + self.run_ext(module, ModuleHash::xxhash_random()) } #[allow(clippy::result_large_err)] @@ -971,7 +971,7 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] #[tracing::instrument(level = "debug", skip_all)] pub fn run_with_store(self, module: Module, store: &mut Store) -> Result<(), WasiRuntimeError> { - self.run_with_store_ext(module, ModuleHash::random(), store) + self.run_with_store_ext(module, ModuleHash::xxhash_random(), store) } #[allow(clippy::result_large_err)] From 5988de5b6dfc77ccb7ce117336f46096e4c3de02 Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Mon, 29 Apr 2024 23:37:53 +1000 Subject: [PATCH 17/55] Fix for the sha256 caching issue --- lib/wasix/src/runtime/resolver/inputs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/wasix/src/runtime/resolver/inputs.rs index bccdf806b25..525200009c5 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/wasix/src/runtime/resolver/inputs.rs @@ -292,7 +292,8 @@ impl WebcHash { pub fn for_file(path: &PathBuf) -> Result { // check for a hash at the file location - let path_hash = path.join(".sha256"); + let mut path_hash = path.clone(); + path_hash.set_extension("webc.sha256"); if let Ok(mut file) = File::open(&path_hash) { let mut hash = Vec::new(); if let Ok(amt) = file.read_to_end(&mut hash) { From f7638cfdfca6ca8f1d9055986b639e8a157df322 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 29 Apr 2024 18:01:49 +0330 Subject: [PATCH 18/55] better API --- lib/wasix/src/bin_factory/binary_package.rs | 13 ++-- lib/wasix/src/runtime/module_cache/types.rs | 62 +++++-------------- .../package_loader/load_package_tree.rs | 25 +++++++- lib/wasix/src/syscalls/journal/mod.rs | 2 +- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 2f7cfc0be91..ea903d8dec3 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -23,16 +23,21 @@ pub struct BinaryPackageCommand { metadata: webc::metadata::Command, #[derivative(Debug = "ignore")] pub(crate) atom: SharedBytes, - hash: OnceCell, + hash: ModuleHash, } impl BinaryPackageCommand { - pub fn new(name: String, metadata: webc::metadata::Command, atom: SharedBytes) -> Self { + pub fn new( + name: String, + metadata: webc::metadata::Command, + atom: SharedBytes, + hash: ModuleHash, + ) -> Self { Self { name, metadata, atom, - hash: OnceCell::new(), + hash, } } @@ -53,7 +58,7 @@ impl BinaryPackageCommand { } pub fn hash(&self) -> &ModuleHash { - self.hash.get_or_init(|| ModuleHash::xxhash(self.atom())) + &self.hash } } diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/wasix/src/runtime/module_cache/types.rs index 0e665a4474c..8d449fd0187 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/wasix/src/runtime/module_cache/types.rs @@ -138,58 +138,42 @@ pub enum ModuleHash { impl ModuleHash { /// Create a new [`ModuleHash`] from the raw xxhash hash. pub fn xxhash_from_bytes(key: [u8; 8]) -> Self { - Self::from_bytes(key) + Self::XXHash(key) } /// Create a new [`ModuleHash`] from the raw sha256 hash. pub fn sha256_from_bytes(key: [u8; 32]) -> Self { - Self::from_bytes(key) - } - - fn from_bytes(key: [u8; N]) -> Self { - if N == 8 { - let key = key.as_slice().try_into().unwrap(); - ModuleHash::XXHash(key) - } else if N == 32 { - let key = key.as_slice().try_into().unwrap(); - ModuleHash::Sha256(key) - } else { - panic!("Only keys with size 8 or 32 are accepted") - } + Self::Sha256(key) } /// Creates a random xxhash for the module pub fn xxhash_random() -> Self { - Self::random::<8>() + let mut rand = rand::thread_rng(); + let mut key = [0u8; 8]; + rand.fill_bytes(&mut key); + Self::xxhash_from_bytes(key) } /// Creates a random sha256 hash for the module pub fn sha256_random() -> Self { - Self::random::<32>() - } - - // Creates a random hash for the module - fn random() -> Self { let mut rand = rand::thread_rng(); - let mut key = [0u8; N]; + let mut key = [0u8; 32]; rand.fill_bytes(&mut key); - Self::from_bytes(key) + Self::sha256_from_bytes(key) } /// Parse a XXHash hash from a hex-encoded string. pub fn xxhash_parse_hex(hex_str: &str) -> Result { - Self::parse_hex::<8>(hex_str) + let mut hash = [0_u8; 8]; + hex::decode_to_slice(hex_str, &mut hash)?; + Ok(Self::xxhash_from_bytes(hash)) } /// Parse a Sha256 hash from a hex-encoded string. pub fn sha256_parse_hex(hex_str: &str) -> Result { - Self::parse_hex::<32>(hex_str) - } - - fn parse_hex(hex_str: &str) -> Result { - let mut hash = [0_u8; N]; + let mut hash = [0_u8; 32]; hex::decode_to_slice(hex_str, &mut hash)?; - Ok(Self::from_bytes(hash)) + Ok(Self::sha256_from_bytes(hash)) } /// Generate a new [`ModuleCache`] based on the XXHash hash of some bytes. @@ -211,22 +195,10 @@ impl ModuleHash { } /// Get the raw hash. - pub fn as_bytes(self) -> [u8; N] { + pub fn as_bytes(&self) -> &[u8] { match self { - ModuleHash::XXHash(bytes) => { - if N == 8 { - bytes.as_slice().try_into().unwrap() - } else { - panic!("Requested for {N} bytes, byt XXHash is 8 bytes"); - } - } - ModuleHash::Sha256(bytes) => { - if N == 32 { - bytes.as_slice().try_into().unwrap() - } else { - panic!("Requested for {N} bytes, but Sha256 is 32 bytes") - } - } + ModuleHash::XXHash(bytes) => bytes.as_slice(), + ModuleHash::Sha256(bytes) => bytes.as_slice(), } } } @@ -261,7 +233,7 @@ mod tests { #[test] fn key_is_displayed_as_hex() { - let key = ModuleHash::from_bytes([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]); + let key = ModuleHash::xxhash_from_bytes([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]); let repr = key.to_string(); diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index b35c726437f..9f98c6bd906 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -6,6 +6,7 @@ use std::{ }; use anyhow::{Context, Error}; +use base64::Engine; use futures::{future::BoxFuture, StreamExt, TryStreamExt}; use once_cell::sync::OnceCell; use petgraph::visit::EdgeRef; @@ -19,6 +20,7 @@ use webc::{ use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, runtime::{ + module_cache::ModuleHash, package_loader::PackageLoader, resolver::{ DependencyGraph, ItemLocation, PackageSummary, Resolution, ResolvedFileSystemMapping, @@ -148,13 +150,15 @@ fn load_binary_command( return Ok(legacy_atom_hack(webc, command_name, cmd)); } + let hash = atom_hash(webc, &atom_name); + let atom = atom.with_context(|| { format!( "The '{command_name}' command uses the '{atom_name}' atom, but it isn't present in the package {package_id}" ) })?; - let cmd = BinaryPackageCommand::new(command_name.to_string(), cmd.clone(), atom); + let cmd = BinaryPackageCommand::new(command_name.to_string(), cmd.clone(), atom, hash); Ok(Some(cmd)) } @@ -213,13 +217,32 @@ fn legacy_atom_hack( "(hack) The command metadata is malformed. Falling back to the first atom in the WEBC file", ); + let hash = atom_hash(webc, &name); + Some(BinaryPackageCommand::new( command_name.to_string(), metadata.clone(), atom, + hash, )) } +fn atom_hash(webc: &Container, atom_name: &str) -> ModuleHash { + let base64_encoded = webc.manifest().atoms[atom_name] + .signature + .strip_prefix("sha256") + .expect("malformed atom signature: does not have sha256 prefix"); + let sha256 = base64::prelude::BASE64_STANDARD + .decode(base64_encoded) + .expect("malformed base64 encoded hash"); + let hash: [u8; 32] = sha256 + .as_slice() + .try_into() + .expect("sha256 hash is not 32 bytes"); + + ModuleHash::sha256_from_bytes(hash) +} + async fn fetch_dependencies( loader: &dyn PackageLoader, pkg: &ResolvedPackage, diff --git a/lib/wasix/src/syscalls/journal/mod.rs b/lib/wasix/src/syscalls/journal/mod.rs index 984c9b5b686..899c0355010 100644 --- a/lib/wasix/src/syscalls/journal/mod.rs +++ b/lib/wasix/src/syscalls/journal/mod.rs @@ -52,7 +52,7 @@ pub struct JournalSyscallPlayer<'a, 'c> { impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { pub fn new(mut ctx: FunctionEnvMut<'c, WasiEnv>, bootstrapping: bool) -> Self { - let cur_module_hash: [u8; 8] = ctx.data().process.module_hash.as_bytes(); + let cur_module_hash: &[u8] = ctx.data().process.module_hash.as_bytes(); let mut ret = JournalSyscallPlayer { ctx, bootstrapping, From 08126b6a8a361f1630014f5604f194f8f265c937 Mon Sep 17 00:00:00 2001 From: Christoph Herzog Date: Mon, 29 Apr 2024 16:47:26 +0200 Subject: [PATCH 19/55] chore(wasix): More logging improvements --- .../package_loader/load_package_tree.rs | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index b35c726437f..507479e3195 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -97,6 +97,7 @@ fn commands( /// Given a [`webc::metadata::Command`], figure out which atom it uses and load /// that atom into a [`BinaryPackageCommand`]. +#[tracing::instrument(skip_all, fields(%package_id, %command_name))] fn load_binary_command( package_id: &PackageId, command_name: &str, @@ -122,7 +123,7 @@ fn load_binary_command( let package = &containers[package_id]; - let webc = match dependency { + let (webc, resolved_package_id) = match dependency { Some(dep) => { let ix = resolution .graph @@ -137,20 +138,38 @@ fn load_binary_command( .with_context(|| format!("Unable to find the \"{dep}\" dependency for the \"{command_name}\" command in \"{package_id}\""))?; let other_package = graph.node_weight(edge_reference.target()).unwrap(); - &containers[&other_package.id] + let id = &other_package.id; + + tracing::debug!( + dependency=%dep, + resolved_package_id=%id, + "command atom resolution: resolved dependency", + ); + (&containers[id], id) } - None => package, + None => (package, package_id), }; let atom = webc.get_atom(&atom_name); if atom.is_none() && cmd.annotations.is_empty() { + tracing::info!("applying legacy atom hack"); return Ok(legacy_atom_hack(webc, command_name, cmd)); } let atom = atom.with_context(|| { + + let available_atoms = webc.atoms().keys().map(|x| x.as_str()).collect::>().join(","); + + tracing::warn!( + %atom_name, + %resolved_package_id, + %available_atoms, + "invalid command: could not find atom in package", + ); + format!( - "The '{command_name}' command uses the '{atom_name}' atom, but it isn't present in the package {package_id}" + "The '{command_name}' command uses the '{atom_name}' atom, but it isn't present in the package: {resolved_package_id})" ) })?; From 3c57aeb098ee4ee8ba7d68dfd63dd7cdb7a4b746 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 29 Apr 2024 22:30:08 +0330 Subject: [PATCH 20/55] use Vec instead of array for hash --- lib/journal/src/concrete/archived.rs | 2 +- lib/journal/src/concrete/archived_from.rs | 2 +- lib/journal/src/concrete/tests.rs | 2 +- lib/journal/src/entry.rs | 2 +- lib/wasix/src/state/func_env.rs | 2 +- lib/wasix/src/syscalls/journal/actions/init_module.rs | 2 +- lib/wasix/src/syscalls/journal/actions/set_thread.rs | 2 +- lib/wasix/src/syscalls/journal/actions/snapshot.rs | 4 ++-- lib/wasix/src/syscalls/journal/actions/update_memory.rs | 2 +- lib/wasix/src/syscalls/journal/mod.rs | 6 +++--- tests/lib/wast/src/wasi_wast.rs | 3 ++- 11 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/journal/src/concrete/archived.rs b/lib/journal/src/concrete/archived.rs index 9ba35130d91..5b61ce4efd4 100644 --- a/lib/journal/src/concrete/archived.rs +++ b/lib/journal/src/concrete/archived.rs @@ -950,7 +950,7 @@ pub enum ArchivedJournalEntry<'a> { #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes), repr(align(8)))] pub struct JournalEntryInitModuleV1 { - pub wasm_hash: [u8; 8], + pub wasm_hash: Vec, } #[repr(C)] diff --git a/lib/journal/src/concrete/archived_from.rs b/lib/journal/src/concrete/archived_from.rs index 867945dbd73..e151aedb8d7 100644 --- a/lib/journal/src/concrete/archived_from.rs +++ b/lib/journal/src/concrete/archived_from.rs @@ -649,7 +649,7 @@ impl<'a> TryFrom> for JournalEntry<'a> { Ok(match value { ArchivedJournalEntry::InitModuleV1(ArchivedJournalEntryInitModuleV1 { wasm_hash }) => { Self::InitModuleV1 { - wasm_hash: *wasm_hash, + wasm_hash: wasm_hash.to_vec(), } } ArchivedJournalEntry::ClearEtherealV1(ArchivedJournalEntryClearEtherealV1 { diff --git a/lib/journal/src/concrete/tests.rs b/lib/journal/src/concrete/tests.rs index faebfcc296d..c4eae636a99 100644 --- a/lib/journal/src/concrete/tests.rs +++ b/lib/journal/src/concrete/tests.rs @@ -49,7 +49,7 @@ pub fn run_test<'a>(record: JournalEntry<'a>) { #[test] pub fn test_record_init_module() { run_test(JournalEntry::InitModuleV1 { - wasm_hash: [13u8; 8], + wasm_hash: vec![13u8; 8], }); } diff --git a/lib/journal/src/entry.rs b/lib/journal/src/entry.rs index 29190d9d350..9fe01815510 100644 --- a/lib/journal/src/entry.rs +++ b/lib/journal/src/entry.rs @@ -84,7 +84,7 @@ pub enum SocketOptTimeType { #[serde(rename_all = "snake_case")] pub enum JournalEntry<'a> { InitModuleV1 { - wasm_hash: [u8; 8], + wasm_hash: Vec, }, ClearEtherealV1, UpdateMemoryRegionV1 { diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 001f3290f94..2a72d494c84 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -285,7 +285,7 @@ impl WasiFunctionEnv { // The first event we save is an event that records the module hash. // Note: This is used to detect if an incorrect journal is used on the wrong // process or if a process has been recompiled - let wasm_hash = self.data(&store).process.module_hash.as_bytes(); + let wasm_hash = self.data(&store).process.module_hash.as_bytes().to_vec(); let mut ctx = self.env.clone().into_mut(&mut store); crate::journal::JournalEffector::save_event( &mut ctx, diff --git a/lib/wasix/src/syscalls/journal/actions/init_module.rs b/lib/wasix/src/syscalls/journal/actions/init_module.rs index 2e0dfe59a15..6250d2cacad 100644 --- a/lib/wasix/src/syscalls/journal/actions/init_module.rs +++ b/lib/wasix/src/syscalls/journal/actions/init_module.rs @@ -4,7 +4,7 @@ impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { #[allow(clippy::result_large_err)] pub(crate) unsafe fn action_init_module( &mut self, - wasm_hash: [u8; 8], + wasm_hash: Vec, differ_ethereal: Option<&mut Vec>>, ) -> Result<(), WasiRuntimeError> { tracing::trace!("Replay journal - InitModule {:?}", wasm_hash); diff --git a/lib/wasix/src/syscalls/journal/actions/set_thread.rs b/lib/wasix/src/syscalls/journal/actions/set_thread.rs index 2d21e1de6bf..d97e7d041a5 100644 --- a/lib/wasix/src/syscalls/journal/actions/set_thread.rs +++ b/lib/wasix/src/syscalls/journal/actions/set_thread.rs @@ -13,7 +13,7 @@ impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { layout: WasiMemoryLayout, differ_ethereal: Option<&mut Vec>>, ) -> Result<(), WasiRuntimeError> { - if Some(self.cur_module_hash) != self.journal_module_hash { + if Some(&self.cur_module_hash) != self.journal_module_hash.as_ref() { tracing::trace!(%id, "Skipping journal entry - SetThread call_stack={} bytes memory_stack={} bytes store_data={} bytes", call_stack.len(), memory_stack.len(), store_data.len()); return Ok(()); } diff --git a/lib/wasix/src/syscalls/journal/actions/snapshot.rs b/lib/wasix/src/syscalls/journal/actions/snapshot.rs index 3b6e5439afc..add81d0e87b 100644 --- a/lib/wasix/src/syscalls/journal/actions/snapshot.rs +++ b/lib/wasix/src/syscalls/journal/actions/snapshot.rs @@ -13,11 +13,11 @@ impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { // in a clean state) let mut clear_ethereal = false; if self.journal_module_hash.is_some() - && Some(self.cur_module_hash) != self.journal_module_hash + && Some(&self.cur_module_hash) != self.journal_module_hash.as_ref() { tracing::error!( "The WASM module hash does not match the journal module hash (journal_hash={:x?} vs module_hash{:x?}) - forcing a restart", - self.journal_module_hash.unwrap(), + self.journal_module_hash.as_ref().unwrap(), self.cur_module_hash ); self.clear_ethereal(differ_ethereal); diff --git a/lib/wasix/src/syscalls/journal/actions/update_memory.rs b/lib/wasix/src/syscalls/journal/actions/update_memory.rs index 746e5355e7a..743183a2186 100644 --- a/lib/wasix/src/syscalls/journal/actions/update_memory.rs +++ b/lib/wasix/src/syscalls/journal/actions/update_memory.rs @@ -8,7 +8,7 @@ impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { data: Cow<'a, [u8]>, differ_ethereal: Option<&mut Vec>>, ) -> Result<(), WasiRuntimeError> { - if Some(self.cur_module_hash) != self.journal_module_hash { + if Some(&self.cur_module_hash) != self.journal_module_hash.as_ref() { tracing::trace!("Ignored journal - UpdateMemory"); return Ok(()); } diff --git a/lib/wasix/src/syscalls/journal/mod.rs b/lib/wasix/src/syscalls/journal/mod.rs index 899c0355010..b88c89091a7 100644 --- a/lib/wasix/src/syscalls/journal/mod.rs +++ b/lib/wasix/src/syscalls/journal/mod.rs @@ -31,9 +31,9 @@ pub struct JournalSyscallPlayer<'a, 'c> { pub ctx: FunctionEnvMut<'c, WasiEnv>, pub bootstrapping: bool, - pub journal_module_hash: Option<[u8; 8]>, + pub journal_module_hash: Option>, pub rewind: Option, - pub cur_module_hash: [u8; 8], + pub cur_module_hash: Vec, pub real_fd: HashSet, // We delay the spawning of threads until the end as its @@ -52,7 +52,7 @@ pub struct JournalSyscallPlayer<'a, 'c> { impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { pub fn new(mut ctx: FunctionEnvMut<'c, WasiEnv>, bootstrapping: bool) -> Self { - let cur_module_hash: &[u8] = ctx.data().process.module_hash.as_bytes(); + let cur_module_hash = ctx.data().process.module_hash.as_bytes().to_vec(); let mut ret = JournalSyscallPlayer { ctx, bootstrapping, diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index 5dc1e7ffbd1..9648c2624f2 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -121,7 +121,8 @@ impl<'a> WasiTest<'a> { wasm_module.read_to_end(&mut out)?; out }; - let module_hash = ModuleHash::hash(&wasm_bytes); + // FIXME: Amin? + let module_hash = ModuleHash::xxhash(&wasm_bytes); let module = Module::new(store, wasm_bytes)?; let (builder, _tempdirs, mut stdin_tx, stdout_rx, stderr_rx) = From 04c47280b61fcca7a6e66a215491035e1a60baf4 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 30 Apr 2024 00:10:41 +0330 Subject: [PATCH 21/55] fix prefix --- lib/wasix/src/runtime/package_loader/load_package_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 9f98c6bd906..6f6d99c2903 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -230,7 +230,7 @@ fn legacy_atom_hack( fn atom_hash(webc: &Container, atom_name: &str) -> ModuleHash { let base64_encoded = webc.manifest().atoms[atom_name] .signature - .strip_prefix("sha256") + .strip_prefix("sha256:") .expect("malformed atom signature: does not have sha256 prefix"); let sha256 = base64::prelude::BASE64_STANDARD .decode(base64_encoded) From 66f3b08fe65594a57624d05650a30c2c1a14069a Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 30 Apr 2024 08:43:25 +0200 Subject: [PATCH 22/55] [skip ci] feat(wasmer_config): Make `Package` fields non-mandatory This commit makes some fields of the `wasmer_config::package::Package` type non mandatory. In particular, `name`, `version` and `description` aren't mandatory anymore. This introduces *two ways* to describe unnamed packages - no `package` section in the manifest and `package` section with no `name` field in it, and a third kind of package in the package taxonomy, which I deliberately name, here _unversioned packages_, that is packages with a `name` (and any other field) but no `version`. --- lib/cli/src/commands/app/create.rs | 49 ++++++------ lib/cli/src/commands/app/deploy.rs | 109 +++++++++++++++----------- lib/cli/src/commands/package/build.rs | 14 ++-- lib/cli/src/commands/publish.rs | 83 ++++++++++---------- lib/config/src/package/mod.rs | 11 +-- lib/registry/src/package/builder.rs | 26 +++--- lib/registry/src/publish.rs | 33 +++++--- 7 files changed, 185 insertions(+), 140 deletions(-) diff --git a/lib/cli/src/commands/app/create.rs b/lib/cli/src/commands/app/create.rs index 37a64dac050..517c3f1b196 100644 --- a/lib/cli/src/commands/app/create.rs +++ b/lib/cli/src/commands/app/create.rs @@ -559,31 +559,36 @@ impl AppCreator { ), }; - let full = format!("{}@{}", pkg.name, pkg.version); - let mut pkg_ident = NamedPackageIdent::from_str(&pkg.name) - .with_context(|| format!("local package manifest has invalid name: '{full}'"))?; - - // Pin the version. - pkg_ident.tag = Some(Tag::from_str(&pkg.version.to_string()).unwrap()); - - if self.interactive { - eprintln!("Found local package: '{}'", full.green()); - - let msg = format!("Use package '{pkg_ident}'"); - - let theme = dialoguer::theme::ColorfulTheme::default(); - let should_use = Confirm::with_theme(&theme) - .with_prompt(&msg) - .interact_opt()? - .unwrap_or_default(); - - if should_use { - Some(pkg_ident) + if let (Some(name), Some(version)) = (pkg.name, pkg.version) { + let full = format!("{}@{}", name, version); + let mut pkg_ident = NamedPackageIdent::from_str(&name).with_context(|| { + format!("local package manifest has invalid name: '{full}'") + })?; + + // Pin the version. + pkg_ident.tag = Some(Tag::from_str(&version.to_string()).unwrap()); + + if self.interactive { + eprintln!("Found local package: '{}'", full.green()); + + let msg = format!("Use package '{pkg_ident}'"); + + let theme = dialoguer::theme::ColorfulTheme::default(); + let should_use = Confirm::with_theme(&theme) + .with_prompt(&msg) + .interact_opt()? + .unwrap_or_default(); + + if should_use { + Some(pkg_ident) + } else { + None + } } else { - None + Some(pkg_ident) } } else { - Some(pkg_ident) + None } } else { None diff --git a/lib/cli/src/commands/app/deploy.rs b/lib/cli/src/commands/app/deploy.rs index d5e3a422fd4..934c3caa059 100644 --- a/lib/cli/src/commands/app/deploy.rs +++ b/lib/cli/src/commands/app/deploy.rs @@ -327,58 +327,73 @@ impl AsyncCliCommand for CmdAppDeploy { if let Ok(Some((manifest_path, manifest))) = load_package_manifest(&base_dir_path) { if let Some(package) = &manifest.package { - if package.name == n.full_name() { - eprintln!( - "Found local package (manifest path: {}).", - manifest_path.display() - ); - eprintln!("The `package` field in `app.yaml` specified the same named package ({}).", package.name); - eprintln!("This behaviour is deprecated."); - let theme = dialoguer::theme::ColorfulTheme::default(); - if self.non_interactive { - eprintln!("Hint: replace `package: {}` with `package: .` to replicate the intended behaviour.", n); - anyhow::bail!("deprecated deploy behaviour") - } else if Confirm::with_theme(&theme) - .with_prompt("Change package to '.' in app.yaml?") - .interact()? - { - app_config.package = PackageSource::Path(String::from(".")); - // We have to write it right now. - let new_config_raw = serde_yaml::to_string(&app_config)?; - std::fs::write(&app_config_path, new_config_raw).with_context( - || { - format!( - "Could not write file: '{}'", - app_config_path.display() - ) - }, - )?; - - log::info!( - "Using package {} ({})", - app_config.package, - n.full_name() + if let Some(name) = &package.name { + if name == &n.full_name() { + eprintln!( + "Found local package (manifest path: {}).", + manifest_path.display() ); + eprintln!("The `package` field in `app.yaml` specified the same named package ({}).", name); + eprintln!("This behaviour is deprecated."); + + let theme = dialoguer::theme::ColorfulTheme::default(); + if self.non_interactive { + eprintln!("Hint: replace `package: {}` with `package: .` to replicate the intended behaviour.", n); + anyhow::bail!("deprecated deploy behaviour") + } else if Confirm::with_theme(&theme) + .with_prompt("Change package to '.' in app.yaml?") + .interact()? + { + app_config.package = PackageSource::Path(String::from(".")); + // We have to write it right now. + let new_config_raw = serde_yaml::to_string(&app_config)?; + std::fs::write(&app_config_path, new_config_raw).with_context( + || { + format!( + "Could not write file: '{}'", + app_config_path.display() + ) + }, + )?; + + log::info!( + "Using package {} ({})", + app_config.package, + n.full_name() + ); - let package_id = self.publish(owner.clone(), manifest_path).await?; - - app_config.package = package_id.into(); - - DeployAppOpts { - app: &app_config, - original_config: Some( - app_config.clone().to_yaml_value().unwrap(), - ), - allow_create: true, - make_default: !self.no_default, - owner: Some(owner), - wait, - } - } else { - eprintln!( + let package_id = + self.publish(owner.clone(), manifest_path).await?; + + app_config.package = package_id.into(); + + DeployAppOpts { + app: &app_config, + original_config: Some( + app_config.clone().to_yaml_value().unwrap(), + ), + allow_create: true, + make_default: !self.no_default, + owner: Some(owner), + wait, + } + } else { + eprintln!( "{}: the package will not be published and the deployment will fail if the package does not already exist.", "Warning".yellow().bold() ); + DeployAppOpts { + app: &app_config, + original_config: Some( + app_config.clone().to_yaml_value().unwrap(), + ), + allow_create: true, + make_default: !self.no_default, + owner: Some(owner), + wait, + } + } + } else { DeployAppOpts { app: &app_config, original_config: Some( diff --git a/lib/cli/src/commands/package/build.rs b/lib/cli/src/commands/package/build.rs index 940f26dd299..67093906ce4 100644 --- a/lib/cli/src/commands/package/build.rs +++ b/lib/cli/src/commands/package/build.rs @@ -55,11 +55,15 @@ impl PackageBuild { let pkg = webc::wasmer_package::Package::from_manifest(manifest_path)?; let pkg_hash = PackageHash::from_sha256_bytes(pkg.webc_hash()); let name = if let Some(manifest_pkg) = manifest.package { - format!( - "{}-{}.webc", - manifest_pkg.name.replace('/', "-"), - manifest_pkg.version - ) + if let Some(name) = manifest_pkg.name { + if let Some(version) = manifest_pkg.version { + format!("{}-{}.webc", name.replace('/', "-"), version) + } else { + format!("{}-{}.webc", name.replace('/', "-"), pkg_hash) + } + } else { + format!("{}.webc", pkg_hash) + } } else { format!("{}.webc", pkg_hash) }; diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs index 7c47fafa8a7..17aff62cb48 100644 --- a/lib/cli/src/commands/publish.rs +++ b/lib/cli/src/commands/publish.rs @@ -123,51 +123,54 @@ impl AsyncCliCommand for Publish { let mut version = self.version.clone(); if let Some(ref mut pkg) = manifest.package { - let mut latest_version = { - let v = wasmer_api::query::get_package_version( - &client, - pkg.name.clone(), - "latest".into(), - ) - .await?; - if let Some(v) = v { - semver::Version::parse(&v.version) - .with_context(|| "While parsing registry version of package")? - } else { - pkg.version.clone() - } - }; - - if pkg.version < latest_version { - if self.bump { - latest_version.patch += 1; - version = Some(latest_version); - } else if interactive { - latest_version.patch += 1; - let theme = dialoguer::theme::ColorfulTheme::default(); - if Confirm::with_theme(&theme) - .with_prompt(format!( - "Do you want to bump the package to a new version? ({} -> {})", - pkg.version, latest_version - )) - .interact() - .unwrap_or_default() - { + if let (Some(pkg_name), Some(pkg_version)) = (&pkg.name, &pkg.version) { + let pkg_name = pkg_name.clone(); + let pkg_version = pkg_version.clone(); + + let mut latest_version = { + let v = wasmer_api::query::get_package_version( + &client, + pkg_name.clone(), + "latest".into(), + ) + .await?; + if let Some(v) = v { + semver::Version::parse(&v.version) + .with_context(|| "While parsing registry version of package")? + } else { + pkg_version.clone() + } + }; + + if pkg_version < latest_version { + if self.bump { + latest_version.patch += 1; version = Some(latest_version); + } else if interactive { + latest_version.patch += 1; + let theme = dialoguer::theme::ColorfulTheme::default(); + if Confirm::with_theme(&theme) + .with_prompt(format!( + "Do you want to bump the package to a new version? ({} -> {})", + pkg_version, latest_version + )) + .interact() + .unwrap_or_default() + { + pkg.version = Some(latest_version); + } + } else if latest_version > pkg_version { + eprintln!("Registry has a newer version of this package."); + eprintln!( + "If a package with version {} already exists, publishing will fail.", + pkg_version + ); } - } else if latest_version > pkg.version { - eprintln!("Registry has a newer version of this package."); - eprintln!( - "If a package with version {} already exists, publishing will fail.", - pkg.version - ); } - } - // If necessary, update the manifest. - if let Some(version) = version.clone() { + // If necessary, update the manifest. if version != pkg.version { - pkg.version = version; + pkg.version = version.clone(); let contents = toml::to_string(&manifest).with_context(|| { format!( diff --git a/lib/config/src/package/mod.rs b/lib/config/src/package/mod.rs index 1a3ca32a746..398ca12e6eb 100644 --- a/lib/config/src/package/mod.rs +++ b/lib/config/src/package/mod.rs @@ -113,13 +113,14 @@ const LICENSE_PATHS: &[&str; 3] = &["LICENSE", "LICENSE.md", "COPYING"]; #[non_exhaustive] pub struct Package { /// The package's name in the form `namespace/name`. - #[builder(setter(into))] - pub name: String, + #[builder(setter(into, strip_option), default)] + pub name: Option, /// The package's version number. - pub version: Version, + #[builder(setter(into, strip_option), default)] + pub version: Option, /// A brief description of the package. - #[builder(setter(into))] - pub description: String, + #[builder(setter(into, strip_option), default)] + pub description: Option, /// A SPDX license specifier for this package. #[builder(setter(into, strip_option), default)] pub license: Option, diff --git a/lib/registry/src/package/builder.rs b/lib/registry/src/package/builder.rs index 9af14587dc0..56c13eac0ec 100644 --- a/lib/registry/src/package/builder.rs +++ b/lib/registry/src/package/builder.rs @@ -119,13 +119,13 @@ impl Publish { if let Some(package_name) = self.package_name.as_ref() { if let Some(ref mut package) = manifest.package { - package.name = package_name.clone(); + package.name = Some(package_name.clone()); } } if let Some(version) = self.version.as_ref() { if let Some(ref mut package) = manifest.package { - package.version = version.clone(); + package.version = Some(version.clone()); } } @@ -673,16 +673,20 @@ mod validate { ) -> Result { match &manifest.package { Some(pkg) => { - let result = - crate::query_package_from_registry(registry, &pkg.name, None, Some(auth_token)); - - match result { - Ok(package_version) => Ok(package_version.is_private != pkg.private), - Err(QueryPackageError::NoPackageFound { .. }) => { - // The package hasn't been published yet - Ok(false) + if let Some(ref name) = pkg.name { + let result = + crate::query_package_from_registry(registry, name, None, Some(auth_token)); + + match result { + Ok(package_version) => Ok(package_version.is_private != pkg.private), + Err(QueryPackageError::NoPackageFound { .. }) => { + // The package hasn't been published yet + Ok(false) + } + Err(e) => Err(e.into()), } - Err(e) => Err(e.into()), + } else { + Ok(false) } } diff --git a/lib/registry/src/publish.rs b/lib/registry/src/publish.rs index ca101ddb907..570d3864ec4 100644 --- a/lib/registry/src/publish.rs +++ b/lib/registry/src/publish.rs @@ -130,21 +130,25 @@ pub async fn try_chunked_uploading( upload_package(&signed_url.url, archive_path, archived_data_size, timeout).await?; - let name = package.as_ref().map(|p| p.name.clone()); + let name = package.as_ref().and_then(|p| p.name.clone()); let namespace = match patch_namespace { Some(n) => Some(n), - None => package - .as_ref() - .map(|p| String::from(p.name.split_once('/').unwrap().0)), + None => package.as_ref().and_then(|p| { + p.name + .as_ref() + .map(|p| String::from(p.split_once('/').unwrap().0)) + }), }; let q = PublishPackageMutationChunked::build_query(publish_package_mutation_chunked::Variables { name, namespace, - version: package.as_ref().map(|p| p.version.to_string()), - description: package.as_ref().map(|p| p.description.clone()), + version: package + .as_ref() + .and_then(|p| p.version.as_ref().map(|v| v.to_string())), + description: package.as_ref().and_then(|p| p.description.clone()), manifest: manifest_string.to_string(), license: package.as_ref().and_then(|p| p.license.clone()), license_file: license_file.to_owned(), @@ -215,7 +219,12 @@ pub async fn try_chunked_uploading( let package_ident = PackageIdent::Named(NamedPackageIdent::from_str(&format!( "{}@{}", - package.name, package.version + package + .name + .expect("Unnamed package was published as named"), + package + .version + .expect("Unversioned package was published as versioned") ))?); eprintln!("Package published successfully"); // println!("🚀 Successfully published package `{}`", package_ident); @@ -306,8 +315,12 @@ fn google_signed_url( timeout: Duration, ) -> Result { let get_google_signed_url = GetSignedUrl::build_query(get_signed_url::Variables { - name: package.as_ref().map(|p| p.name.to_string()), - version: package.as_ref().map(|p| p.version.to_string()), + name: package + .as_ref() + .and_then(|p| p.name.as_ref().map(|n| n.to_string())), + version: package + .as_ref() + .and_then(|p| p.version.as_ref().map(|v| v.to_string())), filename: match package { Some(_) => None, None => Some(format!("unnamed_package_{}", rand::random::())), @@ -325,7 +338,7 @@ fn google_signed_url( let url = _response.url.ok_or_else(|| match package { Some(pkg) => { anyhow::anyhow!( - "could not get signed url for package {}@{}", + "could not get signed url for package {:?}@{:?}", pkg.name, pkg.version ) From 601e549ed4c4456a9061892c9d743573aa181533 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 30 Apr 2024 09:11:30 +0200 Subject: [PATCH 23/55] fix(wasmer-config): Fix tests --- lib/config/src/package/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/config/src/package/mod.rs b/lib/config/src/package/mod.rs index 398ca12e6eb..cedc4d00260 100644 --- a/lib/config/src/package/mod.rs +++ b/lib/config/src/package/mod.rs @@ -976,9 +976,9 @@ mod tests { fn test_to_string() { Manifest { package: Some(Package { - name: "package/name".to_string(), - version: Version::parse("1.0.0").unwrap(), - description: "test".to_string(), + name: Some("package/name".to_string()), + version: Some(Version::parse("1.0.0").unwrap()), + description: Some("test".to_string()), license: None, license_file: None, readme: None, From 0d0cafa2114185bddb5343ac226e06f332053b20 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 30 Apr 2024 12:43:18 +0330 Subject: [PATCH 24/55] enable socks support --- Cargo.lock | 13 +++++++++++++ lib/wasix/Cargo.toml | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d6e327bed3..d600d27b43e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3877,6 +3877,7 @@ dependencies = [ "tokio 1.37.0", "tokio-native-tls", "tokio-rustls", + "tokio-socks", "tokio-util", "tower-service", "url", @@ -5176,6 +5177,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tokio-socks" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0" +dependencies = [ + "either", + "futures-util", + "thiserror", + "tokio 1.37.0", +] + [[package]] name = "tokio-stream" version = "0.1.15" diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 6efa5a1a971..d586e898588 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -104,13 +104,13 @@ web-sys = { version = "0.3.64", features = [ [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest] version = "0.11" default-features = false -features = ["rustls-tls", "json", "stream"] +features = ["rustls-tls", "json", "stream", "socks"] optional = true [target.'cfg(target_arch = "riscv64")'.dependencies.reqwest] version = "0.11" default-features = false -features = ["native-tls", "json", "stream"] +features = ["native-tls", "json", "stream", "socks"] optional = true [target.'cfg(unix)'.dependencies] From 628bbe49fc0193f3f65c5b4e3c54ee76cfa20776 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 30 Apr 2024 17:24:26 +0330 Subject: [PATCH 25/55] use Box instead of Vec --- lib/journal/src/concrete/archived.rs | 2 +- lib/journal/src/concrete/archived_from.rs | 2 +- lib/journal/src/concrete/tests.rs | 2 +- lib/journal/src/entry.rs | 2 +- lib/wasix/src/state/func_env.rs | 2 +- lib/wasix/src/syscalls/journal/actions/init_module.rs | 2 +- lib/wasix/src/syscalls/journal/mod.rs | 6 +++--- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/journal/src/concrete/archived.rs b/lib/journal/src/concrete/archived.rs index 5b61ce4efd4..d03b8583dd0 100644 --- a/lib/journal/src/concrete/archived.rs +++ b/lib/journal/src/concrete/archived.rs @@ -950,7 +950,7 @@ pub enum ArchivedJournalEntry<'a> { #[derive(Debug, Clone, RkyvSerialize, RkyvDeserialize, Archive)] #[archive_attr(derive(CheckBytes), repr(align(8)))] pub struct JournalEntryInitModuleV1 { - pub wasm_hash: Vec, + pub wasm_hash: Box<[u8]>, } #[repr(C)] diff --git a/lib/journal/src/concrete/archived_from.rs b/lib/journal/src/concrete/archived_from.rs index e151aedb8d7..853973061bf 100644 --- a/lib/journal/src/concrete/archived_from.rs +++ b/lib/journal/src/concrete/archived_from.rs @@ -649,7 +649,7 @@ impl<'a> TryFrom> for JournalEntry<'a> { Ok(match value { ArchivedJournalEntry::InitModuleV1(ArchivedJournalEntryInitModuleV1 { wasm_hash }) => { Self::InitModuleV1 { - wasm_hash: wasm_hash.to_vec(), + wasm_hash: Box::from(wasm_hash.get()), } } ArchivedJournalEntry::ClearEtherealV1(ArchivedJournalEntryClearEtherealV1 { diff --git a/lib/journal/src/concrete/tests.rs b/lib/journal/src/concrete/tests.rs index c4eae636a99..722d9d1faf9 100644 --- a/lib/journal/src/concrete/tests.rs +++ b/lib/journal/src/concrete/tests.rs @@ -49,7 +49,7 @@ pub fn run_test<'a>(record: JournalEntry<'a>) { #[test] pub fn test_record_init_module() { run_test(JournalEntry::InitModuleV1 { - wasm_hash: vec![13u8; 8], + wasm_hash: Box::new([13u8; 8]), }); } diff --git a/lib/journal/src/entry.rs b/lib/journal/src/entry.rs index 9fe01815510..b5e09519c12 100644 --- a/lib/journal/src/entry.rs +++ b/lib/journal/src/entry.rs @@ -84,7 +84,7 @@ pub enum SocketOptTimeType { #[serde(rename_all = "snake_case")] pub enum JournalEntry<'a> { InitModuleV1 { - wasm_hash: Vec, + wasm_hash: Box<[u8]>, }, ClearEtherealV1, UpdateMemoryRegionV1 { diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 2a72d494c84..6eff4e5166b 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -285,7 +285,7 @@ impl WasiFunctionEnv { // The first event we save is an event that records the module hash. // Note: This is used to detect if an incorrect journal is used on the wrong // process or if a process has been recompiled - let wasm_hash = self.data(&store).process.module_hash.as_bytes().to_vec(); + let wasm_hash = Box::from(self.data(&store).process.module_hash.as_bytes()); let mut ctx = self.env.clone().into_mut(&mut store); crate::journal::JournalEffector::save_event( &mut ctx, diff --git a/lib/wasix/src/syscalls/journal/actions/init_module.rs b/lib/wasix/src/syscalls/journal/actions/init_module.rs index 6250d2cacad..a8a5b153299 100644 --- a/lib/wasix/src/syscalls/journal/actions/init_module.rs +++ b/lib/wasix/src/syscalls/journal/actions/init_module.rs @@ -4,7 +4,7 @@ impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { #[allow(clippy::result_large_err)] pub(crate) unsafe fn action_init_module( &mut self, - wasm_hash: Vec, + wasm_hash: Box<[u8]>, differ_ethereal: Option<&mut Vec>>, ) -> Result<(), WasiRuntimeError> { tracing::trace!("Replay journal - InitModule {:?}", wasm_hash); diff --git a/lib/wasix/src/syscalls/journal/mod.rs b/lib/wasix/src/syscalls/journal/mod.rs index b88c89091a7..847d28240d6 100644 --- a/lib/wasix/src/syscalls/journal/mod.rs +++ b/lib/wasix/src/syscalls/journal/mod.rs @@ -31,9 +31,9 @@ pub struct JournalSyscallPlayer<'a, 'c> { pub ctx: FunctionEnvMut<'c, WasiEnv>, pub bootstrapping: bool, - pub journal_module_hash: Option>, + pub journal_module_hash: Option>, pub rewind: Option, - pub cur_module_hash: Vec, + pub cur_module_hash: Box<[u8]>, pub real_fd: HashSet, // We delay the spawning of threads until the end as its @@ -52,7 +52,7 @@ pub struct JournalSyscallPlayer<'a, 'c> { impl<'a, 'c> JournalSyscallPlayer<'a, 'c> { pub fn new(mut ctx: FunctionEnvMut<'c, WasiEnv>, bootstrapping: bool) -> Self { - let cur_module_hash = ctx.data().process.module_hash.as_bytes().to_vec(); + let cur_module_hash: Box<[u8]> = Box::from(ctx.data().process.module_hash.as_bytes()); let mut ret = JournalSyscallPlayer { ctx, bootstrapping, From 1b28920b90388bd33e672ec4d25e43c8cae79fe3 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 30 Apr 2024 18:06:33 +0330 Subject: [PATCH 26/55] improvement on extracting the hash from atom signature --- lib/wasix/src/runtime/module_cache/mod.rs | 2 + lib/wasix/src/runtime/module_cache/types.rs | 34 ++++++++++++++ .../package_loader/load_package_tree.rs | 45 +++++++++---------- 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/lib/wasix/src/runtime/module_cache/mod.rs b/lib/wasix/src/runtime/module_cache/mod.rs index 88baf4293ac..bc0260dc72d 100644 --- a/lib/wasix/src/runtime/module_cache/mod.rs +++ b/lib/wasix/src/runtime/module_cache/mod.rs @@ -45,6 +45,8 @@ pub use self::{ types::{CacheError, ModuleCache, ModuleHash}, }; +pub(crate) use self::types::hash_from_signature; + #[cfg(feature = "sys-thread")] pub use self::filesystem::FileSystemCache; diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/wasix/src/runtime/module_cache/types.rs index 8d449fd0187..7ec5ef6caf0 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/wasix/src/runtime/module_cache/types.rs @@ -5,6 +5,8 @@ use std::{ path::PathBuf, }; +use anyhow::Context; +use base64::Engine as Base64Engine; use rand::RngCore; use sha2::Digest; use wasmer::{Engine, Module}; @@ -203,6 +205,38 @@ impl ModuleHash { } } +/// Loads the hash from an encoded string. This is not a public API since it relies on how we +/// encode the atom signature in pirita. +pub(crate) fn hash_from_signature(encoded: &str) -> Result { + if let Some(base64_encoded) = encoded.strip_prefix("xxhash:") { + let hash = base64::prelude::BASE64_STANDARD + .decode(base64_encoded) + .context("malformed base64 encoded hash")?; + + let hash: [u8; 8] = hash + .as_slice() + .try_into() + .context("xxhash hash must be 8 bytes")?; + + Ok(ModuleHash::xxhash_from_bytes(hash)) + } else if let Some(base64_encoded) = encoded.strip_prefix("sha256:") { + let hash = base64::prelude::BASE64_STANDARD + .decode(base64_encoded) + .context("malformed base64 encoded hash")?; + + let hash: [u8; 32] = hash + .as_slice() + .try_into() + .context("sha256 hash must be 32 bytes")?; + + Ok(ModuleHash::sha256_from_bytes(hash)) + } else { + return Err(anyhow::Error::msg( + "Only xxhash and sha256 encoded strings are accepted", + )); + } +} + impl Display for ModuleHash { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { fn format(f: &mut Formatter<'_>, bytes: &[u8; N]) -> fmt::Result { diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 5efa8d229ca..13d6512278a 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -6,7 +6,6 @@ use std::{ }; use anyhow::{Context, Error}; -use base64::Engine; use futures::{future::BoxFuture, StreamExt, TryStreamExt}; use once_cell::sync::OnceCell; use petgraph::visit::EdgeRef; @@ -20,7 +19,7 @@ use webc::{ use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, runtime::{ - module_cache::ModuleHash, + module_cache, package_loader::PackageLoader, resolver::{ DependencyGraph, ItemLocation, PackageSummary, Resolution, ResolvedFileSystemMapping, @@ -159,7 +158,7 @@ fn load_binary_command( return Ok(legacy_atom_hack(webc, command_name, cmd)); } - let hash = atom_hash(webc, &atom_name); + let hash = module_cache::hash_from_signature(&webc.manifest().atoms[&atom_name].signature)?; let atom = atom.with_context(|| { @@ -236,30 +235,26 @@ fn legacy_atom_hack( "(hack) The command metadata is malformed. Falling back to the first atom in the WEBC file", ); - let hash = atom_hash(webc, &name); + let hash = module_cache::hash_from_signature(&webc.manifest().atoms[&name].signature); - Some(BinaryPackageCommand::new( - command_name.to_string(), - metadata.clone(), - atom, - hash, - )) -} + match hash { + Ok(hash) => Some(BinaryPackageCommand::new( + command_name.to_string(), + metadata.clone(), + atom, + hash, + )), + Err(e) => { + tracing::debug!( + command_name, + atom.name = name.as_str(), + error = e.to_string(), + "Failed to get the atom hash from the manifest", + ); -fn atom_hash(webc: &Container, atom_name: &str) -> ModuleHash { - let base64_encoded = webc.manifest().atoms[atom_name] - .signature - .strip_prefix("sha256:") - .expect("malformed atom signature: does not have sha256 prefix"); - let sha256 = base64::prelude::BASE64_STANDARD - .decode(base64_encoded) - .expect("malformed base64 encoded hash"); - let hash: [u8; 32] = sha256 - .as_slice() - .try_into() - .expect("sha256 hash is not 32 bytes"); - - ModuleHash::sha256_from_bytes(hash) + None + } + } } async fn fetch_dependencies( From 2f911f4c868dc30ed965691ae0b8897b223dc8aa Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 30 Apr 2024 18:42:30 +0330 Subject: [PATCH 27/55] add test to pin the behavior --- lib/cli/src/commands/run/mod.rs | 2 - lib/wasix/src/bin_factory/binary_package.rs | 50 ++++++++++++++++++++- lib/wasix/src/runtime/mod.rs | 2 - 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index db33994ee66..468584cb9b6 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -705,7 +705,6 @@ impl ExecutableTarget { .load_module_sync(&wasm) .with_context(|| format!("Unable to compile \"{}\"", path.display()))?; - // FIXME: Amin Ok(ExecutableTarget::WebAssembly { module, module_hash: ModuleHash::xxhash(&wasm), @@ -718,7 +717,6 @@ impl ExecutableTarget { let module = unsafe { Module::deserialize_from_file(&engine, path)? }; let module_hash = { let wasm = std::fs::read(path)?; - // FIXME: Amin ModuleHash::xxhash(wasm) }; diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index ea903d8dec3..6abe0f9bbc6 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -15,7 +15,6 @@ use crate::{ Runtime, }; -// FIXME: Amin #[derive(Derivative, Clone)] #[derivative(Debug)] pub struct BinaryPackageCommand { @@ -162,6 +161,7 @@ impl BinaryPackage { #[cfg(test)] mod tests { + use sha2::Digest; use tempfile::TempDir; use virtual_fs::AsyncReadExt; @@ -228,4 +228,52 @@ mod tests { f.read_to_string(&mut buffer).await.unwrap(); assert_eq!(buffer, file_txt); } + + #[tokio::test] + #[cfg_attr( + not(feature = "sys-thread"), + ignore = "The tokio task manager isn't available on this platform" + )] + async fn commands_use_the_atom_signature() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "a dummy package" + + [[module]] + name = "foo" + source = "foo.wasm" + abi = "wasi" + + [[command]] + name = "cmd" + module = "foo" + "#; + let manifest = temp.path().join("wasmer.toml"); + std::fs::write(&manifest, wasmer_toml).unwrap(); + + let atom_path = temp.path().join("foo.wasm"); + std::fs::write(&atom_path, b"").unwrap(); + + let webc: Container = webc::wasmer_package::Package::from_manifest(manifest) + .unwrap() + .into(); + + let tasks = task_manager(); + let mut runtime = PluggableRuntime::new(tasks); + runtime.set_package_loader( + BuiltinPackageLoader::new() + .with_shared_http_client(runtime.http_client().unwrap().clone()), + ); + + let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap(); + + assert_eq!(pkg.commands.len(), 1); + let command = pkg.get_command("cmd").unwrap(); + let atom_sha256_hash: [u8; 32] = sha2::Sha256::digest(webc.get_atom("foo").unwrap()).into(); + let module_hash = ModuleHash::sha256_from_bytes(atom_sha256_hash); + assert_eq!(command.hash(), &module_hash); + } } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index c532aeb3d6e..4ec940347fb 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -104,7 +104,6 @@ where fn load_module<'a>(&'a self, wasm: &'a [u8]) -> BoxFuture<'a, Result> { let engine = self.engine(); let module_cache = self.module_cache(); - // FIXME: Amin let hash = ModuleHash::xxhash(wasm); let task = async move { load_module(&engine, &module_cache, wasm, hash).await }; @@ -554,7 +553,6 @@ impl Runtime for OverriddenRuntime { if self.engine.is_some() || self.module_cache.is_some() { let engine = self.engine(); let module_cache = self.module_cache(); - // FIXME: Amin let hash = ModuleHash::xxhash(wasm); let task = async move { load_module(&engine, &module_cache, wasm, hash).await }; From 5f6471c49d2ecc1137480589f736da2ea33c9fab Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 30 Apr 2024 18:44:09 +0330 Subject: [PATCH 28/55] remove lefover comment --- tests/lib/wast/src/wasi_wast.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index 9648c2624f2..9ceecd10127 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -121,7 +121,6 @@ impl<'a> WasiTest<'a> { wasm_module.read_to_end(&mut out)?; out }; - // FIXME: Amin? let module_hash = ModuleHash::xxhash(&wasm_bytes); let module = Module::new(store, wasm_bytes)?; From 3d0a4dfd1100ff6e5d55892ba13f48af2d7544aa Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Wed, 1 May 2024 18:00:19 +0330 Subject: [PATCH 29/55] use new webc API for accessing atom signature --- Cargo.lock | 7 ++-- Cargo.toml | 2 +- lib/wasix/src/runtime/module_cache/mod.rs | 2 - lib/wasix/src/runtime/module_cache/types.rs | 36 +++-------------- .../package_loader/load_package_tree.rs | 39 +++++++------------ 5 files changed, 25 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d600d27b43e..4d8396b8463 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5603,7 +5603,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "static_assertions", ] @@ -7145,9 +7145,9 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.0-alpha3" +version = "6.0.0-alpha5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c544aa307af3ad0326ae962a1715400c6c456e91e45bb2c2d860fdccc128be3c" +checksum = "c10e8cca5fe3a8f85803e8cf67bb4f166c9fb326d4581c26a1a48e22f7922762" dependencies = [ "anyhow", "base64 0.21.7", @@ -7160,6 +7160,7 @@ dependencies = [ "indexmap 1.9.3", "leb128", "lexical-sort", + "libc", "once_cell", "path-clean", "rand", diff --git a/Cargo.toml b/Cargo.toml index 961363f30e9..f302186d9ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.0.0-alpha3", default-features = false, features = ["package"] } +webc = { version = "6.0.0-alpha5", default-features = false, features = ["package"] } edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" diff --git a/lib/wasix/src/runtime/module_cache/mod.rs b/lib/wasix/src/runtime/module_cache/mod.rs index bc0260dc72d..88baf4293ac 100644 --- a/lib/wasix/src/runtime/module_cache/mod.rs +++ b/lib/wasix/src/runtime/module_cache/mod.rs @@ -45,8 +45,6 @@ pub use self::{ types::{CacheError, ModuleCache, ModuleHash}, }; -pub(crate) use self::types::hash_from_signature; - #[cfg(feature = "sys-thread")] pub use self::filesystem::FileSystemCache; diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/wasix/src/runtime/module_cache/types.rs index 7ec5ef6caf0..ea3dc6c27ff 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/wasix/src/runtime/module_cache/types.rs @@ -5,8 +5,6 @@ use std::{ path::PathBuf, }; -use anyhow::Context; -use base64::Engine as Base64Engine; use rand::RngCore; use sha2::Digest; use wasmer::{Engine, Module}; @@ -205,35 +203,11 @@ impl ModuleHash { } } -/// Loads the hash from an encoded string. This is not a public API since it relies on how we -/// encode the atom signature in pirita. -pub(crate) fn hash_from_signature(encoded: &str) -> Result { - if let Some(base64_encoded) = encoded.strip_prefix("xxhash:") { - let hash = base64::prelude::BASE64_STANDARD - .decode(base64_encoded) - .context("malformed base64 encoded hash")?; - - let hash: [u8; 8] = hash - .as_slice() - .try_into() - .context("xxhash hash must be 8 bytes")?; - - Ok(ModuleHash::xxhash_from_bytes(hash)) - } else if let Some(base64_encoded) = encoded.strip_prefix("sha256:") { - let hash = base64::prelude::BASE64_STANDARD - .decode(base64_encoded) - .context("malformed base64 encoded hash")?; - - let hash: [u8; 32] = hash - .as_slice() - .try_into() - .context("sha256 hash must be 32 bytes")?; - - Ok(ModuleHash::sha256_from_bytes(hash)) - } else { - return Err(anyhow::Error::msg( - "Only xxhash and sha256 encoded strings are accepted", - )); +impl From for ModuleHash { + fn from(value: webc::metadata::AtomSignature) -> Self { + match value { + webc::metadata::AtomSignature::Sha256(bytes) => ModuleHash::Sha256(bytes), + } } } diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 13d6512278a..a7c14e06b98 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -19,7 +19,6 @@ use webc::{ use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, runtime::{ - module_cache, package_loader::PackageLoader, resolver::{ DependencyGraph, ItemLocation, PackageSummary, Resolution, ResolvedFileSystemMapping, @@ -155,10 +154,10 @@ fn load_binary_command( if atom.is_none() && cmd.annotations.is_empty() { tracing::info!("applying legacy atom hack"); - return Ok(legacy_atom_hack(webc, command_name, cmd)); + return legacy_atom_hack(webc, command_name, cmd); } - let hash = module_cache::hash_from_signature(&webc.manifest().atoms[&atom_name].signature)?; + let hash = webc.manifest().atom_signature(&atom_name)?.into(); let atom = atom.with_context(|| { @@ -225,8 +224,12 @@ fn legacy_atom_hack( webc: &Container, command_name: &str, metadata: &webc::metadata::Command, -) -> Option { - let (name, atom) = webc.atoms().into_iter().next()?; +) -> Result, anyhow::Error> { + let (name, atom) = webc + .atoms() + .into_iter() + .next() + .ok_or_else(|| anyhow::Error::msg("container does not have any atom"))?; tracing::debug!( command_name, @@ -235,26 +238,14 @@ fn legacy_atom_hack( "(hack) The command metadata is malformed. Falling back to the first atom in the WEBC file", ); - let hash = module_cache::hash_from_signature(&webc.manifest().atoms[&name].signature); - - match hash { - Ok(hash) => Some(BinaryPackageCommand::new( - command_name.to_string(), - metadata.clone(), - atom, - hash, - )), - Err(e) => { - tracing::debug!( - command_name, - atom.name = name.as_str(), - error = e.to_string(), - "Failed to get the atom hash from the manifest", - ); + let hash = webc.manifest().atom_signature(&name)?.into(); - None - } - } + Ok(Some(BinaryPackageCommand::new( + command_name.to_string(), + metadata.clone(), + atom, + hash, + ))) } async fn fetch_dependencies( From 748b6b5142b33af6bffdf8b48bdc2a28d85bebad Mon Sep 17 00:00:00 2001 From: Johnathan Sharratt Date: Wed, 1 May 2024 08:06:47 +1000 Subject: [PATCH 30/55] Added a fix so that try_read calls close the connection without returning errors and causing a panic --- lib/virtual-net/src/tcp_pair.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/virtual-net/src/tcp_pair.rs b/lib/virtual-net/src/tcp_pair.rs index 61d622c613e..ee73b2989e2 100644 --- a/lib/virtual-net/src/tcp_pair.rs +++ b/lib/virtual-net/src/tcp_pair.rs @@ -250,7 +250,10 @@ impl SocketBuffer { } State::Dead => { tracing::trace!("try_read: socket is dead"); - return Err(NetworkError::ConnectionReset); + // Note: Returning `ConnectionReset` here may seem logical as other functions return this + // however this code path is not always handled properly. In particular `tokio` inside + // WASIX will panic if it receives this code. + return Ok(0); } State::Closed | State::Shutdown => { tracing::trace!("try_read: socket is closed or shutdown"); From 724d2cc48cbbca2a86a25be45072424c77363158 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 14:20:40 +0330 Subject: [PATCH 31/55] handle local package resolution --- lib/cli/src/commands/run/mod.rs | 11 +++---- lib/wasix/src/bin_factory/binary_package.rs | 33 ++++++++++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 468584cb9b6..2c7a226a8cc 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -673,15 +673,12 @@ impl ExecutableTarget { pb: &ProgressBar, ) -> Result { pb.set_message(format!("Loading \"{}\" into memory", dir.display())); - - let manifest_path = dir.join("wasmer.toml"); - let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?; - let container = Container::from(webc); - pb.set_message("Resolving dependencies"); let inner_runtime = runtime.clone(); - let pkg = runtime.task_manager().spawn_and_block_on(async move { - BinaryPackage::from_webc(&container, inner_runtime.as_ref()).await + let pkg = runtime.task_manager().spawn_and_block_on({ + let path = dir.to_path_buf(); + + async move { BinaryPackage::from_dir(&path, inner_runtime.as_ref()).await } })??; Ok(ExecutableTarget::Package(pkg)) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 6abe0f9bbc6..dbf2706ca7c 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -1,8 +1,9 @@ -use std::sync::Arc; +use std::{os::unix::ffi::OsStrExt, path::Path, sync::Arc}; use anyhow::Context; use derivative::*; use once_cell::sync::OnceCell; +use sha2::Digest; use virtual_fs::FileSystem; use wasmer_config::package::{PackageHash, PackageId, PackageSource}; use webc::{compat::SharedBytes, Container}; @@ -79,6 +80,36 @@ pub struct BinaryPackage { } impl BinaryPackage { + #[tracing::instrument(level = "debug", skip_all)] + pub async fn from_dir( + dir: &Path, + rt: &(dyn Runtime + Send + Sync), + ) -> Result { + let source = rt.source(); + + // since each package must be in its own directory, hash of the `dir` should provide a good enough + // unique identifier for the package + let hash = sha2::Sha256::digest(dir.as_os_str().as_bytes()).into(); + let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash)); + + let manifest_path = dir.join("wasmer.toml"); + let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?; + let container = Container::from(webc); + let manifest = container.manifest(); + + let root = PackageInfo::from_manifest(id, manifest, container.version())?; + let root_id = root.id.clone(); + + let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?; + let pkg = rt + .package_loader() + .load_package_tree(&container, &resolution) + .await + .map_err(|e| anyhow::anyhow!(e))?; + + Ok(pkg) + } + /// Load a [`webc::Container`] and all its dependencies into a /// [`BinaryPackage`]. #[tracing::instrument(level = "debug", skip_all)] From 41af2d8909d5b4d40d6c81bc4c5bf5453fb57bf3 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 14:39:14 +0330 Subject: [PATCH 32/55] fix platform dependent API usage --- lib/wasix/src/bin_factory/binary_package.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index dbf2706ca7c..e7d63c3e3d8 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -1,4 +1,4 @@ -use std::{os::unix::ffi::OsStrExt, path::Path, sync::Arc}; +use std::{path::Path, sync::Arc}; use anyhow::Context; use derivative::*; @@ -89,7 +89,7 @@ impl BinaryPackage { // since each package must be in its own directory, hash of the `dir` should provide a good enough // unique identifier for the package - let hash = sha2::Sha256::digest(dir.as_os_str().as_bytes()).into(); + let hash = sha2::Sha256::digest(dir.display().to_string().as_bytes()).into(); let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash)); let manifest_path = dir.join("wasmer.toml"); From ec7f2321c55536ad3eb30a4dc19d396a47179b39 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 14:48:06 +0330 Subject: [PATCH 33/55] use new webc version and fix publish code path --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/cli/src/commands/package/build.rs | 8 +++++--- lib/wasix/src/bin_factory/binary_package.rs | 4 +++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d8396b8463..f1e70ea0006 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7145,9 +7145,9 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.0-alpha5" +version = "6.0.0-alpha6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10e8cca5fe3a8f85803e8cf67bb4f166c9fb326d4581c26a1a48e22f7922762" +checksum = "f90e7ef808f844f15d1680a7cea4a0c5082ccaae0dc09621855a655f71989eb1" dependencies = [ "anyhow", "base64 0.21.7", diff --git a/Cargo.toml b/Cargo.toml index f302186d9ee..487a8a9a5fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.0.0-alpha5", default-features = false, features = ["package"] } +webc = { version = "6.0.0-alpha6", default-features = false, features = ["package"] } edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" diff --git a/lib/cli/src/commands/package/build.rs b/lib/cli/src/commands/package/build.rs index 67093906ce4..80db3505a4c 100644 --- a/lib/cli/src/commands/package/build.rs +++ b/lib/cli/src/commands/package/build.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use anyhow::Context; use dialoguer::console::{style, Emoji}; use indicatif::ProgressBar; +use sha2::Digest; use wasmer_config::package::PackageHash; use crate::utils::load_package_manifest; @@ -53,7 +54,10 @@ impl PackageBuild { ) }; let pkg = webc::wasmer_package::Package::from_manifest(manifest_path)?; - let pkg_hash = PackageHash::from_sha256_bytes(pkg.webc_hash()); + let data = pkg.serialize()?; + let hash = sha2::Sha256::digest(&data).into(); + + let pkg_hash = PackageHash::from_sha256_bytes(hash); let name = if let Some(manifest_pkg) = manifest.package { if let Some(name) = manifest_pkg.name { if let Some(version) = manifest_pkg.version { @@ -116,8 +120,6 @@ impl PackageBuild { ); } - let data = pkg.serialize()?; - pb.println(format!( "{} {}Writing package...", style("[3/3]").bold().dim(), diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index e7d63c3e3d8..6d729ead819 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -121,7 +121,9 @@ impl BinaryPackage { let manifest = container.manifest(); let id = PackageInfo::package_id_from_manifest(manifest)?.unwrap_or_else(|| { - PackageId::Hash(PackageHash::from_sha256_bytes(container.webc_hash())) + PackageId::Hash(PackageHash::from_sha256_bytes( + container.webc_hash().unwrap(), + )) }); let root = PackageInfo::from_manifest(id, manifest, container.version())?; From 41f380d89a10464ad2efe5e232627f6c90095f26 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 16:48:31 +0330 Subject: [PATCH 34/55] fix tests --- lib/wasix/src/bin_factory/binary_package.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 6d729ead819..dee299aefee 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -237,9 +237,6 @@ mod tests { std::fs::create_dir_all(&out).unwrap(); let file_txt = "Hello, World!"; std::fs::write(out.join("file.txt"), file_txt).unwrap(); - let webc: Container = webc::wasmer_package::Package::from_manifest(manifest) - .unwrap() - .into(); let tasks = task_manager(); let mut runtime = PluggableRuntime::new(tasks); runtime.set_package_loader( @@ -247,7 +244,9 @@ mod tests { .with_shared_http_client(runtime.http_client().unwrap().clone()), ); - let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap(); + let pkg = BinaryPackage::from_dir(&temp.path(), &runtime) + .await + .unwrap(); // We should have mapped "./out/file.txt" on the host to // "/public/file.txt" on the guest. @@ -290,7 +289,7 @@ mod tests { let atom_path = temp.path().join("foo.wasm"); std::fs::write(&atom_path, b"").unwrap(); - let webc: Container = webc::wasmer_package::Package::from_manifest(manifest) + let webc: Container = webc::wasmer_package::Package::from_manifest(&manifest) .unwrap() .into(); @@ -301,7 +300,9 @@ mod tests { .with_shared_http_client(runtime.http_client().unwrap().clone()), ); - let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap(); + let pkg = BinaryPackage::from_dir(&temp.path(), &runtime) + .await + .unwrap(); assert_eq!(pkg.commands.len(), 1); let command = pkg.get_command("cmd").unwrap(); From f5c5e427a813f96f298860dbc04b1c2a1c14b4da Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 22:03:52 +0330 Subject: [PATCH 35/55] return error upon hash absence --- lib/wasix/src/bin_factory/binary_package.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index dee299aefee..d73755f1da5 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -120,11 +120,13 @@ impl BinaryPackage { let source = rt.source(); let manifest = container.manifest(); - let id = PackageInfo::package_id_from_manifest(manifest)?.unwrap_or_else(|| { - PackageId::Hash(PackageHash::from_sha256_bytes( - container.webc_hash().unwrap(), - )) - }); + let id = PackageInfo::package_id_from_manifest(manifest)? + .or_else(|| { + container + .webc_hash() + .map(|hash| PackageId::Hash(PackageHash::from_sha256_bytes(hash))) + }) + .ok_or_else(|| anyhow::Error::msg("webc file did not provide its hash"))?; let root = PackageInfo::from_manifest(id, manifest, container.version())?; let root_id = root.id.clone(); From 0f2f558fa9acb8455cba80e53a26a707990e426c Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 17:31:18 +0200 Subject: [PATCH 36/55] [skip ci] new publish flow --- Cargo.lock | 666 ++++++++++-------- Cargo.toml | 2 +- lib/backend-api/schema.graphql | 98 ++- lib/backend-api/src/query.rs | 83 ++- lib/backend-api/src/types.rs | 95 ++- lib/cli/Cargo.toml | 141 ++-- lib/cli/src/commands/app/create.rs | 6 +- lib/cli/src/commands/app/deploy.rs | 40 +- lib/cli/src/commands/login.rs | 12 +- lib/cli/src/commands/mod.rs | 20 +- lib/cli/src/commands/package/build.rs | 11 +- lib/cli/src/commands/package/common/macros.rs | 79 +++ lib/cli/src/commands/package/common/mod.rs | 289 ++++++++ lib/cli/src/commands/package/common/wait.rs | 151 ++++ lib/cli/src/commands/package/mod.rs | 11 +- lib/cli/src/commands/package/publish.rs | 152 ++++ lib/cli/src/commands/package/push.rs | 227 ++++++ lib/cli/src/commands/package/tag.rs | 407 +++++++++++ lib/cli/src/commands/publish.rs | 252 ------- lib/cli/src/opts.rs | 33 + lib/registry/src/graphql/subscriptions.rs | 1 + lib/registry/src/subscriptions.rs | 2 + lib/wasix/src/bin_factory/binary_package.rs | 11 +- 23 files changed, 2103 insertions(+), 686 deletions(-) create mode 100644 lib/cli/src/commands/package/common/macros.rs create mode 100644 lib/cli/src/commands/package/common/mod.rs create mode 100644 lib/cli/src/commands/package/common/wait.rs create mode 100644 lib/cli/src/commands/package/publish.rs create mode 100644 lib/cli/src/commands/package/push.rs create mode 100644 lib/cli/src/commands/package/tag.rs delete mode 100644 lib/cli/src/commands/publish.rs diff --git a/Cargo.lock b/Cargo.lock index d600d27b43e..cab5806b5be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,9 +57,9 @@ checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "android-tzdata" @@ -84,47 +84,48 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -138,9 +139,9 @@ checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e" [[package]] name = "anyhow" -version = "1.0.81" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" [[package]] name = "arbitrary" @@ -206,9 +207,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86a9249d1447a85f95810c620abea82e001fe58a31713fcce614caf52499f905" +checksum = "4e9eabd7a98fe442131a17c316bd9349c43695e49e730c3c8e12cfb5f4da2693" dependencies = [ "flate2", "futures-core", @@ -219,13 +220,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -252,9 +253,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" @@ -283,6 +284,18 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bincode" version = "1.3.3" @@ -377,9 +390,9 @@ checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" [[package]] name = "bumpalo" -version = "3.15.4" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytecheck" @@ -463,7 +476,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "thiserror", @@ -495,12 +508,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.90" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -515,18 +529,24 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chrono" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "wasm-bindgen", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -553,7 +573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" dependencies = [ "ciborium-io", - "half 2.4.0", + "half 2.4.1", ] [[package]] @@ -597,7 +617,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -617,9 +637,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "colored" @@ -657,9 +677,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" +checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" dependencies = [ "crossterm", "strum", @@ -1031,7 +1051,7 @@ dependencies = [ "bitflags 2.5.0", "crossterm_winapi", "libc", - "parking_lot 0.12.1", + "parking_lot 0.12.2", "winapi 0.3.9", ] @@ -1087,32 +1107,43 @@ dependencies = [ [[package]] name = "cynic-codegen" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac40efa3e97024222fa95d4c85ce093e2337ed5cdf7279374777132b419f50c" +checksum = "ac766f4d59eff577638395a36be631a098ab764f839f96b3dce814d738bd9efa" dependencies = [ "counter", + "cynic-parser", "darling 0.20.8", - "graphql-parser", "once_cell", "ouroboros", "proc-macro2", "quote", "strsim", - "syn 2.0.58", + "syn 2.0.61", "thiserror", ] +[[package]] +name = "cynic-parser" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411ab40d3f72ca1442d50a60e572838e5a2f0719b93a77a29647117ffdf67687" +dependencies = [ + "indexmap 2.2.6", + "lalrpop-util", + "logos", +] + [[package]] name = "cynic-proc-macros" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4890059f47b6864d53231fd7d336893fc4a28bc1fff62c3ad4966a82994b84" +checksum = "1a0c8aa1aa2e669424cfdd4a0cfbaaf5c07620bfa7f6ecee6cbc71c13f75ceae" dependencies = [ "cynic-codegen", "darling 0.20.8", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -1160,7 +1191,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -1182,7 +1213,7 @@ checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ "darling_core 0.20.8", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -1192,17 +1223,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if 1.0.0", - "hashbrown 0.14.3", - "lock_api 0.4.11", + "hashbrown 0.14.5", + "lock_api 0.4.12", "once_cell", - "parking_lot_core 0.9.9", + "parking_lot_core 0.9.10", ] [[package]] name = "data-encoding" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "deranged" @@ -1233,7 +1264,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -1419,7 +1450,7 @@ dependencies = [ "anyhow", "bytesize", "once_cell", - "parking_lot 0.12.1", + "parking_lot 0.12.2", "rand_chacha", "rand_core", "schemars", @@ -1428,7 +1459,7 @@ dependencies = [ "serde_path_to_error", "serde_yaml 0.8.26", "sparx", - "time 0.3.34", + "time 0.3.36", "url", "uuid", "wcgi-host", @@ -1443,7 +1474,7 @@ dependencies = [ "anyhow", "bytesize", "once_cell", - "parking_lot 0.12.1", + "parking_lot 0.12.2", "rand_chacha", "rand_core", "schemars", @@ -1452,7 +1483,7 @@ dependencies = [ "serde_path_to_error", "serde_yaml 0.8.26", "sparx", - "time 0.3.34", + "time 0.3.36", "url", "uuid", "wcgi-host", @@ -1484,9 +1515,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "encode_unicode" @@ -1496,9 +1527,9 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if 1.0.0", ] @@ -1533,7 +1564,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -1554,7 +1585,7 @@ dependencies = [ "darling 0.20.8", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -1596,9 +1627,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fern" @@ -1640,9 +1671,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -1790,7 +1821,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -1853,9 +1884,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1872,7 +1903,7 @@ checksum = "b0e085ded9f1267c32176b40921b9754c474f7dd96f7e808d4a982e48aa1e854" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -2033,9 +2064,9 @@ checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" [[package]] name = "half" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eceaaeec696539ddaf7b333340f1af35a5aa87ae3e4f3ead0532f72affab2e" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if 1.0.0", "crunchy", @@ -2090,9 +2121,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.11", "allocator-api2", @@ -2104,7 +2135,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" dependencies = [ - "hashbrown 0.14.3", + "hashbrown 0.14.5", ] [[package]] @@ -2229,7 +2260,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.6", + "socket2 0.5.7", "tokio 1.37.0", "tower-service", "tracing", @@ -2245,7 +2276,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.11", + "rustls 0.21.12", "tokio 1.37.0", "tokio-rustls", ] @@ -2366,7 +2397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown 0.14.5", "serde", ] @@ -2394,7 +2425,7 @@ dependencies = [ "libc", "llvm-sys", "once_cell", - "parking_lot 0.12.1", + "parking_lot 0.12.2", ] [[package]] @@ -2519,6 +2550,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itertools" version = "0.10.5" @@ -2545,9 +2582,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] @@ -2571,6 +2608,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "lalrpop-util" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" +dependencies = [ + "regex-automata 0.4.6", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2607,9 +2653,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "libfuzzer-sys" @@ -2680,7 +2726,7 @@ dependencies = [ "lazy_static", "libc", "regex", - "semver 1.0.22", + "semver 1.0.23", ] [[package]] @@ -2694,9 +2740,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -2717,6 +2763,38 @@ version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +[[package]] +name = "logos" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c000ca4d908ff18ac99b93a062cb8958d331c3220719c52e77cb19cc6ac5d2c1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc487311295e0002e452025d6b580b77bb17286de87b57138f3b5db711cded68" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax 0.6.29", + "syn 2.0.61", +] + +[[package]] +name = "logos-derive" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbfc0d229f1f42d790440136d941afd806bc9e949e2bcb8faa813b0f00d1267e" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lz4_flex" version = "0.11.3" @@ -2738,11 +2816,11 @@ dependencies = [ [[package]] name = "mac_address" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4863ee94f19ed315bf3bc00299338d857d4b5bc856af375cc97d237382ad3856" +checksum = "5aa12182b93606fff55b70a5cfe6130eaf7407c2ea4f2c2bcc8b113b67c9928f" dependencies = [ - "nix 0.23.2", + "nix 0.28.0", "winapi 0.3.9", ] @@ -2840,15 +2918,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "memoffset" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.7.1" @@ -3003,28 +3072,28 @@ dependencies = [ [[package]] name = "nix" -version = "0.23.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", - "cc", "cfg-if 1.0.0", "libc", - "memoffset 0.6.5", + "memoffset 0.7.1", + "pin-utils", ] [[package]] name = "nix" -version = "0.26.4" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "cfg-if 1.0.0", + "cfg_aliases", "libc", - "memoffset 0.7.1", - "pin-utils", + "memoffset 0.9.1", ] [[package]] @@ -3065,11 +3134,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -3091,9 +3159,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] @@ -3223,7 +3291,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -3266,7 +3334,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -3288,12 +3356,12 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ - "lock_api 0.4.11", - "parking_lot_core 0.9.9", + "lock_api 0.4.12", + "parking_lot_core 0.9.10", ] [[package]] @@ -3313,22 +3381,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.4.1", + "redox_syscall 0.5.1", "smallvec 1.13.2", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "path-clean" @@ -3360,9 +3428,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" +checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" dependencies = [ "memchr", "thiserror", @@ -3371,9 +3439,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73541b156d32197eecda1a4014d7f868fd2bcb3c550d5386087cfba442bf69c" +checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" dependencies = [ "pest", "pest_generator", @@ -3381,22 +3449,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c35eeed0a3fab112f75165fdc026b3913f4183133f19b49be773ac9ea966e8bd" +checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] name = "pest_meta" -version = "2.7.9" +version = "2.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2adbf29bb9776f28caece835398781ab24435585fe0d4dc1374a61db5accedca" +checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" dependencies = [ "once_cell", "pest", @@ -3430,7 +3498,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -3565,9 +3633,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.79" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b" dependencies = [ "unicode-ident", ] @@ -3580,7 +3648,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", "version_check", "yansi 1.0.1", ] @@ -3648,9 +3716,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -3726,6 +3794,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "redox_users" version = "0.4.5" @@ -3739,22 +3816,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4846d4c50d1721b1a3bef8af76924eef20d5e723647333798c1b519b3a9473f" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fddb4f8d99b0a2ebafc65a87a69a7b9875e4b1ae1f00db265d300ef7f28bccc" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -3867,7 +3944,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-pemfile", "serde", "serde_json", @@ -3936,9 +4013,9 @@ dependencies = [ [[package]] name = "rmp" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" dependencies = [ "byteorder", "num-traits", @@ -4021,7 +4098,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.22", + "semver 1.0.23", ] [[package]] @@ -4038,9 +4115,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.32" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.5.0", "errno", @@ -4051,9 +4128,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log 0.4.21", "ring", @@ -4070,7 +4147,7 @@ dependencies = [ "log 0.4.21", "ring", "rustls-pki-types", - "rustls-webpki 0.102.2", + "rustls-webpki 0.102.3", "subtle", "zeroize", ] @@ -4098,9 +4175,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.4.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" +checksum = "51f344d206c5e1b010eec27349b815a4805f70a778895959d70b74b9b529b30a" [[package]] name = "rustls-webpki" @@ -4114,9 +4191,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.2" +version = "0.102.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" dependencies = [ "ring", "rustls-pki-types", @@ -4125,9 +4202,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "092474d1a01ea8278f69e6a358998405fae5b8b963ddaeb2b0b04a128bf1dfb0" [[package]] name = "rusty_jsc" @@ -4185,9 +4262,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "salsa20" @@ -4218,9 +4295,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" +checksum = "fc6e7ed6919cb46507fb01ff1654309219f62b4d603822501b0b80d42f6f21ef" dependencies = [ "dyn-clone", "schemars_derive", @@ -4232,14 +4309,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" +checksum = "185f2b7aa7e02d418e453790dde16890256bbd2bcd04b7dc5348811052b53f49" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 1.0.109", + "syn 2.0.61", ] [[package]] @@ -4289,11 +4366,11 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -4302,9 +4379,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -4312,9 +4389,9 @@ dependencies = [ [[package]] name = "self_cell" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bf37232d3bb9a2c4e641ca2a11d83b5062066f88df7fed36c28772046d65ba" +checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" [[package]] name = "semver" @@ -4336,9 +4413,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ "serde", ] @@ -4360,9 +4437,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.197" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" dependencies = [ "serde_derive", ] @@ -4399,24 +4476,24 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] name = "serde_derive_internals" -version = "0.26.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +checksum = "330f01ce65a3a5fe59a60c82f3c9a024b573b8a6e875bd233fe5f934e71d54e3" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.61", ] [[package]] @@ -4496,7 +4573,7 @@ dependencies = [ "futures 0.3.30", "lazy_static", "log 0.4.21", - "parking_lot 0.12.1", + "parking_lot 0.12.2", "serial_test_derive", ] @@ -4508,7 +4585,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -4560,9 +4637,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -4632,9 +4709,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4655,7 +4732,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api 0.4.11", + "lock_api 0.4.12", ] [[package]] @@ -4678,21 +4755,21 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "strum" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" [[package]] name = "strum_macros" -version = "0.25.3" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -4714,9 +4791,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.58" +version = "2.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687" +checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9" dependencies = [ "proc-macro2", "quote", @@ -4846,9 +4923,9 @@ dependencies = [ [[package]] name = "test-log" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b319995299c65d522680decf80f2c108d85b861d81dfe340a10d16cee29d9e6" +checksum = "3dffced63c2b5c7be278154d76b479f9f9920ed34e7574201407f0b14e2bbb93" dependencies = [ "test-log-macros", "tracing-subscriber", @@ -4856,13 +4933,13 @@ dependencies = [ [[package]] name = "test-log-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8f546451eaa38373f549093fe9fd05e7d2bade739e2ddf834b9968621d60107" +checksum = "5999e24eaa32083191ba4e425deb75cdf25efefabe5aaccb7446dd0d4122a3f5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -4880,22 +4957,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -4927,9 +5004,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -4948,9 +5025,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -5032,7 +5109,7 @@ dependencies = [ "num_cpus", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.6", + "socket2 0.5.7", "tokio-macros", "windows-sys 0.48.0", ] @@ -5117,7 +5194,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -5155,7 +5232,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.11", + "rustls 0.21.12", "tokio 1.37.0", ] @@ -5262,7 +5339,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log 0.4.21", - "rustls 0.21.11", + "rustls 0.21.12", "rustls-native-certs", "tokio 1.37.0", "tokio-rustls", @@ -5305,16 +5382,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes 1.6.0", "futures-core", "futures-sink", "pin-project-lite", "tokio 1.37.0", - "tracing", ] [[package]] @@ -5348,7 +5424,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.9", + "toml_edit 0.22.12", ] [[package]] @@ -5375,15 +5451,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.9" +version = "0.22.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e40bb779c5187258fd7aad0eb68cb8706a0a81fa712fbea808ab43c4b8374c4" +checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.5", + "winnow 0.6.8", ] [[package]] @@ -5454,7 +5530,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -5551,12 +5627,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "trybuild" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ad7eb6319ebadebca3dacf1f85a93bc54b73dd81b9036795f73de7ddfe27d5a" +checksum = "bfe21c256d6fba8499cf9d9b1c24971bec43a369d81c52e024adc7670cf112df" dependencies = [ "glob", - "once_cell", "serde", "serde_derive", "serde_json", @@ -5590,7 +5665,7 @@ dependencies = [ "httparse", "log 0.4.21", "rand", - "rustls 0.21.11", + "rustls 0.21.12", "sha1", "thiserror", "url", @@ -5603,7 +5678,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "static_assertions", ] @@ -5681,9 +5756,9 @@ checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6" [[package]] name = "unicode-xid" @@ -5720,17 +5795,17 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.9.6" +version = "2.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f214ce18d8b2cbe84ed3aa6486ed3f5b285cf8d8fbdbce9f3f767a724adc35" +checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "flate2", "log 0.4.21", "once_cell", "rustls 0.22.4", "rustls-pki-types", - "rustls-webpki 0.102.2", + "rustls-webpki 0.102.3", "url", "webpki-roots 0.26.1", ] @@ -6064,7 +6139,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", "wasm-bindgen-shared", ] @@ -6098,7 +6173,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6131,7 +6206,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] @@ -6264,7 +6339,7 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "time 0.3.34", + "time 0.3.36", "tokio 1.37.0", "tracing", "url", @@ -6411,6 +6486,7 @@ dependencies = [ "indicatif", "interfaces", "is-terminal", + "lazy_static", "libc", "log 0.4.21", "mac_address", @@ -6419,14 +6495,14 @@ dependencies = [ "object 0.32.2", "once_cell", "opener", - "parking_lot 0.12.1", + "parking_lot 0.12.2", "pathdiff", "predicates 3.1.0", "pretty_assertions", "regex", "reqwest", "rkyv", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "serde_yaml 0.9.34+deprecated", @@ -6437,7 +6513,7 @@ dependencies = [ "tempfile", "thiserror", "time 0.1.45", - "time 0.3.34", + "time 0.3.36", "tldextract", "tokio 1.37.0", "tokio-tungstenite", @@ -6553,7 +6629,7 @@ dependencies = [ "rayon", "regex", "rustc_version 0.4.0", - "semver 1.0.22", + "semver 1.0.23", "smallvec 1.13.2", "target-lexicon 0.12.14", "wasmer-compiler", @@ -6591,7 +6667,7 @@ dependencies = [ "indexmap 2.2.6", "pretty_assertions", "schemars", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_cbor", "serde_json", @@ -6604,17 +6680,17 @@ dependencies = [ [[package]] name = "wasmer-config" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b405c9856adaf65ee91eeeeaac6fcc6b127188648c60ae4e89de63f506c74e6" +checksum = "7439f813ad16b3fc8cbf2be3b54192e2c4b0a4c213d1fb3d7cf82da249c39aea" dependencies = [ "anyhow", "bytesize", "derive_builder", "hex", - "indexmap 1.9.3", + "indexmap 2.2.6", "schemars", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_cbor", "serde_json", @@ -6645,7 +6721,7 @@ dependencies = [ "lazy_static", "libc", "log 0.4.21", - "time 0.3.34", + "time 0.3.36", "wasmer", "wasmer-types", ] @@ -6784,13 +6860,13 @@ dependencies = [ "reqwest", "rpassword", "rusqlite", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_json", "tar", "tempfile", "thiserror", - "time 0.3.34", + "time 0.3.36", "tldextract", "tokio 1.37.0", "tokio-tungstenite", @@ -6900,7 +6976,7 @@ dependencies = [ "reqwest", "rkyv", "rusty_pool", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_cbor", "serde_derive", @@ -6956,7 +7032,7 @@ dependencies = [ "num_enum", "pretty_assertions", "serde", - "time 0.3.34", + "time 0.3.36", "tracing", "wai-bindgen-gen-core", "wai-bindgen-gen-rust", @@ -7054,7 +7130,7 @@ checksum = "9dbe55c8f9d0dbd25d9447a5a889ff90c0cc3feaa7395310d3d826b2c703eaab" dependencies = [ "bitflags 2.5.0", "indexmap 2.2.6", - "semver 1.0.22", + "semver 1.0.23", ] [[package]] @@ -7145,9 +7221,9 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.0-alpha3" +version = "6.0.0-alpha6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c544aa307af3ad0326ae962a1715400c6c456e91e45bb2c2d860fdccc128be3c" +checksum = "f90e7ef808f844f15d1680a7cea4a0c5082ccaae0dc09621855a655f71989eb1" dependencies = [ "anyhow", "base64 0.21.7", @@ -7160,10 +7236,11 @@ dependencies = [ "indexmap 1.9.3", "leb128", "lexical-sort", + "libc", "once_cell", "path-clean", "rand", - "semver 1.0.22", + "semver 1.0.23", "serde", "serde_cbor", "serde_json", @@ -7174,7 +7251,7 @@ dependencies = [ "thiserror", "toml 0.7.8", "url", - "wasmer-config 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-config 0.1.1", ] [[package]] @@ -7239,11 +7316,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi 0.3.9", + "windows-sys 0.52.0", ] [[package]] @@ -7258,7 +7335,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -7289,7 +7366,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -7309,17 +7386,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -7330,9 +7408,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -7348,9 +7426,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -7366,9 +7444,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -7384,9 +7468,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -7402,9 +7486,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -7414,9 +7498,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -7432,9 +7516,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" @@ -7447,9 +7531,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dffa400e67ed5a4dd237983829e66475f0a4a26938c4b04c21baede6262215b8" +checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" dependencies = [ "memchr", ] @@ -7523,22 +7607,22 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.58", + "syn 2.0.61", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 961363f30e9..487a8a9a5fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.0.0-alpha3", default-features = false, features = ["package"] } +webc = { version = "6.0.0-alpha6", default-features = false, features = ["package"] } edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" diff --git a/lib/backend-api/schema.graphql b/lib/backend-api/schema.graphql index 82df13b7bda..8c94716758f 100644 --- a/lib/backend-api/schema.graphql +++ b/lib/backend-api/schema.graphql @@ -361,7 +361,7 @@ type Package implements Likeable & Node & PackageOwner { """List of app templates for this package""" appTemplates(offset: Int, before: String, after: String, first: Int, last: Int): AppTemplateConnection! packagewebcSet(offset: Int, before: String, after: String, first: Int, last: Int): PackageWebcConnection! - versions: [PackageVersion]! + versions: [PackageVersion] collectionSet: [Collection!]! categories(offset: Int, before: String, after: String, first: Int, last: Int): CategoryConnection! keywords(offset: Int, before: String, after: String, first: Int, last: Int): PackageKeywordConnection! @@ -504,7 +504,7 @@ scalar JSONString type WebcImage implements Node { """The ID of the object""" id: ID! - version: RegistryWebcImageVersionChoices! + version: WebcVersion """""" fileSize: BigInt! @@ -518,11 +518,8 @@ type WebcImage implements Node { webcUrl: String! } -enum RegistryWebcImageVersionChoices { - """v2""" +enum WebcVersion { V2 - - """v3""" V3 } @@ -645,7 +642,9 @@ type DeployAppVersion implements Node { last: Int ): LogConnection! usageMetrics(forRange: MetricRange!, variant: MetricType!): [UsageMetric]! - sourcePackageVersion: PackageVersion! + sourcePackageVersion: PackageVersion + sourcePackageRelease: PackageWebc + sourcePackage: Package! aggregateMetrics: AggregateMetrics! volumes: [AppVersionVolume] favicon: URL @@ -806,6 +805,24 @@ enum LogStream { RUNTIME } +type PackageWebc implements Node & PackageReleaseInterface & PackageInstance { + """The ID of the object""" + id: ID! + createdAt: DateTime! + updatedAt: DateTime! + package: Package! + webc: WebcImage + piritaManifest: JSONString + piritaOffsets: JSONString + piritaVolumes: JSONString + isArchived: Boolean! + clientName: String + publishedBy: User! + webcV3: WebcImage + tag: String! + webcUrl: String! +} + type AppVersionVolume { name: String! mountPaths: [AppVersionVolumeMountPath]! @@ -1052,11 +1069,6 @@ type PackageDistribution { webcVersion: WebcVersion } -enum WebcVersion { - V2 - V3 -} - type PackageVersionFilesystem { wasm: String! host: String! @@ -1235,24 +1247,6 @@ type PackageWebcEdge { cursor: String! } -type PackageWebc implements Node & PackageReleaseInterface & PackageInstance { - """The ID of the object""" - id: ID! - createdAt: DateTime! - updatedAt: DateTime! - package: Package! - webc: WebcImage - piritaManifest: JSONString - piritaOffsets: JSONString - piritaVolumes: JSONString - isArchived: Boolean! - clientName: String - publishedBy: User! - webcV3: WebcImage - tag: String! - webcUrl: String! -} - type Collection { slug: String! displayName: String! @@ -2834,6 +2828,8 @@ type Mutation { mfa2EmailGetToken(input: MFAGenerateEmailOTPInput!): MFAEmailGenerationResponse publishPublicKey(input: PublishPublicKeyInput!): PublishPublicKeyPayload publishPackage(input: PublishPackageInput!): PublishPackagePayload + pushPackageRelease(input: PushPackageReleaseInput!): PushPackageReleasePayload + tagPackageRelease(input: TagPackageReleaseInput!): TagPackageReleasePayload updatePackage(input: UpdatePackageInput!): UpdatePackagePayload likePackage(input: LikePackageInput!): LikePackagePayload unlikePackage(input: UnlikePackageInput!): UnlikePackagePayload @@ -3586,6 +3582,48 @@ enum UploadFormat { webcv3 } +type PushPackageReleasePayload { + success: Boolean! + packageWebc: PackageWebc + clientMutationId: String +} + +input PushPackageReleaseInput { + namespace: String! + signedUrl: String! + name: String + + """Whether the package is private""" + private: Boolean = false + clientMutationId: String +} + +type TagPackageReleasePayload { + success: Boolean! + packageVersion: PackageVersion + clientMutationId: String +} + +input TagPackageReleaseInput { + packageReleaseId: ID! + name: String! + version: String! + manifest: String! + namespace: String + description: String + license: String + licenseFile: String + readme: String + repository: String + homepage: String + signature: InputSignature + + """The package icon""" + icon: String + private: Boolean = false + clientMutationId: String +} + type UpdatePackagePayload { package: Package! clientMutationId: String diff --git a/lib/backend-api/src/query.rs b/lib/backend-api/src/query.rs index 058af2968ed..c5867b2c772 100644 --- a/lib/backend-api/src/query.rs +++ b/lib/backend-api/src/query.rs @@ -13,8 +13,9 @@ use crate::{ types::{ self, CreateNamespaceVars, DeployApp, DeployAppConnection, DeployAppVersion, DeployAppVersionConnection, DnsDomain, GetCurrentUserWithAppsVars, GetDeployAppAndVersion, - GetDeployAppVersionsVars, GetNamespaceAppsVars, Log, LogStream, PackageVersionConnection, - PublishDeployAppVars, UpsertDomainFromZoneFileVars, + GetDeployAppVersionsVars, GetNamespaceAppsVars, GetSignedUrlForPackageUploadVariables, Log, + LogStream, PackageVersionConnection, PublishDeployAppVars, PushPackageReleasePayload, + SignedUrl, TagPackageReleasePayload, UpsertDomainFromZoneFileVars, }, GraphQLApiFailure, WasmerClient, }; @@ -54,6 +55,84 @@ pub async fn fetch_webc_package( webc::compat::Container::from_bytes(data).context("failed to parse webc package") } +/// Get a signed URL to upload packages. +pub async fn get_signed_url_for_package_upload( + client: &WasmerClient, + expires_after_seconds: Option, + filename: Option<&str>, + name: Option<&str>, + version: Option<&str>, +) -> Result, anyhow::Error> { + client + .run_graphql(types::GetSignedUrlForPackageUpload::build( + GetSignedUrlForPackageUploadVariables { + expires_after_seconds, + filename, + name, + version, + }, + )) + .await + .map(|r| r.get_signed_url_for_package_upload) +} +/// Push a package to the registry. +pub async fn push_package_release( + client: &WasmerClient, + name: Option<&str>, + namespace: &str, + signed_url: &str, + private: Option, +) -> Result, anyhow::Error> { + client + .run_graphql_strict(types::PushPackageRelease::build( + types::PushPackageReleaseVariables { + name, + namespace, + private, + signed_url, + }, + )) + .await + .map(|r| r.push_package_release) +} + +#[allow(clippy::too_many_arguments)] +pub async fn tag_package_release( + client: &WasmerClient, + description: Option<&str>, + homepage: Option<&str>, + license: Option<&str>, + license_file: Option<&str>, + manifest: &str, + name: &str, + namespace: Option<&str>, + package_release_id: &cynic::Id, + private: Option, + readme: Option<&str>, + repository: Option<&str>, + version: &str, +) -> Result, anyhow::Error> { + client + .run_graphql_strict(types::TagPackageRelease::build( + types::TagPackageReleaseVariables { + description, + homepage, + license, + license_file, + manifest, + name, + namespace, + package_release_id, + private, + readme, + repository, + version, + }, + )) + .await + .map(|r| r.tag_package_release) +} + /// Get the currently logged in used, together with all accessible namespaces. /// /// You can optionally filter the namespaces by the user role. diff --git a/lib/backend-api/src/types.rs b/lib/backend-api/src/types.rs index d7097f3a604..ca6d8800c19 100644 --- a/lib/backend-api/src/types.rs +++ b/lib/backend-api/src/types.rs @@ -83,21 +83,6 @@ mod queries { V3, } - #[derive(cynic::Enum, Clone, Copy, Debug)] - pub enum RegistryWebcImageVersionChoices { - V2, - V3, - } - - impl From for WebcVersion { - fn from(v: RegistryWebcImageVersionChoices) -> Self { - match v { - RegistryWebcImageVersionChoices::V2 => WebcVersion::V2, - RegistryWebcImageVersionChoices::V3 => WebcVersion::V3, - } - } - } - #[derive(cynic::QueryFragment, Debug, Clone, Serialize)] pub struct WebcImage { pub created_at: DateTime, @@ -106,7 +91,7 @@ mod queries { pub webc_sha256: String, pub file_size: BigInt, pub manifest: JSONString, - pub version: RegistryWebcImageVersionChoices, + pub version: Option, } #[derive(cynic::QueryFragment, Debug, Clone, Serialize)] @@ -184,6 +169,61 @@ mod queries { Oldest, } + #[derive(cynic::QueryVariables, Debug)] + pub struct PushPackageReleaseVariables<'a> { + pub name: Option<&'a str>, + pub namespace: &'a str, + pub private: Option, + pub signed_url: &'a str, + } + + #[derive(cynic::QueryFragment, Debug)] + #[cynic(graphql_type = "Mutation", variables = "PushPackageReleaseVariables")] + pub struct PushPackageRelease { + #[arguments(input: { name: $name, namespace: $namespace, private: $private, signedUrl: $signed_url })] + pub push_package_release: Option, + } + + #[derive(cynic::QueryFragment, Debug)] + pub struct PushPackageReleasePayload { + pub package_webc: Option, + pub success: bool, + } + + #[derive(cynic::QueryVariables, Debug)] + pub struct TagPackageReleaseVariables<'a> { + pub description: Option<&'a str>, + pub homepage: Option<&'a str>, + pub license: Option<&'a str>, + pub license_file: Option<&'a str>, + pub manifest: &'a str, + pub name: &'a str, + pub namespace: Option<&'a str>, + pub package_release_id: &'a cynic::Id, + pub private: Option, + pub readme: Option<&'a str>, + pub repository: Option<&'a str>, + pub version: &'a str, + } + + #[derive(cynic::QueryFragment, Debug)] + #[cynic(graphql_type = "Mutation", variables = "TagPackageReleaseVariables")] + pub struct TagPackageRelease { + #[arguments(input: { description: $description, homepage: $homepage, license: $license, licenseFile: $license_file, manifest: $manifest, name: $name, namespace: $namespace, packageReleaseId: $package_release_id, private: $private, readme: $readme, repository: $repository, version: $version })] + pub tag_package_release: Option, + } + + #[derive(cynic::QueryFragment, Debug)] + pub struct TagPackageReleasePayload { + pub success: bool, + } + + #[derive(cynic::InputObject, Debug)] + pub struct InputSignature<'a> { + pub public_key_key_id: &'a str, + pub data: &'a str, + } + #[derive(cynic::QueryVariables, Debug, Clone, Default)] pub struct AllPackageVersionsVars { pub offset: Option, @@ -253,6 +293,29 @@ mod queries { } } + #[derive(cynic::QueryVariables, Debug)] + pub struct GetSignedUrlForPackageUploadVariables<'a> { + pub expires_after_seconds: Option, + pub filename: Option<&'a str>, + pub name: Option<&'a str>, + pub version: Option<&'a str>, + } + + #[derive(cynic::QueryFragment, Debug)] + #[cynic( + graphql_type = "Query", + variables = "GetSignedUrlForPackageUploadVariables" + )] + pub struct GetSignedUrlForPackageUpload { + #[arguments(name: $name, version: $version, filename: $filename, expiresAfterSeconds: $expires_after_seconds)] + pub get_signed_url_for_package_upload: Option, + } + + #[derive(cynic::QueryFragment, Debug)] + pub struct SignedUrl { + pub url: String, + } + #[derive(cynic::QueryFragment, Debug)] pub struct PackageWebcConnection { pub page_info: PageInfo, diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 3f5488eb811..faf56f22e69 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -29,17 +29,24 @@ required-features = ["headless"] # Don't add the compiler features in default, please add them on the Makefile # since we might want to autoconfigure them depending on the availability on the host. default = [ - "sys", - "wat", - "wast", - "compiler", - "journal", - "wasmer-artifact-create", - "static-artifact-create", + "sys", + "wat", + "wast", + "compiler", + "journal", + "wasmer-artifact-create", + "static-artifact-create", ] # Tun-tap client for connecting to Wasmer Edge VPNs -tun-tap = ["dep:tun-tap", "virtual-net/tokio-tungstenite", "tokio-tungstenite", "mio", "futures-util", "mac_address", "dep:interfaces"] +tun-tap = [ + "dep:tun-tap", + "virtual-net/tokio-tungstenite", + "tokio-tungstenite", + "mio", + "mac_address", + "dep:interfaces", +] journal = ["wasmer-wasix/journal"] fuse = ["dep:fuse", "dep:time01", "dep:shared-buffer", "dep:rkyv"] backend = [] @@ -49,27 +56,62 @@ jsc = ["backend", "wasmer/jsc", "wasmer/std"] wast = ["wasmer-wast"] host-net = ["virtual-net/host-net"] wat = ["wasmer/wat"] -compiler = ["backend", "wasmer/compiler", "wasmer-compiler/translator", "wasmer-compiler/compiler"] -wasmer-artifact-create = ["compiler", "wasmer/wasmer-artifact-load", "wasmer/wasmer-artifact-create", "wasmer-compiler/wasmer-artifact-load", "wasmer-compiler/wasmer-artifact-create", "wasmer-object"] -static-artifact-create = ["compiler", "wasmer/static-artifact-load", "wasmer/static-artifact-create", "wasmer-compiler/static-artifact-load", "wasmer-compiler/static-artifact-create", "wasmer-object"] -wasmer-artifact-load = ["compiler", "wasmer/wasmer-artifact-load", "wasmer-compiler/wasmer-artifact-load"] -static-artifact-load = ["compiler", "wasmer/static-artifact-load", "wasmer-compiler/static-artifact-load"] +compiler = [ + "backend", + "wasmer/compiler", + "wasmer-compiler/translator", + "wasmer-compiler/compiler", +] +wasmer-artifact-create = [ + "compiler", + "wasmer/wasmer-artifact-load", + "wasmer/wasmer-artifact-create", + "wasmer-compiler/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-create", + "wasmer-object", +] +static-artifact-create = [ + "compiler", + "wasmer/static-artifact-load", + "wasmer/static-artifact-create", + "wasmer-compiler/static-artifact-load", + "wasmer-compiler/static-artifact-create", + "wasmer-object", +] +wasmer-artifact-load = [ + "compiler", + "wasmer/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-load", +] +static-artifact-load = [ + "compiler", + "wasmer/static-artifact-load", + "wasmer-compiler/static-artifact-load", +] singlepass = ["wasmer-compiler-singlepass", "compiler"] cranelift = ["wasmer-compiler-cranelift", "compiler"] llvm = ["wasmer-compiler-llvm", "compiler"] -disable-all-logging = ["wasmer-wasix/disable-all-logging", "log/release_max_level_off"] +disable-all-logging = [ + "wasmer-wasix/disable-all-logging", + "log/release_max_level_off", +] headless = [] headless-minimal = ["headless", "disable-all-logging"] # Optional -enable-serde = ["wasmer/enable-serde", "wasmer-vm/enable-serde", "wasmer-compiler/enable-serde", "wasmer-wasix/enable-serde"] +enable-serde = [ + "wasmer/enable-serde", + "wasmer-vm/enable-serde", + "wasmer-compiler/enable-serde", + "wasmer-wasix/enable-serde", +] [dependencies] # Repo-local dependencies. wasmer = { version = "=4.3.0-alpha.1", path = "../api", default-features = false } wasmer-compiler = { version = "=4.3.0-alpha.1", path = "../compiler", features = [ - "compiler", + "compiler", ], optional = true } wasmer-compiler-cranelift = { version = "=4.3.0-alpha.1", path = "../compiler-cranelift", optional = true } wasmer-compiler-singlepass = { version = "=4.3.0-alpha.1", path = "../compiler-singlepass", optional = true } @@ -77,25 +119,25 @@ wasmer-compiler-llvm = { version = "=4.3.0-alpha.1", path = "../compiler-llvm", wasmer-emscripten = { version = "=4.3.0-alpha.1", path = "../emscripten" } wasmer-vm = { version = "=4.3.0-alpha.1", path = "../vm", optional = true } wasmer-wasix = { workspace = true, features = [ - "logging", - "webc_runner_rt_wcgi", - "webc_runner_rt_dcgi", - "webc_runner_rt_dproxy", - "webc_runner_rt_emscripten", - "host-fs", - "ctrlc" + "logging", + "webc_runner_rt_wcgi", + "webc_runner_rt_dcgi", + "webc_runner_rt_dproxy", + "webc_runner_rt_emscripten", + "host-fs", + "ctrlc", ] } wasmer-wast = { version = "=4.3.0-alpha.1", path = "../../tests/lib/wast", optional = true } wasmer-types = { version = "=4.3.0-alpha.1", path = "../types", features = [ - "enable-serde", + "enable-serde", ] } wasmer-registry = { version = "5.11.0", path = "../registry", features = [ - "build-package", - "clap", + "build-package", + "clap", ] } wasmer-object = { version = "=4.3.0-alpha.1", path = "../object", optional = true } virtual-fs = { version = "0.11.3", path = "../virtual-fs", default-features = false, features = [ - "host-fs", + "host-fs", ] } virtual-net = { version = "0.6.5", path = "../virtual-net" } virtual-mio = { version = "0.3.1", path = "../virtual-io" } @@ -106,6 +148,7 @@ webc = { workspace = true } wasmer-api = { version = "=0.0.26", path = "../backend-api" } edge-schema.workspace = true edge-util = { version = "=0.1.0" } +lazy_static = "1.4.0" # Used by the mount command @@ -152,7 +195,11 @@ sha2 = "0.10.6" object = "0.32.0" wasm-coredump-builder = { version = "0.1.11", optional = true } tracing = { version = "0.1" } -tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "json"] } +tracing-subscriber = { version = "0.3", features = [ + "env-filter", + "fmt", + "json", +] } async-trait = "0.1.68" tokio = { version = "1.28.1", features = ["macros", "rt-multi-thread"] } once_cell = "1.17.1" @@ -167,15 +214,15 @@ interfaces = { version = "0.0.9", optional = true } uuid = { version = "1.3.0", features = ["v4"] } time = { version = "0.3.17", features = ["macros"] } -serde_yaml = {workspace = true} +serde_yaml = { workspace = true } comfy-table = "7.0.1" # Used by tuntap and connect -futures-util = { version = "0.3", optional = true } +futures-util = "0.3" mio = { version = "0.8", optional = true } tokio-tungstenite = { version = "0.20.1", features = [ - "rustls-tls-webpki-roots", + "rustls-tls-webpki-roots", ], optional = true } mac_address = { version = "1.1.5", optional = true } tun-tap = { version = "0.1.3", features = ["tokio"], optional = true } @@ -186,34 +233,34 @@ tun-tap = { version = "0.1.3", features = ["tokio"], optional = true } clap = { version = "4.3.0-alpha.1", features = ["derive", "env"] } [target.'cfg(target_family = "wasm")'.dependencies] clap = { version = "4.3.0-alpha.1", default-features = false, features = [ - "std", - "help", - "usage", - "error-context", - "suggestions", - "derive", - "env", + "std", + "help", + "usage", + "error-context", + "suggestions", + "derive", + "env", ] } [target.'cfg(not(target_arch = "riscv64"))'.dependencies] reqwest = { version = "^0.11", default-features = false, features = [ - "rustls-tls", - "json", - "multipart", - "gzip", + "rustls-tls", + "json", + "multipart", + "gzip", ] } [target.'cfg(target_arch = "riscv64")'.dependencies] reqwest = { version = "^0.11", default-features = false, features = [ - "native-tls", - "json", - "multipart", + "native-tls", + "json", + "multipart", ] } [build-dependencies] chrono = { version = "^0.4", default-features = false, features = [ - "std", - "clock", + "std", + "clock", ] } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/lib/cli/src/commands/app/create.rs b/lib/cli/src/commands/app/create.rs index 517c3f1b196..9e8f33894aa 100644 --- a/lib/cli/src/commands/app/create.rs +++ b/lib/cli/src/commands/app/create.rs @@ -2,7 +2,7 @@ use crate::{ commands::AsyncCliCommand, - opts::{ApiOpts, ItemFormatOpts}, + opts::{ApiOpts, ItemFormatOpts, WasmerEnv}, utils::{ load_package_manifest, package_wizard::{CreateMode, PackageType, PackageWizard}, @@ -83,6 +83,9 @@ pub struct CmdAppCreate { #[allow(missing_docs)] pub api: ApiOpts, + #[clap(flatten)] + pub env: WasmerEnv, + #[clap(flatten)] #[allow(missing_docs)] pub fmt: ItemFormatOpts, @@ -364,6 +367,7 @@ impl CmdAppCreate { { let cmd_deploy = CmdAppDeploy { api: self.api.clone(), + env: self.env.clone(), fmt: ItemFormatOpts { format: self.fmt.format, }, diff --git a/lib/cli/src/commands/app/deploy.rs b/lib/cli/src/commands/app/deploy.rs index 934c3caa059..576b0c2f320 100644 --- a/lib/cli/src/commands/app/deploy.rs +++ b/lib/cli/src/commands/app/deploy.rs @@ -1,7 +1,7 @@ use super::AsyncCliCommand; use crate::{ - commands::{app::create::CmdAppCreate, Publish}, - opts::{ApiOpts, ItemFormatOpts}, + commands::{app::create::CmdAppCreate, package::publish::PackagePublish, PublishWait}, + opts::{ApiOpts, ItemFormatOpts, WasmerEnv}, utils::load_package_manifest, }; use anyhow::Context; @@ -18,7 +18,6 @@ use wasmer_config::{ app::AppConfigV1, package::{PackageIdent, PackageSource}, }; -use wasmer_registry::wasmer_env::{WasmerEnv, WASMER_DIR}; /// Deploy an app to Wasmer Edge. #[derive(clap::Parser, Debug)] @@ -26,6 +25,9 @@ pub struct CmdAppDeploy { #[clap(flatten)] pub api: ApiOpts, + #[clap(flatten)] + pub env: WasmerEnv, + #[clap(flatten)] pub fmt: ItemFormatOpts, @@ -96,27 +98,18 @@ impl CmdAppDeploy { ), }; - let env = WasmerEnv::new( - if let Ok(dir) = std::env::var("WASMER_DIR") { - PathBuf::from(dir) - } else { - WASMER_DIR.clone() - }, - self.api.registry.clone().map(|u| u.to_string().into()), - self.api.token.clone(), - None, - ); - - let publish_cmd = Publish { - env, + let publish_cmd = PackagePublish { + env: self.env.clone(), dry_run: false, quiet: false, package_name: None, - version: None, + package_version: None, no_validate: false, - package_path: Some(manifest_dir_path.to_str().unwrap().to_string()), - wait: !self.no_wait, - wait_all: !self.no_wait, + package_path: manifest_dir_path.clone(), + wait: match self.no_wait { + true => PublishWait::None, + false => PublishWait::Container, + }, timeout: humantime::Duration::from_str("2m").unwrap(), package_namespace: match manifest.package { Some(_) => None, @@ -124,12 +117,10 @@ impl CmdAppDeploy { }, non_interactive: self.non_interactive, bump: self.bump, + api: self.api.clone(), }; - match publish_cmd.run_async().await? { - Some(id) => Ok(id), - None => anyhow::bail!("Error while publishing package. Stopping."), - } + publish_cmd.run_async().await } async fn get_owner( @@ -188,6 +179,7 @@ impl CmdAppDeploy { app_name: None, no_wait: self.no_wait, api: self.api.clone(), + env: self.env.clone(), fmt: ItemFormatOpts { format: self.fmt.format, }, diff --git a/lib/cli/src/commands/login.rs b/lib/cli/src/commands/login.rs index 7e5555c3b79..7986dbd2a9f 100644 --- a/lib/cli/src/commands/login.rs +++ b/lib/cli/src/commands/login.rs @@ -105,7 +105,7 @@ pub struct Login { pub token: Option, /// The directory cached artefacts are saved to. #[clap(long, env = "WASMER_CACHE_DIR")] - cache_dir: Option, + pub cache_dir: Option, } impl Login { @@ -225,9 +225,7 @@ impl Login { Ok((listener, server_url)) } - /// execute [List] - #[tokio::main] - pub async fn execute(&self) -> Result<(), anyhow::Error> { + pub async fn run_async(&self) -> Result<(), anyhow::Error> { let env = self.wasmer_env(); let registry = env.registry_endpoint()?; @@ -324,6 +322,12 @@ impl Login { }; Ok(()) } + + /// execute [List] + #[tokio::main] + pub async fn execute(&self) -> Result<(), anyhow::Error> { + self.run_async().await + } } async fn preflight(req: Request) -> Result, anyhow::Error> { diff --git a/lib/cli/src/commands/mod.rs b/lib/cli/src/commands/mod.rs index 52ba7b1c20c..ca1ad56defb 100644 --- a/lib/cli/src/commands/mod.rs +++ b/lib/cli/src/commands/mod.rs @@ -23,7 +23,6 @@ mod journal; mod login; pub(crate) mod namespace; mod package; -mod publish; mod run; mod self_update; pub mod ssh; @@ -134,6 +133,9 @@ impl WasmerCmd { Some(Cmd::Package(cmd)) => match cmd { Package::Download(cmd) => cmd.execute(), Package::Build(cmd) => cmd.execute().map(|_| ()), + Package::Tag(cmd) => cmd.run(), + Package::Push(cmd) => cmd.run(), + Package::Publish(cmd) => cmd.run().map(|_| ()), }, Some(Cmd::Container(cmd)) => match cmd { crate::commands::Container::Unpack(cmd) => cmd.execute(), @@ -214,9 +216,9 @@ enum Cmd { /// Login into a wasmer.io-like registry Login(Login), - /// Login into a wasmer.io-like registry + /// Publish a package to a registry [alias: package publish] #[clap(name = "publish")] - Publish(Publish), + Publish(crate::commands::package::publish::PackagePublish), /// Wasmer cache Cache(Cache), @@ -325,10 +327,10 @@ enum Cmd { /// Shows the current logged in user for the current active registry Whoami(Whoami), - /// Add a Wasmer package's bindings to your application. + /// Add a Wasmer package's bindings to your application Add(Add), - /// Run a WebAssembly file or Wasmer container. + /// Run a WebAssembly file or Wasmer container #[clap(alias = "run-unstable")] Run(Run), @@ -344,17 +346,17 @@ enum Cmd { Container(crate::commands::Container), // Edge commands - /// Deploy apps to Wasmer Edge. [alias: app deploy] + /// Deploy apps to Wasmer Edge [alias: app deploy] Deploy(crate::commands::app::deploy::CmdAppDeploy), - /// Manage deployed Edge apps. + /// Manage deployed Edge apps #[clap(subcommand, alias = "apps")] App(crate::commands::app::CmdApp), - /// Run commands/packages on Wasmer Edge in an interactive shell session. + /// Run commands/packages on Wasmer Edge in an interactive shell session Ssh(crate::commands::ssh::CmdSsh), - /// Manage Wasmer namespaces. + /// Manage Wasmer namespaces #[clap(subcommand, alias = "namespaces")] Namespace(crate::commands::namespace::CmdNamespace), diff --git a/lib/cli/src/commands/package/build.rs b/lib/cli/src/commands/package/build.rs index 67093906ce4..1e092889e7d 100644 --- a/lib/cli/src/commands/package/build.rs +++ b/lib/cli/src/commands/package/build.rs @@ -4,6 +4,7 @@ use anyhow::Context; use dialoguer::console::{style, Emoji}; use indicatif::ProgressBar; use wasmer_config::package::PackageHash; +use webc::wasmer_package::Package; use crate::utils::load_package_manifest; @@ -44,7 +45,7 @@ impl PackageBuild { } } - pub(crate) fn execute(&self) -> Result { + pub(crate) fn execute(&self) -> Result { let manifest_path = self.manifest_path()?; let Some((_, manifest)) = load_package_manifest(&manifest_path)? else { anyhow::bail!( @@ -53,7 +54,9 @@ impl PackageBuild { ) }; let pkg = webc::wasmer_package::Package::from_manifest(manifest_path)?; - let pkg_hash = PackageHash::from_sha256_bytes(pkg.webc_hash()); + let pkg_hash = PackageHash::from_sha256_bytes(pkg.webc_hash().ok_or(anyhow::anyhow!( + "No webc hash was provided while trying to build" + ))?); let name = if let Some(manifest_pkg) = manifest.package { if let Some(name) = manifest_pkg.name { if let Some(version) = manifest_pkg.version { @@ -84,7 +87,7 @@ impl PackageBuild { // rest of the code writes the package to disk and is irrelevant // to checking. if self.check { - return Ok(pkg_hash); + return Ok(pkg); } pb.println(format!( @@ -133,7 +136,7 @@ impl PackageBuild { out_path.display() )); - Ok(pkg_hash) + Ok(pkg) } fn manifest_path(&self) -> Result { diff --git a/lib/cli/src/commands/package/common/macros.rs b/lib/cli/src/commands/package/common/macros.rs new file mode 100644 index 00000000000..318efa7e667 --- /dev/null +++ b/lib/cli/src/commands/package/common/macros.rs @@ -0,0 +1,79 @@ +macro_rules! make_pb { + ($self:ident, $msg:expr) => {{ + let pb = indicatif::ProgressBar::new_spinner(); + if $self.quiet { + pb.set_draw_target(indicatif::ProgressDrawTarget::hidden()); + } + + pb.enable_steady_tick(std::time::Duration::from_millis(500)); + pb.set_style( + indicatif::ProgressStyle::with_template("{spinner:.magenta} {msg}") + .unwrap() + .tick_strings(&["✶", "✸", "✹", "✺", "✹", "✷"]), + ); + + pb.set_message($msg); + pb + }}; + + ($self:ident, $msg:expr, $($spinner:expr),+) => {{ + let pb = indicatif::ProgressBar::new_spinner(); + if $self.quiet { + pb.set_draw_target(indicatif::ProgressDrawTarget::hidden()); + } + + pb.enable_steady_tick(std::time::Duration::from_millis(500)); + pb.set_style( + indicatif::ProgressStyle::with_template("{spinner:.magenta} {msg}") + .unwrap() + .tick_strings(&[$($spinner),+]), + ); + + pb.set_message($msg); + pb + }}; +} + +macro_rules! pb_ok { + ($pb:expr, $msg: expr) => { + $pb.set_style( + indicatif::ProgressStyle::with_template(&format!("{} {{msg}}", "✔".green().bold())) + .unwrap(), + ); + $pb.finish_with_message(format!("{}", $msg.bold())); + }; +} + +macro_rules! pb_err { + ($pb:expr, $msg: expr) => { + $pb.set_style( + indicatif::ProgressStyle::with_template(&format!("{} {{msg}}", "✘".red().bold())) + .unwrap(), + ); + $pb.finish_with_message(format!("{}", $msg.bold())); + }; +} + +macro_rules! bin_name { + () => { + match std::env::args().nth(0) { + Some(n) => n, + None => String::from("wasmer"), + } + }; +} + +macro_rules! cli_line { + () => { + std::env::args() + .filter(|s| !s.starts_with("-")) + .collect::>() + .join(" ") + }; +} + +pub(crate) use bin_name; +pub(crate) use cli_line; +pub(crate) use make_pb; +pub(crate) use pb_err; +pub(crate) use pb_ok; diff --git a/lib/cli/src/commands/package/common/mod.rs b/lib/cli/src/commands/package/common/mod.rs new file mode 100644 index 00000000000..47eb5e7f4e5 --- /dev/null +++ b/lib/cli/src/commands/package/common/mod.rs @@ -0,0 +1,289 @@ +use crate::{ + commands::Login, + opts::{ApiOpts, WasmerEnv}, + utils::load_package_manifest, +}; +use colored::Colorize; +use dialoguer::Confirm; +use semver::VersionReq; +use std::{collections::BTreeMap, path::PathBuf, str::FromStr}; +use wasmer_api::WasmerClient; +use wasmer_config::package::{Manifest, NamedPackageIdent, PackageHash, PackageIdent}; +use webc::wasmer_package::Package; + +pub mod macros; +pub mod wait; + +// We have PackageId and PackageIdent.. Brace yourselves, here we have their intertransmutunion, +// the PackageSpecifier. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum PackageSpecifier { + Hash { + namespace: String, + hash: PackageHash, + }, + Named { + namespace: String, + name: String, + tag: Tag, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum Tag { + Version(semver::Version), // <-- This is the reason.. + Hash(PackageHash), +} + +impl Into for PackageSpecifier { + fn into(self) -> PackageIdent { + match self { + PackageSpecifier::Hash { hash, .. } => PackageIdent::Hash(hash), + PackageSpecifier::Named { + namespace, + name, + tag, + } => match tag { + Tag::Version(v) => PackageIdent::Named(NamedPackageIdent { + registry: None, + namespace: Some(namespace), + name, + tag: Some(wasmer_config::package::Tag::VersionReq( + VersionReq::parse(&v.to_string()).unwrap(), + )), + }), + Tag::Hash(h) => PackageIdent::Named(NamedPackageIdent { + registry: None, + namespace: Some(namespace), + name, + tag: Some(wasmer_config::package::Tag::Named(h.to_string())), + }), + }, + } + } +} + +pub(super) fn into_specifier( + manifest: &Manifest, + hash: &PackageHash, + namespace: String, +) -> anyhow::Result { + Ok(match &manifest.package { + None => PackageSpecifier::Hash { + namespace, + hash: hash.clone(), + }, + Some(n) => match &n.name { + Some(name) => { + let named = NamedPackageIdent::from_str(&name)?; + match &n.version { + Some(v) => PackageSpecifier::Named { + namespace, + name: named.name.clone(), + tag: Tag::Version(v.clone()), + }, + None => PackageSpecifier::Named { + namespace, + name: named.name.clone(), + tag: Tag::Hash(hash.clone()), + }, + } + } + None => PackageSpecifier::Hash { + namespace, + hash: hash.clone(), + }, + }, + }) +} + +pub(super) fn on_error(e: anyhow::Error) -> anyhow::Error { + #[cfg(feature = "telemetry")] + sentry::integrations::anyhow::capture_anyhow(&e); + + e +} + +// HACK: We want to invalidate the cache used for GraphQL queries so +// the current user sees the results of publishing immediately. There +// are cleaner ways to achieve this, but for now we're just going to +// clear out the whole GraphQL query cache. +// See https://github.com/wasmerio/wasmer/pull/3983 for more +pub(super) fn invalidate_graphql_query_cache(cache_dir: &PathBuf) -> Result<(), anyhow::Error> { + let cache_dir = cache_dir.join("queries"); + std::fs::remove_dir_all(cache_dir)?; + + Ok(()) +} + +// Upload a package to a signed url. +pub(super) async fn upload( + client: &WasmerClient, + hash: &PackageHash, + timeout: humantime::Duration, + package: &Package, +) -> anyhow::Result { + let hash_str = hash.to_string(); + + let url = { + let default_timeout_secs = Some(60 * 30); + let q = wasmer_api::query::get_signed_url_for_package_upload( + &client, + default_timeout_secs, + Some(&hash_str), + None, + None, + ); + + match q.await? { + Some(u) => u.url, + None => anyhow::bail!( + "The backend did not provide a valid signed URL to upload the package" + ), + } + }; + + tracing::info!("signed url is: {url}"); + + let client = reqwest::Client::builder() + .default_headers(reqwest::header::HeaderMap::default()) + .timeout(timeout.into()) + .build() + .unwrap(); + + let res = client + .post(&url) + .header(reqwest::header::CONTENT_LENGTH, "0") + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header("x-goog-resumable", "start"); + + let result = res.send().await?; + + if result.status() != reqwest::StatusCode::from_u16(201).unwrap() { + return Err(anyhow::anyhow!( + "Uploading package failed: got HTTP {:?} when uploading", + result.status() + )); + } + + let headers = result + .headers() + .into_iter() + .filter_map(|(k, v)| { + let k = k.to_string(); + let v = v.to_str().ok()?.to_string(); + Some((k.to_lowercase(), v)) + }) + .collect::>(); + + let session_uri = headers + .get("location") + .ok_or_else(|| { + anyhow::anyhow!("The upload server did not provide the upload URL correctly") + })? + .clone(); + + tracing::info!("session uri is: {session_uri}"); + /* XXX: If the package is large this line may result in + * a surge in memory use. + * + * In the future, we might want a way to stream bytes + * from the webc instead of a complete in-memory + * representation. + */ + let bytes = package.serialize()?; + + let total_bytes = bytes.len(); + let chunk_size = 1_048_576; // 1MB - 315s / 100MB + let mut chunks = bytes.chunks(chunk_size); + let mut total_bytes_sent = 0; + + let client = reqwest::Client::builder().build().unwrap(); + + while let Some(chunk) = chunks.next() { + // TODO: add upload pbar. + + let n = chunk.len(); + + let start = total_bytes_sent; + let end = start + chunk.len().saturating_sub(1); + let content_range = format!("bytes {start}-{end}/{total_bytes}"); + + let res = client + .put(&session_uri) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .header(reqwest::header::CONTENT_LENGTH, format!("{}", chunk.len())) + .header("Content-Range".to_string(), content_range) + .body(chunk.to_vec()); + + res.send() + .await + .map(|response| response.error_for_status()) + .map_err(|e| { + anyhow::anyhow!("cannot send request to {session_uri} (chunk {start}..{end}): {e}",) + })??; + + total_bytes_sent += n; + } + + Ok(url) +} + +/// Read and return a manifest given a path. +/// +// The difference with the `load_package_manifest` is that +// this function returns an error if no manifest is found. +pub(super) fn get_manifest(path: &PathBuf) -> anyhow::Result<(PathBuf, Manifest)> { + load_package_manifest(&path).and_then(|j| { + j.ok_or_else(|| anyhow::anyhow!("No valid manifest found in path '{}'", path.display())) + }) +} + +pub(super) async fn login_user( + api: &ApiOpts, + env: &WasmerEnv, + interactive: bool, + msg: &str, +) -> anyhow::Result { + if let Ok(client) = api.client() { + return Ok(client); + } + + let theme = dialoguer::theme::ColorfulTheme::default(); + + if api.token.is_none() { + if interactive { + eprintln!( + "{}: You need to be logged in to {msg}.", + "WARN".yellow().bold() + ); + + if Confirm::with_theme(&theme) + .with_prompt("Do you want to login now?") + .interact()? + { + Login { + no_browser: false, + wasmer_dir: env.wasmer_dir.clone(), + registry: api + .registry + .clone() + .map(|l| wasmer_registry::wasmer_env::Registry::from(l.to_string())), + token: api.token.clone(), + cache_dir: Some(env.cache_dir.clone()), + } + .run_async() + .await?; + // self.api = ApiOpts::default(); + } else { + anyhow::bail!("Stopping the push flow as the user is not logged in.") + } + } else { + let bin_name = self::macros::bin_name!(); + eprintln!("You are not logged in. Use the `--token` flag or log in (use `{bin_name} login`) to {msg}."); + anyhow::bail!("Stopping execution as the user is not logged in.") + } + } + + api.client() +} diff --git a/lib/cli/src/commands/package/common/wait.rs b/lib/cli/src/commands/package/common/wait.rs new file mode 100644 index 00000000000..21c6e8809d0 --- /dev/null +++ b/lib/cli/src/commands/package/common/wait.rs @@ -0,0 +1,151 @@ +use super::macros::*; +use colored::Colorize; +use futures_util::StreamExt; +use indicatif::ProgressBar; +use wasmer_api::WasmerClient; + +/// Different conditions that can be "awaited" when publishing a package. +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +pub enum PublishWait { + None, + Container, + NativeExecutables, + Bindings, + All, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WaitPackageState { + pub container: bool, + pub native_executables: bool, + pub bindings: bool, +} + +impl WaitPackageState { + pub fn is_any(self) -> bool { + self.container || self.native_executables || self.bindings + } + + pub fn new_none() -> Self { + Self { + container: false, + native_executables: false, + bindings: false, + } + } + + pub fn new_all() -> Self { + Self { + container: true, + native_executables: true, + bindings: true, + } + } + + pub fn new_container() -> Self { + Self { + container: true, + native_executables: false, + bindings: false, + } + } + + pub fn new_exe() -> Self { + Self { + container: true, + native_executables: true, + bindings: false, + } + } + + pub fn new_bindings() -> Self { + Self { + container: true, + native_executables: false, + bindings: true, + } + } +} + +impl From for WaitPackageState { + fn from(value: PublishWait) -> Self { + match value { + PublishWait::None => Self::new_none(), + PublishWait::Container => Self::new_container(), + PublishWait::NativeExecutables => Self::new_exe(), + PublishWait::Bindings => Self::new_bindings(), + PublishWait::All => Self::new_all(), + } + } +} + +pub async fn wait_package( + client: &WasmerClient, + to_wait: PublishWait, + package_version_id: wasmer_api::types::Id, + pb: &ProgressBar, + timeout: humantime::Duration, +) -> anyhow::Result<()> { + if let PublishWait::None = to_wait { + return Ok(()); + } + + pb.set_message("Waiting for package to become available..."); + let registry_url = client.graphql_endpoint().to_string(); + let login_token = client.auth_token().unwrap_or_default().to_string(); + let package_version_id = package_version_id.into_inner(); + + let (mut stream, _) = wasmer_registry::subscriptions::subscribe_package_version_ready( + ®istry_url, + &login_token, + &package_version_id, + ) + .await?; + + let mut state: WaitPackageState = to_wait.into(); + + let deadline: std::time::Instant = + std::time::Instant::now() + std::time::Duration::from_secs(timeout.as_secs()); + + loop { + if !state.is_any() { + break; + } + + if std::time::Instant::now() > deadline { + return Err(anyhow::anyhow!( + "Timed out waiting for package version to become ready" + )); + } + + let data = match tokio::time::timeout_at(deadline.into(), stream.next()).await { + Err(_) => { + return Err(anyhow::anyhow!( + "Timed out waiting for package version to become ready" + )); + } + Ok(None) => { + break; + } + Ok(Some(data)) => data, + }; + + if let Some(data) = data.unwrap().data { + match data.package_version_ready.state { + wasmer_registry::subscriptions::PackageVersionState::WEBC_GENERATED => { + state.container = false + } + wasmer_registry::subscriptions::PackageVersionState::BINDINGS_GENERATED => { + state.bindings = false + } + wasmer_registry::subscriptions::PackageVersionState::NATIVE_EXES_GENERATED => { + state.native_executables = true + } + wasmer_registry::subscriptions::PackageVersionState::Other(_) => {} + } + } + } + + pb_ok!(pb, "Package is available!"); + Ok(()) +} diff --git a/lib/cli/src/commands/package/mod.rs b/lib/cli/src/commands/package/mod.rs index 0d616bc74bd..700ab1be823 100644 --- a/lib/cli/src/commands/package/mod.rs +++ b/lib/cli/src/commands/package/mod.rs @@ -1,8 +1,12 @@ mod build; +mod common; mod download; +pub mod publish; +mod push; +mod tag; pub use build::PackageBuild; -pub use download::PackageDownload; +pub use common::wait::PublishWait; /// Package related commands. #[derive(clap::Subcommand, Debug)] @@ -10,6 +14,9 @@ pub use download::PackageDownload; // the command struct. #[allow(missing_docs)] pub enum Package { - Download(PackageDownload), + Download(download::PackageDownload), Build(build::PackageBuild), + Tag(tag::PackageTag), + Push(push::PackagePush), + Publish(publish::PackagePublish), } diff --git a/lib/cli/src/commands/package/publish.rs b/lib/cli/src/commands/package/publish.rs new file mode 100644 index 00000000000..f1d668f0ad1 --- /dev/null +++ b/lib/cli/src/commands/package/publish.rs @@ -0,0 +1,152 @@ +use crate::{ + commands::{ + package::{ + common::{macros::*, wait::*, *}, + push::PackagePush, + tag::PackageTag, + }, + AsyncCliCommand, + }, + opts::{ApiOpts, WasmerEnv}, +}; +use colored::Colorize; +use is_terminal::IsTerminal; +use std::path::PathBuf; +use wasmer_config::package::PackageIdent; + +/// Publish (push and tag) a package to the registry. +#[derive(Debug, clap::Parser)] +pub struct PackagePublish { + #[clap(flatten)] + pub api: ApiOpts, + + #[clap(flatten)] + pub env: WasmerEnv, + + /// Run the publish logic without sending anything to the registry server + #[clap(long, name = "dry-run")] + pub dry_run: bool, + + /// Run the publish command without any output + #[clap(long)] + pub quiet: bool, + + /// Override the namespace of the package to upload + #[clap(long = "namespace")] + pub package_namespace: Option, + + /// Override the name of the package to upload + #[clap(long = "name")] + pub package_name: Option, + + /// Override the package version of the uploaded package in the wasmer.toml + #[clap(long = "version")] + pub package_version: Option, + + /// Skip validation of the uploaded package + #[clap(long)] + pub no_validate: bool, + + /// Directory containing the `wasmer.toml`, or a custom *.toml manifest file. + /// + /// Defaults to current working directory. + #[clap(name = "path", default_value = ".")] + pub package_path: PathBuf, + + /// Wait for package to be available on the registry before exiting. + #[clap( + long, + require_equals = true, + num_args = 0..=1, + default_value_t = PublishWait::None, + default_missing_value = "container", + value_enum + )] + pub wait: PublishWait, + + /// Timeout (in seconds) for the publish query to the registry. + /// + /// Note that this is not the timeout for the entire publish process, but + /// for each individual query to the registry during the publish flow. + #[clap(long, default_value = "5m")] + pub timeout: humantime::Duration, + + /// Whether or not the patch field of the version of the package - if any - should be bumped. + #[clap(long, conflicts_with = "version")] + pub bump: bool, + + /// Do not prompt for user input. + #[clap(long, default_value_t = !std::io::stdin().is_terminal())] + pub non_interactive: bool, +} + +#[async_trait::async_trait] +impl AsyncCliCommand for PackagePublish { + type Output = PackageIdent; + + async fn run_async(self) -> Result { + tracing::info!("Checking if user is logged in"); + let client = login_user( + &self.api, + &self.env, + !self.non_interactive, + "publish a package", + ) + .await?; + + tracing::info!("Loading manifest"); + let (manifest_path, manifest) = get_manifest(&self.package_path)?; + tracing::info!("Got manifest at path {}", manifest_path.display()); + + let (package_namespace, package_hash) = { + let push_cmd = PackagePush { + api: self.api.clone(), + env: self.env.clone(), + dry_run: self.dry_run, + quiet: self.quiet, + package_namespace: self.package_namespace.clone(), + timeout: self.timeout.clone(), + bump: self.bump.clone(), + non_interactive: self.non_interactive.clone(), + wait: self.wait.clone(), + package_path: self.package_path.clone(), + }; + + push_cmd.push(&client, &manifest, &manifest_path).await? + }; + + let ident = PackageTag { + api: self.api.clone(), + env: self.env.clone(), + dry_run: self.dry_run.clone(), + quiet: self.quiet, + package_namespace: Some(package_namespace), + package_name: self.package_name.clone(), + package_version: self.package_version.clone(), + timeout: self.timeout.clone(), + bump: self.bump.clone(), + non_interactive: self.non_interactive.clone(), + package_path: self.package_path.clone(), + } + .tag(&client, &manifest, package_hash) + .await?; + + tracing::info!("Proceeding to invalidate query cache.."); + + if let Err(e) = invalidate_graphql_query_cache(&self.env.cache_dir) { + tracing::warn!( + error = &*e, + "Unable to invalidate the cache used for package version queries", + ); + } + + if !self.quiet && !self.non_interactive { + eprintln!( + "{} You can now run your package with {}", + "𖥔".green().bold(), + format!("`{} run {ident}`", bin_name!()).bold() + ); + } + Ok(ident) + } +} diff --git a/lib/cli/src/commands/package/push.rs b/lib/cli/src/commands/package/push.rs new file mode 100644 index 00000000000..7c6c5cee697 --- /dev/null +++ b/lib/cli/src/commands/package/push.rs @@ -0,0 +1,227 @@ +use super::common::{macros::*, wait::*, *}; +use crate::{ + commands::{AsyncCliCommand, PackageBuild}, + opts::{ApiOpts, WasmerEnv}, +}; +use colored::Colorize; +use is_terminal::IsTerminal; +use std::path::PathBuf; +use wasmer_api::WasmerClient; +use wasmer_config::package::{Manifest, PackageHash}; +use webc::wasmer_package::Package; + +/// Push a package to the registry. +/// +/// The result of this operation is that the hash of the package can be used to reference the +/// pushed package. +#[derive(Debug, clap::Parser)] +pub struct PackagePush { + #[clap(flatten)] + pub api: ApiOpts, + + #[clap(flatten)] + pub env: WasmerEnv, + + /// Run the publish logic without sending anything to the registry server + #[clap(long, name = "dry-run")] + pub dry_run: bool, + + /// Run the publish command without any output + #[clap(long)] + pub quiet: bool, + + /// Override the namespace of the package to upload + #[clap(long = "namespace")] + pub package_namespace: Option, + + /// Timeout (in seconds) for the publish query to the registry. + /// + /// Note that this is not the timeout for the entire publish process, but + /// for each individual query to the registry during the publish flow. + #[clap(long, default_value = "5m")] + pub timeout: humantime::Duration, + + /// Whether or not the patch field of the version of the package - if any - should be bumped. + #[clap(long, conflicts_with = "version")] + pub bump: bool, + + /// Do not prompt for user input. + #[clap(long, default_value_t = !std::io::stdin().is_terminal())] + pub non_interactive: bool, + + /// Wait for package to be available on the registry before exiting. + #[clap( + long, + require_equals = true, + num_args = 0..=1, + default_value_t = PublishWait::None, + default_missing_value = "container", + value_enum + )] + pub wait: PublishWait, + + /// Directory containing the `wasmer.toml`, or a custom *.toml manifest file. + /// + /// Defaults to current working directory. + #[clap(name = "path", default_value = ".")] + pub package_path: PathBuf, +} + +impl PackagePush { + async fn get_namespace( + &self, + client: &WasmerClient, + manifest: &Manifest, + ) -> anyhow::Result { + if let Some(owner) = &self.package_namespace { + return Ok(owner.clone()); + } + + if let Some(pkg) = &manifest.package { + if let Some(ns) = &pkg.name { + if let Some(first) = ns.split("/").next() { + return Ok(first.to_string()); + } + } + } + + if self.non_interactive { + // if not interactive we can't prompt the user to choose the owner of the app. + anyhow::bail!("No package namespace specified: use --namespace XXX"); + } + + let user = wasmer_api::query::current_user_with_namespaces(&client, None).await?; + let owner = crate::utils::prompts::prompt_for_namespace( + "Who should own this package?", + None, + Some(&user), + )?; + + Ok(owner.clone()) + } + + fn get_privacy(&self, manifest: &Manifest) -> bool { + match &manifest.package { + Some(pkg) => pkg.private, + None => true, + } + } + + async fn should_push(&self, client: &WasmerClient, hash: &PackageHash) -> anyhow::Result { + let res = wasmer_api::query::get_package_release(client, &hash.to_string()).await; + tracing::info!("{:?}", res); + res.map(|p| !p.is_some()) + } + + async fn do_push( + &self, + client: &WasmerClient, + namespace: &str, + package: &Package, + package_hash: &PackageHash, + private: bool, + ) -> anyhow::Result<()> { + let pb = make_pb!(self, "Uploading the package to the registry.."); + + let signed_url = upload(client, package_hash, self.timeout.clone(), package).await?; + + let id = match wasmer_api::query::push_package_release( + client, + None, + namespace, + &signed_url, + Some(private), + ) + .await? + { + Some(r) => { + if r.success { + pb_ok!(pb, "Succesfully pushed the package to the registry!"); + r.package_webc.unwrap().id + } else { + anyhow::bail!("An unidentified error occurred while publishing the package. (response had success: false)") + } + } + None => anyhow::bail!("An unidentified error occurred while publishing the package."), // <- This is extremely bad.. + }; + + wait_package(client, self.wait, id, &pb, self.timeout.clone()).await?; + Ok(()) + } + + pub async fn push( + &self, + client: &WasmerClient, + manifest: &Manifest, + manifest_path: &PathBuf, + ) -> anyhow::Result<(String, PackageHash)> { + tracing::info!("Building package"); + let pb = make_pb!( + self, + "Creating the package locally...", + ".", + "o", + "O", + "°", + "O", + "o", + "." + ); + let package = PackageBuild::check(manifest_path.clone()).execute()?; + pb_ok!(pb, "Correctly built package locally"); + + let hash_bytes = package + .webc_hash() + .ok_or(anyhow::anyhow!("No webc hash was provided"))?; + let hash = PackageHash::from_sha256_bytes(hash_bytes.clone()); + tracing::info!("Package has hash: {hash}",); + + let namespace = self.get_namespace(client, &manifest).await?; + + let private = self.get_privacy(&manifest); + tracing::info!("If published, package privacy is {private}"); + + let pb = make_pb!(self, "Checking if package is already in the registry.."); + if self.should_push(&client, &hash).await.map_err(on_error)? { + if !self.dry_run { + tracing::info!("Package should be published"); + pb_ok!(pb, "Package not in the registry yet!"); + + self.do_push(&client, &namespace, &package, &hash, private) + .await + .map_err(on_error)?; + } else { + tracing::info!("Package should be published, but dry-run is set"); + pb_ok!(pb, "Skipping push as dry-run is set"); + } + } else { + tracing::info!("Package should not be published"); + pb_ok!(pb, "Package was already in the registry, no push needed"); + } + + Ok((namespace, hash)) + } +} + +#[async_trait::async_trait] +impl AsyncCliCommand for PackagePush { + type Output = (); + + async fn run_async(self) -> Result { + tracing::info!("Checking if user is logged in"); + let client = login_user( + &self.api, + &self.env, + !self.non_interactive, + "push a package", + ) + .await?; + + tracing::info!("Loading manifest"); + let (manifest_path, manifest) = get_manifest(&self.package_path)?; + tracing::info!("Got manifest at path {}", manifest_path.display()); + + self.push(&client, &manifest, &manifest_path).await?; + Ok(()) + } +} diff --git a/lib/cli/src/commands/package/tag.rs b/lib/cli/src/commands/package/tag.rs new file mode 100644 index 00000000000..0d470ed378f --- /dev/null +++ b/lib/cli/src/commands/package/tag.rs @@ -0,0 +1,407 @@ +use super::common::PackageSpecifier; +use crate::{ + commands::{ + package::common::{macros::*, *}, + AsyncCliCommand, PackageBuild, + }, + opts::{ApiOpts, WasmerEnv}, +}; +use colored::Colorize; +use dialoguer::{theme::ColorfulTheme, Confirm}; +use is_terminal::IsTerminal; +use std::path::PathBuf; +use wasmer_api::WasmerClient; +use wasmer_config::package::{Manifest, PackageHash, PackageIdent}; + +/// Tag an existing package. +#[derive(Debug, clap::Parser)] +pub struct PackageTag { + #[clap(flatten)] + pub api: ApiOpts, + + #[clap(flatten)] + pub env: WasmerEnv, + + /// Run the publish logic without sending anything to the registry server + #[clap(long, name = "dry-run")] + pub dry_run: bool, + + /// Run the publish command without any output + #[clap(long)] + pub quiet: bool, + + /// Override the namespace of the package to upload + #[clap(long = "namespace")] + pub package_namespace: Option, + + /// Override the name of the package to upload + #[clap(long = "name")] + pub package_name: Option, + + /// Override the package version of the uploaded package in the wasmer.toml + #[clap(long = "version")] + pub package_version: Option, + + /// Timeout (in seconds) for the publish query to the registry. + /// + /// Note that this is not the timeout for the entire publish process, but + /// for each individual query to the registry during the publish flow. + #[clap(long, default_value = "5m")] + pub timeout: humantime::Duration, + + /// Whether or not the patch field of the version of the package - if any - should be bumped. + #[clap(long, conflicts_with = "version")] + pub bump: bool, + + /// Do not prompt for user input. + #[clap(long, default_value_t = !std::io::stdin().is_terminal())] + pub non_interactive: bool, + + /// Directory containing the `wasmer.toml`, or a custom *.toml manifest file. + /// + /// Defaults to current working directory. + #[clap(name = "path", default_value = ".")] + pub package_path: PathBuf, +} + +impl PackageTag { + async fn get_namespace( + &self, + client: &WasmerClient, + manifest: &Manifest, + ) -> anyhow::Result { + if let Some(owner) = &self.package_namespace { + return Ok(owner.clone()); + } + + if let Some(pkg) = &manifest.package { + if let Some(ns) = &pkg.name { + if let Some(first) = ns.split("/").next() { + return Ok(first.to_string()); + } + } + } + + if self.non_interactive { + // if not interactive we can't prompt the user to choose the owner of the app. + anyhow::bail!("No package namespace specified: use --namespace XXX"); + } + + let user = wasmer_api::query::current_user_with_namespaces(&client, None).await?; + let owner = crate::utils::prompts::prompt_for_namespace( + "Who should own this package?", + None, + Some(&user), + )?; + + Ok(owner.clone()) + } + + async fn synthesize_tagger( + &self, + client: &WasmerClient, + ident: &PackageSpecifier, + ) -> anyhow::Result { + let mut new_ident = ident.clone(); + + if let Some(user_version) = &self.package_version { + match &mut new_ident { + PackageSpecifier::Hash { .. } => { + match (&self.package_name, &self.package_namespace) { + (Some(name), Some(namespace)) => { + new_ident = PackageSpecifier::Named { + namespace: namespace.clone(), + name: name.clone(), + tag: Tag::Version(user_version.clone()), + } + } + _ => anyhow::bail!( + "The flag `--version` was specified, but no namespace and name were given." + ), + } + } + PackageSpecifier::Named { tag, .. } => *tag = Tag::Version(user_version.clone()), + } + } else if let Some(user_namespace) = &self.package_namespace { + match &mut new_ident { + PackageSpecifier::Hash { hash, namespace } => match &self.package_name { + Some(name) => { + new_ident = PackageSpecifier::Named { + namespace: user_namespace.clone(), + name: name.clone(), + tag: Tag::Hash(hash.clone()), + } + } + _ => *namespace = user_namespace.clone(), + }, + PackageSpecifier::Named { namespace, .. } => *namespace = user_namespace.clone(), + } + } else if let Some(user_name) = &self.package_name { + match &mut new_ident { + PackageSpecifier::Hash { .. } => { + anyhow::bail!("The flag `--name` was specified, but no namespace was given.") + } + PackageSpecifier::Named { name, .. } => *name = user_name.clone(), + } + } + + // At this point, we can resolve the version from the registry if any and if necessary bump + // the version. + if let PackageSpecifier::Named { + name, + namespace, + tag: Tag::Version(user_version), + } = &mut new_ident + { + let full_pkg_name = format!("{namespace}/{name}"); + let pb = make_pb!( + self, + format!("Checking if a version of {full_pkg_name} already exists..") + ); + + if let Some(registry_version) = wasmer_api::query::get_package_version( + client, + full_pkg_name.clone(), + String::from("latest"), + ) + .await? + .map(|p| p.version) + { + let registry_version = semver::Version::parse(®istry_version)?; + pb_ok!( + pb, + format!("Found version {registry_version} of package {full_pkg_name}") + ); + if user_version.clone() < registry_version { + if self.bump { + user_version.patch += 1; + } else if !self.non_interactive { + eprintln!("The registry has a newer version of the package."); + let theme = ColorfulTheme::default(); + let mut new_version = user_version.clone(); + new_version.patch += 1; + if Confirm::with_theme(&theme).with_prompt(format!("Do you want to bump the package's version ({user_version} -> {new_version})?")).interact()? { + user_version.patch += 1; + } + } + } + } else { + pb_ok!(pb, format!("No version of {full_pkg_name} in registry!")); + } + } + + Ok(new_ident) + } + + async fn do_tag( + &self, + client: &WasmerClient, + ident: &PackageSpecifier, + manifest: &Manifest, + package_release_id: &wasmer_api::types::Id, + ) -> anyhow::Result { + tracing::info!( + "Tagging package with registry id {:?} and specifier {:?}", + package_release_id, + ident + ); + + let tagger = self.synthesize_tagger(client, ident).await?; + + let pb = make_pb!(self, "Tagging package..."); + + if let PackageSpecifier::Hash { .. } = &tagger { + pb_ok!(pb, "Package is unnamed, no need to tag it"); + return Ok(tagger); + } + + let PackageSpecifier::Named { + namespace, + name, + tag, + } = tagger.clone() + else { + unreachable!() + }; + + if self.dry_run { + tracing::info!("No tagging to do here, dry-run is set"); + pb_ok!(pb, "Skipping tag (dry-run)"); + return Ok(tagger); + } + + let maybe_description = manifest + .package + .as_ref() + .and_then(|p| p.description.clone()); + let maybe_homepage = manifest.package.as_ref().and_then(|p| p.homepage.clone()); + let maybe_license = manifest.package.as_ref().and_then(|p| p.license.clone()); + let maybe_license_file = manifest + .package + .as_ref() + .and_then(|p| p.license_file.clone()) + .map(|f| f.to_string_lossy().to_string()); + let maybe_readme = manifest + .package + .as_ref() + .and_then(|p| p.readme.clone()) + .map(|f| f.to_string_lossy().to_string()); + let maybe_repository = manifest.package.as_ref().and_then(|p| p.repository.clone()); + let (version, full_name) = ( + match &tag { + Tag::Version(v) => v.to_string(), + Tag::Hash(h) => h.to_string(), + }, + format!("{namespace}/{name}"), + ); + + let private = if let Some(pkg) = &manifest.package { + Some(pkg.private) + } else { + Some(false) + }; + + let manifest_raw = toml::to_string(&manifest)?; + + let r = wasmer_api::query::tag_package_release( + client, + maybe_description.as_deref(), + maybe_homepage.as_deref(), + maybe_license.as_deref(), + maybe_license_file.as_deref(), + &manifest_raw, + &full_name, + None, + package_release_id, + private, + maybe_readme.as_deref(), + maybe_repository.as_deref(), + &version, + ); + + match r.await? { + Some(r) => { + if r.success { + pb_ok!(pb, "Successfully tagged package"); + Ok(tagger) + } else { + pb_err!(pb, "Could not tag package!"); + anyhow::bail!("An unknown error occurred and the tagging failed.") + } + } + None => { + pb_err!(pb, "Could not tag package!"); + anyhow::bail!("The registry returned an empty response.") + } + } + } + + async fn get_package_id( + &self, + client: &WasmerClient, + hash: &PackageHash, + ) -> anyhow::Result { + let pb = make_pb!(self, "Checking if the package exists.."); + + tracing::debug!("Searching for package with hash: {hash}"); + + let pkg = match wasmer_api::query::get_package_release(client, &hash.to_string()).await? { + Some(p) => p, + None => { + pb_err!(pb, "The package is not in the registry!"); + if !self.quiet { + eprintln!("\n\nThe package with the required hash does not exist in the selected registry."); + let bin_name = bin_name!(); + let cli = cli_line!(); + + if cli.contains("publish") && self.dry_run { + eprintln!( + "{}: you are running `{cli}` with `--dry-run` set.\n", + "HINT".bold() + ); + } else { + eprintln!( + "To first push the package to the registry, run `{}`.", + format!("{bin_name} package push").bold() + ); + eprintln!( + "{}: you can also use `{}` to push {} tag your package.\n", + "NOTE".bold(), + format!("{bin_name} package publish").bold(), + "and".italic() + ); + } + } + anyhow::bail!("Can't tag, no matching package found in the registry.") + } + }; + + pb_ok!(pb, "Found package in the registry!"); + + Ok(pkg.id) + } + + pub async fn tag( + &self, + client: &WasmerClient, + manifest: &Manifest, + hash: PackageHash, + ) -> anyhow::Result { + let namespace = self.get_namespace(client, &manifest).await?; + + let ident = into_specifier(&manifest, &hash, namespace)?; + tracing::info!("PackageIdent extracted from manifest is {:?}", ident); + + let package_id = self.get_package_id(&client, &hash).await?; + tracing::info!( + "The package identifier returned from the registry is {:?}", + package_id + ); + + let ident = self + .do_tag(&client, &ident, &manifest, &package_id) + .await + .map_err(on_error)?; + + Ok(ident.into()) + } +} + +#[async_trait::async_trait] +impl AsyncCliCommand for PackageTag { + type Output = (); + + async fn run_async(self) -> Result { + tracing::info!("Checking if user is logged in"); + let client = + login_user(&self.api, &self.env, !self.non_interactive, "tag a package").await?; + + tracing::info!("Loading manifest"); + let (manifest_path, manifest) = get_manifest(&self.package_path)?; + tracing::info!("Got manifest at path {}", manifest_path.display()); + + tracing::info!("Building package"); + let pb = make_pb!( + self, + "Creating the package locally...", + ".", + "o", + "O", + "°", + "O", + "o", + "." + ); + let package = PackageBuild::check(manifest_path.clone()).execute()?; + pb_ok!(pb, "Correctly built package locally"); + + let hash_bytes = package + .webc_hash() + .ok_or(anyhow::anyhow!("No webc hash was provided"))?; + let hash = PackageHash::from_sha256_bytes(hash_bytes); + tracing::info!("Package has hash: {hash}",); + + self.tag(&client, &manifest, hash).await?; + Ok(()) + } +} diff --git a/lib/cli/src/commands/publish.rs b/lib/cli/src/commands/publish.rs deleted file mode 100644 index 17aff62cb48..00000000000 --- a/lib/cli/src/commands/publish.rs +++ /dev/null @@ -1,252 +0,0 @@ -use anyhow::Context as _; -use clap::Parser; -use dialoguer::Confirm; -use is_terminal::IsTerminal; -use wasmer_config::package::PackageIdent; -use wasmer_registry::{publish::PublishWait, wasmer_env::WasmerEnv}; - -use crate::{opts::ApiOpts, utils::load_package_manifest}; - -use super::{AsyncCliCommand, PackageBuild}; - -/// Publish a package to the package registry. -#[derive(Debug, Parser)] -pub struct Publish { - #[clap(flatten)] - pub env: WasmerEnv, - /// Run the publish logic without sending anything to the registry server - #[clap(long, name = "dry-run")] - pub dry_run: bool, - /// Run the publish command without any output - #[clap(long)] - pub quiet: bool, - /// Override the namespace of the package to upload - #[clap(long)] - pub package_namespace: Option, - /// Override the name of the package to upload - #[clap(long)] - pub package_name: Option, - /// Override the package version of the uploaded package in the wasmer.toml - #[clap(long)] - pub version: Option, - /// Skip validation of the uploaded package - #[clap(long)] - pub no_validate: bool, - /// Directory containing the `wasmer.toml`, or a custom *.toml manifest file. - /// - /// Defaults to current working directory. - #[clap(name = "PACKAGE_PATH")] - pub package_path: Option, - /// Wait for package to be available on the registry before exiting. - #[clap(long)] - pub wait: bool, - /// Wait for the package and all dependencies to be available on the registry - /// before exiting. - /// - /// This includes the container, native executables and bindings. - #[clap(long)] - pub wait_all: bool, - /// Timeout (in seconds) for the publish query to the registry. - /// - /// Note that this is not the timeout for the entire publish process, but - /// for each individual query to the registry during the publish flow. - #[clap(long, default_value = "5m")] - pub timeout: humantime::Duration, - - /// Whether or not the patch field of the version of the package - if any - should be bumped. - #[clap(long)] - pub bump: bool, - - /// Do not prompt for user input. - #[clap(long, default_value_t = !std::io::stdin().is_terminal())] - pub non_interactive: bool, -} - -#[async_trait::async_trait] -impl AsyncCliCommand for Publish { - type Output = Option; - - async fn run_async(self) -> Result { - let interactive = !self.non_interactive; - let manifest_dir_path = match self.package_path.as_ref() { - Some(s) => std::env::current_dir()?.join(s), - None => std::env::current_dir()?, - }; - - let (manifest_path, mut manifest) = match load_package_manifest(&manifest_dir_path)? { - Some(r) => r, - None => anyhow::bail!( - "Path '{}' does not contain a valid `wasmer.toml` manifest.", - manifest_dir_path.display() - ), - }; - - let hash = PackageBuild::check(manifest_dir_path).execute()?; - - let api = ApiOpts { - token: self.env.token().clone(), - registry: Some(self.env.registry_endpoint()?), - }; - let client = api.client()?; - - tracing::info!("checking if package with hash {hash} already exists"); - - // [TODO]: Add a simpler query to simply retrieve a boolean value if the package with the - // given hash exists. - let maybe_already_published = - wasmer_api::query::get_package_release(&client, &hash.to_string()).await; - - tracing::info!( - "received response: {:#?} from registry", - maybe_already_published - ); - - let maybe_already_published = maybe_already_published.is_ok_and(|u| u.is_some()); - - if maybe_already_published { - eprintln!( - "Package already present on registry (hash: {})", - &hash.to_string().trim_start_matches("sha256:")[..7] - ); - return Ok(Some(PackageIdent::Hash(hash))); - } - - if manifest.package.is_none() && (self.version.is_some() || self.package_name.is_some()) { - eprintln!("Warning: overrides for package version or package name were specified."); - eprintln!( - "The manifest in path {}, however, specifies an unnamed package,", - manifest_path.display() - ); - eprintln!("that is, a package without name and version."); - } - - let mut version = self.version.clone(); - - if let Some(ref mut pkg) = manifest.package { - if let (Some(pkg_name), Some(pkg_version)) = (&pkg.name, &pkg.version) { - let pkg_name = pkg_name.clone(); - let pkg_version = pkg_version.clone(); - - let mut latest_version = { - let v = wasmer_api::query::get_package_version( - &client, - pkg_name.clone(), - "latest".into(), - ) - .await?; - if let Some(v) = v { - semver::Version::parse(&v.version) - .with_context(|| "While parsing registry version of package")? - } else { - pkg_version.clone() - } - }; - - if pkg_version < latest_version { - if self.bump { - latest_version.patch += 1; - version = Some(latest_version); - } else if interactive { - latest_version.patch += 1; - let theme = dialoguer::theme::ColorfulTheme::default(); - if Confirm::with_theme(&theme) - .with_prompt(format!( - "Do you want to bump the package to a new version? ({} -> {})", - pkg_version, latest_version - )) - .interact() - .unwrap_or_default() - { - pkg.version = Some(latest_version); - } - } else if latest_version > pkg_version { - eprintln!("Registry has a newer version of this package."); - eprintln!( - "If a package with version {} already exists, publishing will fail.", - pkg_version - ); - } - } - - // If necessary, update the manifest. - if version != pkg.version { - pkg.version = version.clone(); - - let contents = toml::to_string(&manifest).with_context(|| { - format!( - "could not serialize manifest from path '{}'", - manifest_path.display() - ) - })?; - - tokio::fs::write(&manifest_path, contents) - .await - .with_context(|| { - format!("could not write manifest to '{}'", manifest_path.display()) - })?; - } - } - } - - let token = self - .env - .token() - .context("could not determine auth token for registry - run 'wasmer login'")?; - - let wait = if self.wait_all { - PublishWait::new_all() - } else if self.wait { - PublishWait::new_container() - } else { - PublishWait::new_none() - }; - - tracing::trace!("wait mode is: {:?}", wait); - - let publish = wasmer_registry::package::builder::Publish { - registry: self.env.registry_endpoint().map(|u| u.to_string()).ok(), - dry_run: self.dry_run, - quiet: self.quiet, - package_name: self.package_name.clone(), - version, - token, - no_validate: self.no_validate, - package_path: self.package_path.clone(), - wait, - timeout: self.timeout.into(), - package_namespace: self.package_namespace, - }; - - tracing::trace!("Sending publish query: {:#?}", publish); - - let res = publish.execute().await.map_err(on_error)?; - - if let Err(e) = invalidate_graphql_query_cache(&self.env) { - tracing::warn!( - error = &*e, - "Unable to invalidate the cache used for package version queries", - ); - } - - Ok(res) - } -} - -fn on_error(e: anyhow::Error) -> anyhow::Error { - #[cfg(feature = "telemetry")] - sentry::integrations::anyhow::capture_anyhow(&e); - - e -} - -// HACK: We want to invalidate the cache used for GraphQL queries so -// the current user sees the results of publishing immediately. There -// are cleaner ways to achieve this, but for now we're just going to -// clear out the whole GraphQL query cache. -// See https://github.com/wasmerio/wasmer/pull/3983 for more -fn invalidate_graphql_query_cache(env: &WasmerEnv) -> Result<(), anyhow::Error> { - let cache_dir = env.cache_dir().join("queries"); - std::fs::remove_dir_all(cache_dir)?; - - Ok(()) -} diff --git a/lib/cli/src/opts.rs b/lib/cli/src/opts.rs index 29b0689d136..ab3f2444fa9 100644 --- a/lib/cli/src/opts.rs +++ b/lib/cli/src/opts.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use anyhow::Context; use wasmer_api::WasmerClient; use wasmer_registry::WasmerConfig; @@ -10,6 +12,37 @@ pub struct ApiOpts { pub registry: Option, } +lazy_static::lazy_static! { + /// The default value for `$WASMER_DIR`. + pub static ref WASMER_DIR: PathBuf = match WasmerConfig::get_wasmer_dir() { + Ok(path) => path, + Err(e) => { + if let Some(install_prefix) = option_env!("WASMER_INSTALL_PREFIX") { + return PathBuf::from(install_prefix); + } + + panic!("Unable to determine the wasmer dir: {e}"); + } + }; + + /// The default value for `$WASMER_DIR`. + pub static ref WASMER_CACHE_DIR: PathBuf = WASMER_DIR.join("cache"); +} + +/// Command-line flags for determining the local "Wasmer Environment". +/// +/// This is where you access `$WASMER_DIR`, the `$WASMER_DIR/wasmer.toml` config +/// file, and specify the current registry. +#[derive(Debug, Clone, PartialEq, clap::Parser)] +pub struct WasmerEnv { + /// Set Wasmer's home directory + #[clap(long, env = "WASMER_DIR", default_value = WASMER_DIR.as_os_str())] + pub wasmer_dir: PathBuf, + /// The directory cached artefacts are saved to. + #[clap(long, env = "WASMER_CACHE_DIR", default_value = WASMER_CACHE_DIR.as_os_str())] + pub cache_dir: PathBuf, +} + struct Login { url: url::Url, token: Option, diff --git a/lib/registry/src/graphql/subscriptions.rs b/lib/registry/src/graphql/subscriptions.rs index 6f0115f4a02..6f2c9986e33 100644 --- a/lib/registry/src/graphql/subscriptions.rs +++ b/lib/registry/src/graphql/subscriptions.rs @@ -7,3 +7,4 @@ use graphql_client::GraphQLQuery; response_derives = "Debug" )] pub struct PackageVersionReady; + diff --git a/lib/registry/src/subscriptions.rs b/lib/registry/src/subscriptions.rs index f310dd4b0c8..911f2b80395 100644 --- a/lib/registry/src/subscriptions.rs +++ b/lib/registry/src/subscriptions.rs @@ -18,6 +18,8 @@ use tokio_tungstenite::{ tungstenite::{client::IntoClientRequest, http::HeaderValue, Message}, }; +pub use crate::graphql::subscriptions::package_version_ready::PackageVersionState; + async fn subscribe_graphql( registry_url: &str, login_token: &str, diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 80907d665b3..bd110c4a5b3 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -84,9 +84,14 @@ impl BinaryPackage { let source = rt.source(); let manifest = container.manifest(); - let id = PackageInfo::package_id_from_manifest(manifest)?.unwrap_or_else(|| { - PackageId::Hash(PackageHash::from_sha256_bytes(container.webc_hash())) - }); + let id = match PackageInfo::package_id_from_manifest(manifest)? { + Some(id) => id, + None => PackageId::Hash(PackageHash::from_sha256_bytes( + container + .webc_hash() + .ok_or(anyhow::anyhow!("No webc hash was provided"))?, + )), + }; let root = PackageInfo::from_manifest(id, manifest, container.version())?; let root_id = root.id.clone(); From 01750eaf408793dee47ad8bf03e2eace0964f263 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Tue, 7 May 2024 19:19:07 +0330 Subject: [PATCH 37/55] prepare wasmer-config 0.2.0 release --- Cargo.lock | 26 +++++++++++++------------- lib/config/Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1e70ea0006..41621f03784 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6269,7 +6269,7 @@ dependencies = [ "tracing", "url", "uuid", - "wasmer-config 0.1.0", + "wasmer-config 0.2.0", "webc", ] @@ -6459,7 +6459,7 @@ dependencies = [ "wasmer-compiler-cranelift", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", - "wasmer-config 0.1.0", + "wasmer-config 0.2.0", "wasmer-emscripten", "wasmer-object", "wasmer-registry", @@ -6583,20 +6583,20 @@ dependencies = [ [[package]] name = "wasmer-config" version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b405c9856adaf65ee91eeeeaac6fcc6b127188648c60ae4e89de63f506c74e6" dependencies = [ "anyhow", "bytesize", "derive_builder", "hex", - "indexmap 2.2.6", - "pretty_assertions", + "indexmap 1.9.3", "schemars", "semver 1.0.22", "serde", "serde_cbor", "serde_json", "serde_yaml 0.9.34+deprecated", - "tempfile", "thiserror", "toml 0.8.12", "url", @@ -6604,21 +6604,21 @@ dependencies = [ [[package]] name = "wasmer-config" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b405c9856adaf65ee91eeeeaac6fcc6b127188648c60ae4e89de63f506c74e6" +version = "0.2.0" dependencies = [ "anyhow", "bytesize", "derive_builder", "hex", - "indexmap 1.9.3", + "indexmap 2.2.6", + "pretty_assertions", "schemars", "semver 1.0.22", "serde", "serde_cbor", "serde_json", "serde_yaml 0.9.34+deprecated", + "tempfile", "thiserror", "toml 0.8.12", "url", @@ -6797,7 +6797,7 @@ dependencies = [ "toml 0.5.11", "tracing", "url", - "wasmer-config 0.1.0", + "wasmer-config 0.2.0", "wasmer-wasm-interface", "wasmparser 0.121.2", "webc", @@ -6931,7 +6931,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-bindgen-test", "wasmer", - "wasmer-config 0.1.0", + "wasmer-config 0.2.0", "wasmer-emscripten", "wasmer-journal", "wasmer-types", @@ -7028,7 +7028,7 @@ dependencies = [ "wasmer-compiler-cranelift", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", - "wasmer-config 0.1.0", + "wasmer-config 0.2.0", "wasmer-emscripten", "wasmer-middlewares", "wasmer-types", @@ -7175,7 +7175,7 @@ dependencies = [ "thiserror", "toml 0.7.8", "url", - "wasmer-config 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-config 0.1.0", ] [[package]] diff --git a/lib/config/Cargo.toml b/lib/config/Cargo.toml index 5c1f35dd2dc..59933f9ea4e 100644 --- a/lib/config/Cargo.toml +++ b/lib/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wasmer-config" -version = "0.1.0" +version = "0.2.0" description = "Configuration types for Wasmer." edition.workspace = true license.workspace = true From 3d264586e982645d7cae37c97f2d5a119ef2e47b Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Mon, 6 May 2024 20:26:23 +0330 Subject: [PATCH 38/55] Move rewind buffer to after TLS area if it's at the bottom of the stack --- lib/journal/src/concrete/archived.rs | 2 + lib/journal/src/concrete/archived_from.rs | 6 +++ lib/wasi-types/src/wasix/mod.rs | 4 ++ lib/wasix/src/state/func_env.rs | 45 ++++++++++++++++++++ lib/wasix/src/syscalls/mod.rs | 27 +++++++++--- lib/wasix/src/syscalls/wasix/thread_spawn.rs | 5 +++ 6 files changed, 84 insertions(+), 5 deletions(-) diff --git a/lib/journal/src/concrete/archived.rs b/lib/journal/src/concrete/archived.rs index d03b8583dd0..85f66df6c2d 100644 --- a/lib/journal/src/concrete/archived.rs +++ b/lib/journal/src/concrete/archived.rs @@ -1802,4 +1802,6 @@ pub struct JournalWasiMemoryLayout { pub stack_lower: u64, pub guard_size: u64, pub stack_size: u64, + pub tls_base: u64, + pub tls_size: u64, } diff --git a/lib/journal/src/concrete/archived_from.rs b/lib/journal/src/concrete/archived_from.rs index 853973061bf..bec9070c5cb 100644 --- a/lib/journal/src/concrete/archived_from.rs +++ b/lib/journal/src/concrete/archived_from.rs @@ -616,6 +616,8 @@ impl From for WasiMemoryLayout { stack_lower: value.stack_lower, guard_size: value.guard_size, stack_size: value.stack_size, + tls_base: value.tls_base, + tls_size: value.tls_size, } } } @@ -627,6 +629,8 @@ impl From<&'_ ArchivedJournalWasiMemoryLayout> for WasiMemoryLayout { stack_lower: value.stack_lower, guard_size: value.guard_size, stack_size: value.stack_size, + tls_base: value.tls_base, + tls_size: value.tls_size, } } } @@ -638,6 +642,8 @@ impl From for JournalWasiMemoryLayout { stack_lower: value.stack_lower, guard_size: value.guard_size, stack_size: value.stack_size, + tls_base: value.tls_base, + tls_size: value.tls_size, } } } diff --git a/lib/wasi-types/src/wasix/mod.rs b/lib/wasi-types/src/wasix/mod.rs index 8f601ed578b..b494d23936a 100644 --- a/lib/wasi-types/src/wasix/mod.rs +++ b/lib/wasi-types/src/wasix/mod.rs @@ -26,4 +26,8 @@ pub struct WasiMemoryLayout { pub guard_size: u64, /// Total size of the stack pub stack_size: u64, + /// Base address of the TLS area + pub tls_base: u64, + /// Total size of the TLS area + pub tls_size: u64, } diff --git a/lib/wasix/src/state/func_env.rs b/lib/wasix/src/state/func_env.rs index 6eff4e5166b..c12a71b2c9d 100644 --- a/lib/wasix/src/state/func_env.rs +++ b/lib/wasix/src/state/func_env.rs @@ -25,6 +25,17 @@ use crate::{ const DEFAULT_STACK_SIZE: u64 = 1_048_576u64; const DEFAULT_STACK_BASE: u64 = DEFAULT_STACK_SIZE; +/// The default size and base address of the TLS area for modules that +/// don't export __tls_base and __tls_size. This is the default value +/// used by clang-15 at the time of writing this code. +const DEFAULT_TLS_BASE: u64 = 1024u64; + +/// This is merely an attempt at a guess. We're avoiding the __pthread +/// struct that each thread has (hoping it comes first), but there's +/// no telling how many thread statics a module has and how much space +/// they take. +const DEFAULT_TLS_SIZE: u64 = 1024u64; + #[derive(Clone, Debug)] pub struct WasiFunctionEnv { pub env: FunctionEnv, @@ -147,6 +158,18 @@ impl WasiFunctionEnv { |v| Ok(v.clone()), )?; + let tls_base = instance + .exports + .get_global("__tls_base") + .map(|a| a.clone()) + .ok(); + + let tls_size = instance + .exports + .get_global("__tls_size") + .map(|a| a.clone()) + .ok(); + let new_inner = WasiInstanceHandles::new(memory, store, instance); let stack_pointer = new_inner.stack_pointer.clone(); @@ -173,12 +196,34 @@ impl WasiFunctionEnv { )); } + let tls_base = if let Some(tls_base) = tls_base { + match tls_base.get(store) { + wasmer::Value::I32(a) => a as u64, + wasmer::Value::I64(a) => a as u64, + _ => DEFAULT_TLS_BASE, + } + } else { + DEFAULT_TLS_BASE + }; + + let tls_size = if let Some(tls_size) = tls_size { + match tls_size.get(store) { + wasmer::Value::I32(a) => a as u64, + wasmer::Value::I64(a) => a as u64, + _ => DEFAULT_TLS_SIZE, + } + } else { + DEFAULT_TLS_SIZE + }; + // Update the stack layout which is need for asyncify let env = self.data_mut(store); let tid = env.tid(); let layout = &mut env.layout; layout.stack_upper = stack_base; layout.stack_size = layout.stack_upper - layout.stack_lower; + layout.tls_base = tls_base; + layout.tls_size = tls_size; // Replace the thread object itself env.thread.set_memory_layout(layout.clone()); diff --git a/lib/wasix/src/syscalls/mod.rs b/lib/wasix/src/syscalls/mod.rs index 2cd424ea9a0..d658006f1bb 100644 --- a/lib/wasix/src/syscalls/mod.rs +++ b/lib/wasix/src/syscalls/mod.rs @@ -870,6 +870,21 @@ pub(crate) fn get_stack_upper(env: &WasiEnv) -> u64 { env.layout.stack_upper } +fn get_unwind_buffer_start(env: &WasiEnv) -> u64 { + let stack_lower = env.layout.stack_lower; + let stack_upper = env.layout.stack_upper; + let tls_base = env.layout.tls_base; + + if stack_upper.saturating_sub(tls_base) < tls_base.saturating_sub(stack_lower) { + // tls_base is closer to stack_upper, so we assume TLS lives at the top of the stack. + // This is the case for threads spawned by wasix-libc. + stack_lower + } else { + // Otherwise, we assume TLS lives on the bottom, in which case we have to "skip" it. + tls_base + env.layout.tls_size + } +} + pub(crate) unsafe fn get_memory_stack_pointer( ctx: &mut FunctionEnvMut<'_, WasiEnv>, ) -> Result { @@ -1137,14 +1152,12 @@ where let memory = unsafe { env.memory_view(&ctx) }; // Write the addresses to the start of the stack space - let unwind_pointer = env.layout.stack_lower; + let unwind_pointer = get_unwind_buffer_start(env); let unwind_data_start = unwind_pointer + (std::mem::size_of::<__wasi_asyncify_t>() as u64); let unwind_data = __wasi_asyncify_t:: { start: wasi_try_ok!(unwind_data_start.try_into().map_err(|_| Errno::Overflow)), - end: wasi_try_ok!(env - .layout - .stack_upper + end: wasi_try_ok!((env.layout.stack_upper - memory_stack.len() as u64) .try_into() .map_err(|_| Errno::Overflow)), }; @@ -1175,6 +1188,9 @@ where trace!( stack_upper = env.layout.stack_upper, stack_lower = env.layout.stack_lower, + tls_base = env.layout.tls_base, + tls_size = env.layout.tls_size, + %unwind_pointer, "wasi[{}:{}]::unwinding (used_stack_space={} total_stack_space={})", ctx.data().pid(), ctx.data().tid(), @@ -1198,6 +1214,7 @@ where let unwind_stack_finish: u64 = unwind_data_result.start.into(); let unwind_size = unwind_stack_finish - unwind_stack_begin; trace!( + %unwind_pointer, "wasi[{}:{}]::unwound (memory_stack_size={} unwind_size={})", ctx.data().pid(), ctx.data().tid(), @@ -1297,7 +1314,7 @@ pub fn rewind_ext( }; // Write the addresses to the start of the stack space - let rewind_pointer = env.layout.stack_lower; + let rewind_pointer = get_unwind_buffer_start(env); let rewind_data_start = rewind_pointer + (std::mem::size_of::<__wasi_asyncify_t>() as u64); let rewind_data_end = rewind_data_start + (rewind_stack.len() as u64); diff --git a/lib/wasix/src/syscalls/wasix/thread_spawn.rs b/lib/wasix/src/syscalls/wasix/thread_spawn.rs index f30d2d772fa..7d792f19a3e 100644 --- a/lib/wasix/src/syscalls/wasix/thread_spawn.rs +++ b/lib/wasix/src/syscalls/wasix/thread_spawn.rs @@ -65,11 +65,16 @@ pub fn thread_spawn_internal_from_wasi( let tls_base: u64 = start.tls_base.try_into().map_err(|_| Errno::Overflow)?; let stack_lower = stack_upper - stack_size; + // TLS size is constant + let tls_size = env.layout.tls_size; + WasiMemoryLayout { stack_upper, stack_lower, guard_size, stack_size, + tls_base, + tls_size, } }; tracing::trace!("spawn with layout {:?}", layout); From 7fb9b4470e068928cf7e2415ca626b40d44066e2 Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Tue, 7 May 2024 16:25:03 +0330 Subject: [PATCH 39/55] Fix tests --- lib/journal/src/concrete/log_file.rs | 8 ++++++++ lib/journal/src/concrete/tests.rs | 2 ++ 2 files changed, 10 insertions(+) diff --git a/lib/journal/src/concrete/log_file.rs b/lib/journal/src/concrete/log_file.rs index 5c17d500a17..b0b1fe4c994 100644 --- a/lib/journal/src/concrete/log_file.rs +++ b/lib/journal/src/concrete/log_file.rs @@ -345,6 +345,8 @@ mod tests { stack_lower: 1024, guard_size: 16, stack_size: 1024, + tls_base: 16, + tls_size: 32, }, start: wasmer_wasix_types::wasix::ThreadStartType::MainThread, }) @@ -374,6 +376,8 @@ mod tests { stack_lower: 1024, guard_size: 16, stack_size: 1024, + tls_base: 16, + tls_size: 32, }, start: wasmer_wasix_types::wasix::ThreadStartType::MainThread, }) @@ -424,6 +428,8 @@ mod tests { stack_lower: 1024, guard_size: 16, stack_size: 1024, + tls_base: 16, + tls_size: 32, }, start: wasmer_wasix_types::wasix::ThreadStartType::MainThread, }) @@ -471,6 +477,8 @@ mod tests { stack_lower: 1024, guard_size: 16, stack_size: 1024, + tls_base: 16, + tls_size: 32, }, start: wasmer_wasix_types::wasix::ThreadStartType::MainThread, }) diff --git a/lib/journal/src/concrete/tests.rs b/lib/journal/src/concrete/tests.rs index 722d9d1faf9..54c5a9b52b1 100644 --- a/lib/journal/src/concrete/tests.rs +++ b/lib/journal/src/concrete/tests.rs @@ -75,6 +75,8 @@ pub fn test_record_set_thread() { stack_lower: 1024, guard_size: 16, stack_size: 1024, + tls_base: 16, + tls_size: 32, }, start: wasmer_wasix_types::wasix::ThreadStartType::MainThread, }); From 5f509798649e580c58cf47aebb27d5ef97d6fcc8 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 18:39:21 +0200 Subject: [PATCH 40/55] [skip ci] Make `WasmerEnv` derive `Default` --- lib/cli/src/opts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/src/opts.rs b/lib/cli/src/opts.rs index ab3f2444fa9..5685cd21558 100644 --- a/lib/cli/src/opts.rs +++ b/lib/cli/src/opts.rs @@ -33,7 +33,7 @@ lazy_static::lazy_static! { /// /// This is where you access `$WASMER_DIR`, the `$WASMER_DIR/wasmer.toml` config /// file, and specify the current registry. -#[derive(Debug, Clone, PartialEq, clap::Parser)] +#[derive(Debug, Clone, PartialEq, clap::Parser, Default)] pub struct WasmerEnv { /// Set Wasmer's home directory #[clap(long, env = "WASMER_DIR", default_value = WASMER_DIR.as_os_str())] From 64e969053f7124302e86fbb97c431abd86f99fb2 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 18:41:05 +0200 Subject: [PATCH 41/55] [skip ci] fix(cli): rename `make_pb` and friends to `_spinner` --- lib/cli/src/commands/package/build.rs | 8 ++-- lib/cli/src/commands/package/common/macros.rs | 30 +++++-------- lib/cli/src/commands/package/common/wait.rs | 2 +- lib/cli/src/commands/package/publish.rs | 7 --- lib/cli/src/commands/package/push.rs | 45 +++++++++---------- lib/cli/src/commands/package/tag.rs | 45 +++++++++---------- 6 files changed, 59 insertions(+), 78 deletions(-) diff --git a/lib/cli/src/commands/package/build.rs b/lib/cli/src/commands/package/build.rs index 71dd780fba5..d87d25a469f 100644 --- a/lib/cli/src/commands/package/build.rs +++ b/lib/cli/src/commands/package/build.rs @@ -46,7 +46,7 @@ impl PackageBuild { } } - pub(crate) fn execute(&self) -> Result { + pub(crate) fn execute(&self) -> Result<(Package, PackageHash), anyhow::Error> { let manifest_path = self.manifest_path()?; let Some((_, manifest)) = load_package_manifest(&manifest_path)? else { anyhow::bail!( @@ -58,7 +58,7 @@ impl PackageBuild { let data = pkg.serialize()?; let hash = sha2::Sha256::digest(&data).into(); let pkg_hash = PackageHash::from_sha256_bytes(hash); - + let name = if let Some(manifest_pkg) = manifest.package { if let Some(name) = manifest_pkg.name { if let Some(version) = manifest_pkg.version { @@ -89,7 +89,7 @@ impl PackageBuild { // rest of the code writes the package to disk and is irrelevant // to checking. if self.check { - return Ok(pkg); + return Ok((pkg, pkg_hash)); } pb.println(format!( @@ -136,7 +136,7 @@ impl PackageBuild { out_path.display() )); - Ok(pkg) + Ok((pkg, pkg_hash)) } fn manifest_path(&self) -> Result { diff --git a/lib/cli/src/commands/package/common/macros.rs b/lib/cli/src/commands/package/common/macros.rs index 318efa7e667..b35f1e8eedb 100644 --- a/lib/cli/src/commands/package/common/macros.rs +++ b/lib/cli/src/commands/package/common/macros.rs @@ -1,7 +1,7 @@ -macro_rules! make_pb { - ($self:ident, $msg:expr) => {{ +macro_rules! make_spinner { + ($quiet:expr, $msg:expr) => {{ let pb = indicatif::ProgressBar::new_spinner(); - if $self.quiet { + if $quiet { pb.set_draw_target(indicatif::ProgressDrawTarget::hidden()); } @@ -16,9 +16,9 @@ macro_rules! make_pb { pb }}; - ($self:ident, $msg:expr, $($spinner:expr),+) => {{ + ($quiet:expr, $msg:expr, $($spinner:expr),+) => {{ let pb = indicatif::ProgressBar::new_spinner(); - if $self.quiet { + if $quiet { pb.set_draw_target(indicatif::ProgressDrawTarget::hidden()); } @@ -34,7 +34,7 @@ macro_rules! make_pb { }}; } -macro_rules! pb_ok { +macro_rules! spinner_ok { ($pb:expr, $msg: expr) => { $pb.set_style( indicatif::ProgressStyle::with_template(&format!("{} {{msg}}", "✔".green().bold())) @@ -44,7 +44,7 @@ macro_rules! pb_ok { }; } -macro_rules! pb_err { +macro_rules! spinner_err { ($pb:expr, $msg: expr) => { $pb.set_style( indicatif::ProgressStyle::with_template(&format!("{} {{msg}}", "✘".red().bold())) @@ -63,17 +63,7 @@ macro_rules! bin_name { }; } -macro_rules! cli_line { - () => { - std::env::args() - .filter(|s| !s.starts_with("-")) - .collect::>() - .join(" ") - }; -} - pub(crate) use bin_name; -pub(crate) use cli_line; -pub(crate) use make_pb; -pub(crate) use pb_err; -pub(crate) use pb_ok; +pub(crate) use make_spinner; +pub(crate) use spinner_err; +pub(crate) use spinner_ok; diff --git a/lib/cli/src/commands/package/common/wait.rs b/lib/cli/src/commands/package/common/wait.rs index 21c6e8809d0..ea91999898b 100644 --- a/lib/cli/src/commands/package/common/wait.rs +++ b/lib/cli/src/commands/package/common/wait.rs @@ -146,6 +146,6 @@ pub async fn wait_package( } } - pb_ok!(pb, "Package is available!"); + spinner_ok!(pb, "Package is available!"); Ok(()) } diff --git a/lib/cli/src/commands/package/publish.rs b/lib/cli/src/commands/package/publish.rs index f1d668f0ad1..1862ed85837 100644 --- a/lib/cli/src/commands/package/publish.rs +++ b/lib/cli/src/commands/package/publish.rs @@ -133,13 +133,6 @@ impl AsyncCliCommand for PackagePublish { tracing::info!("Proceeding to invalidate query cache.."); - if let Err(e) = invalidate_graphql_query_cache(&self.env.cache_dir) { - tracing::warn!( - error = &*e, - "Unable to invalidate the cache used for package version queries", - ); - } - if !self.quiet && !self.non_interactive { eprintln!( "{} You can now run your package with {}", diff --git a/lib/cli/src/commands/package/push.rs b/lib/cli/src/commands/package/push.rs index 7c6c5cee697..8b30d957ece 100644 --- a/lib/cli/src/commands/package/push.rs +++ b/lib/cli/src/commands/package/push.rs @@ -121,7 +121,7 @@ impl PackagePush { package_hash: &PackageHash, private: bool, ) -> anyhow::Result<()> { - let pb = make_pb!(self, "Uploading the package to the registry.."); + let pb = make_spinner!(self.quiet, "Uploading the package to the registry.."); let signed_url = upload(client, package_hash, self.timeout.clone(), package).await?; @@ -136,7 +136,7 @@ impl PackagePush { { Some(r) => { if r.success { - pb_ok!(pb, "Succesfully pushed the package to the registry!"); + spinner_ok!(pb, "Succesfully pushed the package to the registry!"); r.package_webc.unwrap().id } else { anyhow::bail!("An unidentified error occurred while publishing the package. (response had success: false)") @@ -156,24 +156,10 @@ impl PackagePush { manifest_path: &PathBuf, ) -> anyhow::Result<(String, PackageHash)> { tracing::info!("Building package"); - let pb = make_pb!( - self, - "Creating the package locally...", - ".", - "o", - "O", - "°", - "O", - "o", - "." - ); - let package = PackageBuild::check(manifest_path.clone()).execute()?; - pb_ok!(pb, "Correctly built package locally"); + let pb = make_spinner!(self.quiet, "Creating the package locally..."); + let (package, hash) = PackageBuild::check(manifest_path.clone()).execute()?; - let hash_bytes = package - .webc_hash() - .ok_or(anyhow::anyhow!("No webc hash was provided"))?; - let hash = PackageHash::from_sha256_bytes(hash_bytes.clone()); + spinner_ok!(pb, "Correctly built package locally"); tracing::info!("Package has hash: {hash}",); let namespace = self.get_namespace(client, &manifest).await?; @@ -181,22 +167,35 @@ impl PackagePush { let private = self.get_privacy(&manifest); tracing::info!("If published, package privacy is {private}"); - let pb = make_pb!(self, "Checking if package is already in the registry.."); + let pb = make_spinner!( + self.quiet, + "Checking if package is already in the registry.." + ); if self.should_push(&client, &hash).await.map_err(on_error)? { if !self.dry_run { tracing::info!("Package should be published"); - pb_ok!(pb, "Package not in the registry yet!"); + spinner_ok!(pb, "Package not in the registry yet!"); self.do_push(&client, &namespace, &package, &hash, private) .await .map_err(on_error)?; } else { tracing::info!("Package should be published, but dry-run is set"); - pb_ok!(pb, "Skipping push as dry-run is set"); + spinner_ok!(pb, "Skipping push as dry-run is set"); } } else { tracing::info!("Package should not be published"); - pb_ok!(pb, "Package was already in the registry, no push needed"); + spinner_ok!(pb, "Package was already in the registry, no push needed"); + } + + tracing::info!("Proceeding to invalidate query cache.."); + + // Prevent `wasmer run` from using stale (cached) package versions after wasmer publish. + if let Err(e) = invalidate_graphql_query_cache(&self.env.cache_dir) { + tracing::warn!( + error = &*e, + "Unable to invalidate the cache used for package version queries", + ); } Ok((namespace, hash)) diff --git a/lib/cli/src/commands/package/tag.rs b/lib/cli/src/commands/package/tag.rs index 0d470ed378f..2f89c5f7666 100644 --- a/lib/cli/src/commands/package/tag.rs +++ b/lib/cli/src/commands/package/tag.rs @@ -154,8 +154,8 @@ impl PackageTag { } = &mut new_ident { let full_pkg_name = format!("{namespace}/{name}"); - let pb = make_pb!( - self, + let pb = make_spinner!( + self.quiet, format!("Checking if a version of {full_pkg_name} already exists..") ); @@ -168,11 +168,11 @@ impl PackageTag { .map(|p| p.version) { let registry_version = semver::Version::parse(®istry_version)?; - pb_ok!( + spinner_ok!( pb, format!("Found version {registry_version} of package {full_pkg_name}") ); - if user_version.clone() < registry_version { + if user_version.clone() <= registry_version { if self.bump { user_version.patch += 1; } else if !self.non_interactive { @@ -186,7 +186,7 @@ impl PackageTag { } } } else { - pb_ok!(pb, format!("No version of {full_pkg_name} in registry!")); + spinner_ok!(pb, format!("No version of {full_pkg_name} in registry!")); } } @@ -208,10 +208,10 @@ impl PackageTag { let tagger = self.synthesize_tagger(client, ident).await?; - let pb = make_pb!(self, "Tagging package..."); + let pb = make_spinner!(self.quiet, "Tagging package..."); if let PackageSpecifier::Hash { .. } = &tagger { - pb_ok!(pb, "Package is unnamed, no need to tag it"); + spinner_ok!(pb, "Package is unnamed, no need to tag it"); return Ok(tagger); } @@ -226,7 +226,7 @@ impl PackageTag { if self.dry_run { tracing::info!("No tagging to do here, dry-run is set"); - pb_ok!(pb, "Skipping tag (dry-run)"); + spinner_ok!(pb, "Skipping tag (dry-run)"); return Ok(tagger); } @@ -282,15 +282,15 @@ impl PackageTag { match r.await? { Some(r) => { if r.success { - pb_ok!(pb, "Successfully tagged package"); + spinner_ok!(pb, "Successfully tagged package"); Ok(tagger) } else { - pb_err!(pb, "Could not tag package!"); + spinner_err!(pb, "Could not tag package!"); anyhow::bail!("An unknown error occurred and the tagging failed.") } } None => { - pb_err!(pb, "Could not tag package!"); + spinner_err!(pb, "Could not tag package!"); anyhow::bail!("The registry returned an empty response.") } } @@ -301,18 +301,21 @@ impl PackageTag { client: &WasmerClient, hash: &PackageHash, ) -> anyhow::Result { - let pb = make_pb!(self, "Checking if the package exists.."); + let pb = make_spinner!(self.quiet, "Checking if the package exists.."); tracing::debug!("Searching for package with hash: {hash}"); let pkg = match wasmer_api::query::get_package_release(client, &hash.to_string()).await? { Some(p) => p, None => { - pb_err!(pb, "The package is not in the registry!"); + spinner_err!(pb, "The package is not in the registry!"); if !self.quiet { eprintln!("\n\nThe package with the required hash does not exist in the selected registry."); let bin_name = bin_name!(); - let cli = cli_line!(); + let cli = std::env::args() + .filter(|s| !s.starts_with("-")) + .collect::>() + .join(" "); if cli.contains("publish") && self.dry_run { eprintln!( @@ -336,7 +339,7 @@ impl PackageTag { } }; - pb_ok!(pb, "Found package in the registry!"); + spinner_ok!(pb, "Found package in the registry!"); Ok(pkg.id) } @@ -381,8 +384,8 @@ impl AsyncCliCommand for PackageTag { tracing::info!("Got manifest at path {}", manifest_path.display()); tracing::info!("Building package"); - let pb = make_pb!( - self, + let pb = make_spinner!( + self.quiet, "Creating the package locally...", ".", "o", @@ -392,13 +395,9 @@ impl AsyncCliCommand for PackageTag { "o", "." ); - let package = PackageBuild::check(manifest_path.clone()).execute()?; - pb_ok!(pb, "Correctly built package locally"); + let (_, hash) = PackageBuild::check(manifest_path.clone()).execute()?; + spinner_ok!(pb, "Correctly built package locally"); - let hash_bytes = package - .webc_hash() - .ok_or(anyhow::anyhow!("No webc hash was provided"))?; - let hash = PackageHash::from_sha256_bytes(hash_bytes); tracing::info!("Package has hash: {hash}",); self.tag(&client, &manifest, hash).await?; From b86af58fcd3f38835eb8bb81f6fe87b12d46aeaa Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 18:41:30 +0200 Subject: [PATCH 42/55] [skip ci] chore: fmt --- lib/registry/src/graphql/subscriptions.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/registry/src/graphql/subscriptions.rs b/lib/registry/src/graphql/subscriptions.rs index 6f2c9986e33..6f0115f4a02 100644 --- a/lib/registry/src/graphql/subscriptions.rs +++ b/lib/registry/src/graphql/subscriptions.rs @@ -7,4 +7,3 @@ use graphql_client::GraphQLQuery; response_derives = "Debug" )] pub struct PackageVersionReady; - From 65e0ca2cc249290ede86b8f2bdf7f13c9034b765 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 18:41:52 +0200 Subject: [PATCH 43/55] fix(cli): Pass `env` field in `Create` tests --- lib/cli/src/commands/app/create.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cli/src/commands/app/create.rs b/lib/cli/src/commands/app/create.rs index 9e8f33894aa..f7045498898 100644 --- a/lib/cli/src/commands/app/create.rs +++ b/lib/cli/src/commands/app/create.rs @@ -713,6 +713,7 @@ mod tests { package: Some("testuser/static-site-1@0.1.0".to_string()), use_local_manifest: false, new_package_name: None, + env: WasmerEnv::default(), }; cmd.run_async().await.unwrap(); @@ -747,6 +748,7 @@ debug: false package: Some("wasmer/testpkg".to_string()), use_local_manifest: false, new_package_name: None, + env: WasmerEnv::default(), }; cmd.run_async().await.unwrap(); @@ -780,6 +782,7 @@ debug: false package: Some("wasmer/test-js-worker".to_string()), use_local_manifest: false, new_package_name: None, + env: WasmerEnv::default(), }; cmd.run_async().await.unwrap(); @@ -816,6 +819,7 @@ debug: false package: Some("wasmer/test-py-worker".to_string()), use_local_manifest: false, new_package_name: None, + env: WasmerEnv::default(), }; cmd.run_async().await.unwrap(); From 1c3d66723a6a2ed4b4f4e1fc2270943448372a5b Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 20:30:41 +0200 Subject: [PATCH 44/55] feat(cli): `wasmer package tag` requires the hash of the package to tag as an argument --- lib/cli/src/commands/package/publish.rs | 5 ++-- lib/cli/src/commands/package/tag.rs | 31 +++++++------------------ 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/lib/cli/src/commands/package/publish.rs b/lib/cli/src/commands/package/publish.rs index 1862ed85837..3aa100bd35e 100644 --- a/lib/cli/src/commands/package/publish.rs +++ b/lib/cli/src/commands/package/publish.rs @@ -127,12 +127,11 @@ impl AsyncCliCommand for PackagePublish { bump: self.bump.clone(), non_interactive: self.non_interactive.clone(), package_path: self.package_path.clone(), + package_hash, } - .tag(&client, &manifest, package_hash) + .tag(&client, &manifest) .await?; - tracing::info!("Proceeding to invalidate query cache.."); - if !self.quiet && !self.non_interactive { eprintln!( "{} You can now run your package with {}", diff --git a/lib/cli/src/commands/package/tag.rs b/lib/cli/src/commands/package/tag.rs index 2f89c5f7666..feefbb5918f 100644 --- a/lib/cli/src/commands/package/tag.rs +++ b/lib/cli/src/commands/package/tag.rs @@ -2,7 +2,7 @@ use super::common::PackageSpecifier; use crate::{ commands::{ package::common::{macros::*, *}, - AsyncCliCommand, PackageBuild, + AsyncCliCommand, }, opts::{ApiOpts, WasmerEnv}, }; @@ -57,6 +57,11 @@ pub struct PackageTag { #[clap(long, default_value_t = !std::io::stdin().is_terminal())] pub non_interactive: bool, + /// The hash of the package to tag + #[clap(name = "hash")] + pub package_hash: PackageHash, + + /// /// Directory containing the `wasmer.toml`, or a custom *.toml manifest file. /// /// Defaults to current working directory. @@ -348,14 +353,13 @@ impl PackageTag { &self, client: &WasmerClient, manifest: &Manifest, - hash: PackageHash, ) -> anyhow::Result { let namespace = self.get_namespace(client, &manifest).await?; - let ident = into_specifier(&manifest, &hash, namespace)?; + let ident = into_specifier(&manifest, &self.package_hash, namespace)?; tracing::info!("PackageIdent extracted from manifest is {:?}", ident); - let package_id = self.get_package_id(&client, &hash).await?; + let package_id = self.get_package_id(&client, &self.package_hash).await?; tracing::info!( "The package identifier returned from the registry is {:?}", package_id @@ -383,24 +387,7 @@ impl AsyncCliCommand for PackageTag { let (manifest_path, manifest) = get_manifest(&self.package_path)?; tracing::info!("Got manifest at path {}", manifest_path.display()); - tracing::info!("Building package"); - let pb = make_spinner!( - self.quiet, - "Creating the package locally...", - ".", - "o", - "O", - "°", - "O", - "o", - "." - ); - let (_, hash) = PackageBuild::check(manifest_path.clone()).execute()?; - spinner_ok!(pb, "Correctly built package locally"); - - tracing::info!("Package has hash: {hash}",); - - self.tag(&client, &manifest, hash).await?; + self.tag(&client, &manifest).await?; Ok(()) } } From 110617b8f021a6b2e4ce13373d5ca110013f4bf9 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 20:31:22 +0200 Subject: [PATCH 45/55] feat(cli): Print hash of the pushed package when execution terminates correctly --- lib/cli/src/commands/package/push.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/cli/src/commands/package/push.rs b/lib/cli/src/commands/package/push.rs index 8b30d957ece..bd7beb6f6ee 100644 --- a/lib/cli/src/commands/package/push.rs +++ b/lib/cli/src/commands/package/push.rs @@ -136,7 +136,16 @@ impl PackagePush { { Some(r) => { if r.success { - spinner_ok!(pb, "Succesfully pushed the package to the registry!"); + let msg = format!( + "Succesfully pushed package {} to the registry!", + package_hash + .to_string() + .trim_start_matches("sha256:") + .chars() + .take(7) + .collect::() + ); + spinner_ok!(pb, msg); r.package_webc.unwrap().id } else { anyhow::bail!("An unidentified error occurred while publishing the package. (response had success: false)") @@ -220,7 +229,17 @@ impl AsyncCliCommand for PackagePush { let (manifest_path, manifest) = get_manifest(&self.package_path)?; tracing::info!("Got manifest at path {}", manifest_path.display()); - self.push(&client, &manifest, &manifest_path).await?; + let (_, hash) = self.push(&client, &manifest, &manifest_path).await?; + + let pb = make_spinner!(self.quiet, ""); + spinner_ok!( + pb, + format!( + "Correctly pushed package {} to the registry", + hash.to_string() + ) + ); + Ok(()) } } From fc63606ffdbe12ab0469186365fa8493e0fb0cef Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 20:31:56 +0200 Subject: [PATCH 46/55] fix(cli): trim "sha256" and more logging --- lib/cli/src/commands/package/common/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cli/src/commands/package/common/mod.rs b/lib/cli/src/commands/package/common/mod.rs index 47eb5e7f4e5..b54bbded786 100644 --- a/lib/cli/src/commands/package/common/mod.rs +++ b/lib/cli/src/commands/package/common/mod.rs @@ -124,6 +124,7 @@ pub(super) async fn upload( package: &Package, ) -> anyhow::Result { let hash_str = hash.to_string(); + let hash_str = hash_str.trim_start_matches("sha256:"); let url = { let default_timeout_secs = Some(60 * 30); @@ -194,6 +195,8 @@ pub(super) async fn upload( let bytes = package.serialize()?; let total_bytes = bytes.len(); + tracing::info!("webc is {total_bytes} bytes long"); + let chunk_size = 1_048_576; // 1MB - 315s / 100MB let mut chunks = bytes.chunks(chunk_size); let mut total_bytes_sent = 0; From af340415a40fcf4136f4337466b218d56d25242a Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 7 May 2024 20:32:39 +0200 Subject: [PATCH 47/55] chore: bump `webc` crate version to `-alpha8` --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cab5806b5be..ee182efa939 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6680,9 +6680,9 @@ dependencies = [ [[package]] name = "wasmer-config" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7439f813ad16b3fc8cbf2be3b54192e2c4b0a4c213d1fb3d7cf82da249c39aea" +checksum = "54a0f70c177b1c5062cfe0f5308c3317751796fef9403c22a0cd7b4cacd4ccd8" dependencies = [ "anyhow", "bytesize", @@ -7221,9 +7221,9 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.0-alpha6" +version = "6.0.0-alpha8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90e7ef808f844f15d1680a7cea4a0c5082ccaae0dc09621855a655f71989eb1" +checksum = "bbf53893f8df356f1305446c1bc59c4082cb592f39ffcae0a2f10bd8ed100bb9" dependencies = [ "anyhow", "base64 0.21.7", @@ -7251,7 +7251,7 @@ dependencies = [ "thiserror", "toml 0.7.8", "url", - "wasmer-config 0.1.1", + "wasmer-config 0.2.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 487a8a9a5fc..2c3c1eed461 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.0.0-alpha6", default-features = false, features = ["package"] } +webc = { version = "6.0.0-alpha8", default-features = false, features = ["package"] } edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" From ff289d77d4bb1bf0474b6f8155dea0eac4e78222 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 8 May 2024 11:31:09 +0200 Subject: [PATCH 48/55] feat(cli): move deploy to per-se fn --- lib/cli/src/commands/app/deploy.rs | 13 ++++-- lib/cli/src/commands/package/publish.rs | 55 +++++++++++++++---------- 2 files changed, 43 insertions(+), 25 deletions(-) diff --git a/lib/cli/src/commands/app/deploy.rs b/lib/cli/src/commands/app/deploy.rs index 576b0c2f320..2fefc34b31d 100644 --- a/lib/cli/src/commands/app/deploy.rs +++ b/lib/cli/src/commands/app/deploy.rs @@ -87,10 +87,11 @@ pub struct CmdAppDeploy { impl CmdAppDeploy { async fn publish( &self, + client: &WasmerClient, owner: String, manifest_dir_path: PathBuf, ) -> anyhow::Result { - let (_, manifest) = match load_package_manifest(&manifest_dir_path)? { + let (manifest_path, manifest) = match load_package_manifest(&manifest_dir_path)? { Some(r) => r, None => anyhow::bail!( "Could not read or find manifest in path '{}'!", @@ -120,7 +121,9 @@ impl CmdAppDeploy { api: self.api.clone(), }; - publish_cmd.run_async().await + publish_cmd + .publish(&client, &manifest_path, &manifest) + .await } async fn get_owner( @@ -297,7 +300,9 @@ impl AsyncCliCommand for CmdAppDeploy { .display() ); - let package_id = self.publish(owner.clone(), PathBuf::from(path)).await?; + let package_id = self + .publish(&client, owner.clone(), PathBuf::from(path)) + .await?; app_cfg_new.package = package_id.into(); @@ -355,7 +360,7 @@ impl AsyncCliCommand for CmdAppDeploy { ); let package_id = - self.publish(owner.clone(), manifest_path).await?; + self.publish(&client, owner.clone(), manifest_path).await?; app_config.package = package_id.into(); diff --git a/lib/cli/src/commands/package/publish.rs b/lib/cli/src/commands/package/publish.rs index 3aa100bd35e..3a04dec0c50 100644 --- a/lib/cli/src/commands/package/publish.rs +++ b/lib/cli/src/commands/package/publish.rs @@ -11,8 +11,9 @@ use crate::{ }; use colored::Colorize; use is_terminal::IsTerminal; +use wasmer_api::WasmerClient; use std::path::PathBuf; -use wasmer_config::package::PackageIdent; +use wasmer_config::package::{PackageIdent, Manifest}; /// Publish (push and tag) a package to the registry. #[derive(Debug, clap::Parser)] @@ -72,7 +73,7 @@ pub struct PackagePublish { pub timeout: humantime::Duration, /// Whether or not the patch field of the version of the package - if any - should be bumped. - #[clap(long, conflicts_with = "version")] + #[clap(long, conflicts_with = "package_version")] pub bump: bool, /// Do not prompt for user input. @@ -80,24 +81,13 @@ pub struct PackagePublish { pub non_interactive: bool, } -#[async_trait::async_trait] -impl AsyncCliCommand for PackagePublish { - type Output = PackageIdent; - - async fn run_async(self) -> Result { - tracing::info!("Checking if user is logged in"); - let client = login_user( - &self.api, - &self.env, - !self.non_interactive, - "publish a package", - ) - .await?; - - tracing::info!("Loading manifest"); - let (manifest_path, manifest) = get_manifest(&self.package_path)?; - tracing::info!("Got manifest at path {}", manifest_path.display()); - +impl PackagePublish { + pub async fn publish( + &self, + client: &WasmerClient, + manifest_path: &PathBuf, + manifest: &Manifest, + ) -> anyhow::Result { let (package_namespace, package_hash) = { let push_cmd = PackagePush { api: self.api.clone(), @@ -115,7 +105,7 @@ impl AsyncCliCommand for PackagePublish { push_cmd.push(&client, &manifest, &manifest_path).await? }; - let ident = PackageTag { + PackageTag { api: self.api.clone(), env: self.env.clone(), dry_run: self.dry_run.clone(), @@ -130,8 +120,30 @@ impl AsyncCliCommand for PackagePublish { package_hash, } .tag(&client, &manifest) + .await + } +} + +#[async_trait::async_trait] +impl AsyncCliCommand for PackagePublish { + type Output = PackageIdent; + + async fn run_async(self) -> Result { + tracing::info!("Checking if user is logged in"); + let client = login_user( + &self.api, + &self.env, + !self.non_interactive, + "publish a package", + ) .await?; + tracing::info!("Loading manifest"); + let (manifest_path, manifest) = get_manifest(&self.package_path)?; + tracing::info!("Got manifest at path {}", manifest_path.display()); + + let ident = self.publish(&client, &manifest_path, &manifest).await?; + if !self.quiet && !self.non_interactive { eprintln!( "{} You can now run your package with {}", @@ -139,6 +151,7 @@ impl AsyncCliCommand for PackagePublish { format!("`{} run {ident}`", bin_name!()).bold() ); } + Ok(ident) } } From 3abed76eede529e45c155e0854c4ae2c8a54bd8a Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 8 May 2024 11:31:42 +0200 Subject: [PATCH 49/55] feat(cli): continue push-tag flow --- Cargo.lock | 11 ++--- lib/cli/src/commands/package/push.rs | 59 ++++++++++++++-------- lib/cli/src/commands/package/tag.rs | 73 +++++++++++++++++++++------- lib/cli/src/utils/mod.rs | 24 +++++++++ 4 files changed, 122 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9a17b3e125..f2b8cfb7d9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6658,21 +6658,21 @@ dependencies = [ [[package]] name = "wasmer-config" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b405c9856adaf65ee91eeeeaac6fcc6b127188648c60ae4e89de63f506c74e6" +version = "0.2.0" dependencies = [ "anyhow", "bytesize", "derive_builder", "hex", - "indexmap 1.9.3", + "indexmap 2.2.6", + "pretty_assertions", "schemars", "semver 1.0.23", "serde", "serde_cbor", "serde_json", "serde_yaml 0.9.34+deprecated", + "tempfile", "thiserror", "toml 0.8.12", "url", @@ -6695,7 +6695,6 @@ dependencies = [ "serde_cbor", "serde_json", "serde_yaml 0.9.34+deprecated", - "tempfile", "thiserror", "toml 0.8.12", "url", @@ -7252,7 +7251,7 @@ dependencies = [ "thiserror", "toml 0.7.8", "url", - "wasmer-config 0.2.0", + "wasmer-config 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/lib/cli/src/commands/package/push.rs b/lib/cli/src/commands/package/push.rs index bd7beb6f6ee..8081022c0e5 100644 --- a/lib/cli/src/commands/package/push.rs +++ b/lib/cli/src/commands/package/push.rs @@ -91,11 +91,8 @@ impl PackagePush { } let user = wasmer_api::query::current_user_with_namespaces(&client, None).await?; - let owner = crate::utils::prompts::prompt_for_namespace( - "Who should own this package?", - None, - Some(&user), - )?; + let owner = + crate::utils::prompts::prompt_for_namespace("Choose a namespace", None, Some(&user))?; Ok(owner.clone()) } @@ -137,13 +134,7 @@ impl PackagePush { Some(r) => { if r.success { let msg = format!( - "Succesfully pushed package {} to the registry!", - package_hash - .to_string() - .trim_start_matches("sha256:") - .chars() - .take(7) - .collect::() + "Succesfully pushed release to namespace {namespace} on the registry" ); spinner_ok!(pb, msg); r.package_webc.unwrap().id @@ -183,7 +174,8 @@ impl PackagePush { if self.should_push(&client, &hash).await.map_err(on_error)? { if !self.dry_run { tracing::info!("Package should be published"); - spinner_ok!(pb, "Package not in the registry yet!"); + pb.finish_and_clear(); + // spinner_ok!(pb, "Package not in the registry yet!"); self.do_push(&client, &namespace, &package, &hash, private) .await @@ -231,14 +223,39 @@ impl AsyncCliCommand for PackagePush { let (_, hash) = self.push(&client, &manifest, &manifest_path).await?; - let pb = make_spinner!(self.quiet, ""); - spinner_ok!( - pb, - format!( - "Correctly pushed package {} to the registry", - hash.to_string() - ) - ); + if !self.quiet { + let bin_name = bin_name!(); + if let Some(package) = &manifest.package { + if package.name.is_some() { + let mut manifest_path_dir = manifest_path.clone(); + manifest_path_dir.pop(); + + eprintln!( + "You can now tag your package with `{}`", + format!( + "{bin_name} package tag {}{}", + hash.to_string(), + if manifest_path_dir.canonicalize()? == std::env::current_dir()? { + String::new() + } else { + format!(" {}", manifest_path_dir.display()) + } + ) + .bold() + ) + } else { + eprintln!( + "You can now run your package with `{}`", + format!("{bin_name} run {}", hash.to_string()).bold() + ); + } + } else { + eprintln!( + "You can now run your package with `{}`", + format!("{bin_name} run {}", hash.to_string()).bold() + ); + } + } Ok(()) } diff --git a/lib/cli/src/commands/package/tag.rs b/lib/cli/src/commands/package/tag.rs index feefbb5918f..b1c9c717e54 100644 --- a/lib/cli/src/commands/package/tag.rs +++ b/lib/cli/src/commands/package/tag.rs @@ -5,6 +5,7 @@ use crate::{ AsyncCliCommand, }, opts::{ApiOpts, WasmerEnv}, + utils::prompt_for_package_version, }; use colored::Colorize; use dialoguer::{theme::ColorfulTheme, Confirm}; @@ -50,7 +51,7 @@ pub struct PackageTag { pub timeout: humantime::Duration, /// Whether or not the patch field of the version of the package - if any - should be bumped. - #[clap(long, conflicts_with = "version")] + #[clap(long, conflicts_with = "package_version")] pub bump: bool, /// Do not prompt for user input. @@ -93,11 +94,8 @@ impl PackageTag { } let user = wasmer_api::query::current_user_with_namespaces(&client, None).await?; - let owner = crate::utils::prompts::prompt_for_namespace( - "Who should own this package?", - None, - Some(&user), - )?; + let owner = + crate::utils::prompts::prompt_for_namespace("Choose a namespace", None, Some(&user))?; Ok(owner.clone()) } @@ -106,6 +104,7 @@ impl PackageTag { &self, client: &WasmerClient, ident: &PackageSpecifier, + package_release_id: &wasmer_api::types::Id, ) -> anyhow::Result { let mut new_ident = ident.clone(); @@ -155,7 +154,7 @@ impl PackageTag { if let PackageSpecifier::Named { name, namespace, - tag: Tag::Version(user_version), + tag, } = &mut new_ident { let full_pkg_name = format!("{namespace}/{name}"); @@ -164,34 +163,53 @@ impl PackageTag { format!("Checking if a version of {full_pkg_name} already exists..") ); - if let Some(registry_version) = wasmer_api::query::get_package_version( + if let Some(p) = wasmer_api::query::get_package_version( client, full_pkg_name.clone(), String::from("latest"), ) .await? - .map(|p| p.version) { - let registry_version = semver::Version::parse(®istry_version)?; + let registry_version = semver::Version::parse(&p.version)?; spinner_ok!( pb, format!("Found version {registry_version} of package {full_pkg_name}") ); - if user_version.clone() <= registry_version { + + tracing::info!( + "Package to tag has id = {}, while latest version has id = {}", + package_release_id.inner(), + p.id.inner() + ); + + if let Tag::Hash(_) = &tag { + *tag = Tag::Version(registry_version.clone()); + } + + let user_version: &mut semver::Version = match tag { + Tag::Version(v) => v, + Tag::Hash(_) => unreachable!(), + }; + + if user_version.clone() <= registry_version && (&p.id != package_release_id) { if self.bump { - user_version.patch += 1; + *user_version = registry_version.clone(); + user_version.patch = registry_version.patch + 1; } else if !self.non_interactive { - eprintln!("The registry has a newer version of the package."); let theme = ColorfulTheme::default(); - let mut new_version = user_version.clone(); + let mut new_version = registry_version.clone(); new_version.patch += 1; if Confirm::with_theme(&theme).with_prompt(format!("Do you want to bump the package's version ({user_version} -> {new_version})?")).interact()? { - user_version.patch += 1; + *user_version = new_version.clone(); + // TODO: serialize new version? } } } } else { spinner_ok!(pb, format!("No version of {full_pkg_name} in registry!")); + let version = + prompt_for_package_version("Enter the package version", Some("0.1.0"))?; + *tag = Tag::Version(version); } } @@ -211,7 +229,9 @@ impl PackageTag { ident ); - let tagger = self.synthesize_tagger(client, ident).await?; + let tagger = self + .synthesize_tagger(client, ident, package_release_id) + .await?; let pb = make_spinner!(self.quiet, "Tagging package..."); @@ -287,7 +307,13 @@ impl PackageTag { match r.await? { Some(r) => { if r.success { - spinner_ok!(pb, "Successfully tagged package"); + spinner_ok!( + pb, + format!( + "Successfully tagged package {}", + Into::::into(tagger.clone()) + ) + ); Ok(tagger) } else { spinner_err!(pb, "Could not tag package!"); @@ -344,7 +370,17 @@ impl PackageTag { } }; - spinner_ok!(pb, "Found package in the registry!"); + spinner_ok!( + pb, + format!( + "Found package {} in the registry", + hash.to_string() + .trim_start_matches("sha256:") + .chars() + .take(7) + .collect::() + ) + ); Ok(pkg.id) } @@ -388,6 +424,7 @@ impl AsyncCliCommand for PackageTag { tracing::info!("Got manifest at path {}", manifest_path.display()); self.tag(&client, &manifest).await?; + Ok(()) } } diff --git a/lib/cli/src/utils/mod.rs b/lib/cli/src/utils/mod.rs index 14c001e8717..4035b1bb192 100644 --- a/lib/cli/src/utils/mod.rs +++ b/lib/cli/src/utils/mod.rs @@ -135,6 +135,30 @@ pub fn prompt_for_package_name( } } +/// Ask a user for a package name. +/// +/// Will continue looping until the user provides a valid name. +pub fn prompt_for_package_version( + message: &str, + default: Option<&str>, +) -> Result { + loop { + let theme = ColorfulTheme::default(); + let raw: String = dialoguer::Input::with_theme(&theme) + .with_prompt(message) + .with_initial_text(default.unwrap_or_default()) + .interact_text() + .context("could not read user input")?; + + match raw.parse::() { + Ok(p) => break Ok(p), + Err(err) => { + eprintln!("invalid package version: {err}"); + } + } + } +} + /// Defines how to check for a package. pub enum PackageCheckMode { /// The package must exist in the registry. From ceb200e72936a9e1b4727db4e5c1f0f8baa5cc97 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Wed, 8 May 2024 14:03:59 +0330 Subject: [PATCH 50/55] sync modified time stamp upon creating the inode --- lib/wasix/src/syscalls/wasi/path_open.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/wasix/src/syscalls/wasi/path_open.rs b/lib/wasix/src/syscalls/wasi/path_open.rs index 58528347995..243982c6044 100644 --- a/lib/wasix/src/syscalls/wasi/path_open.rs +++ b/lib/wasix/src/syscalls/wasi/path_open.rs @@ -248,6 +248,7 @@ pub(crate) fn path_open_internal( if let Some(handle) = handle { let handle = handle.read().unwrap(); + inode.stat.write().unwrap().st_mtim = handle.last_modified(); if let Some(fd) = handle.get_special_fd() { // We clone the file descriptor so that when its closed // nothing bad happens From e640692188f24369cef320169fbf90b4e9cb1426 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 8 May 2024 12:43:13 +0200 Subject: [PATCH 51/55] feat(cli): Small nits regarding user-facing messages --- lib/cli/src/commands/app/deploy.rs | 9 +++++++-- lib/cli/src/commands/package/common/wait.rs | 4 +--- lib/cli/src/commands/package/tag.rs | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/cli/src/commands/app/deploy.rs b/lib/cli/src/commands/app/deploy.rs index 2fefc34b31d..80a7b89e977 100644 --- a/lib/cli/src/commands/app/deploy.rs +++ b/lib/cli/src/commands/app/deploy.rs @@ -250,9 +250,14 @@ impl AsyncCliCommand for CmdAppDeploy { config_str = format!("{}\nname: {}", config_str, self.app_name.as_ref().unwrap()); } else if app_yaml.get("name").is_none() { if !self.non_interactive { + let default_name = std::env::current_dir().ok().and_then(|dir| { + dir.file_name() + .and_then(|f| f.to_str()) + .map(|s| s.to_owned()) + }); let app_name = crate::utils::prompts::prompt_new_app_name( "Enter the name of the app", - None, + default_name.as_deref(), &owner, self.api.client().ok().as_ref(), ) @@ -551,7 +556,7 @@ pub async fn deploy_app_verbose( let make_default = opts.make_default; - eprintln!("Deploying app {} to Wasmer Edge...\n", pretty_name); + eprintln!("\nDeploying app {} to Wasmer Edge...\n", pretty_name); let wait = opts.wait; let version = deploy_app(client, opts).await?; diff --git a/lib/cli/src/commands/package/common/wait.rs b/lib/cli/src/commands/package/common/wait.rs index ea91999898b..d061b0f2ba5 100644 --- a/lib/cli/src/commands/package/common/wait.rs +++ b/lib/cli/src/commands/package/common/wait.rs @@ -1,5 +1,3 @@ -use super::macros::*; -use colored::Colorize; use futures_util::StreamExt; use indicatif::ProgressBar; use wasmer_api::WasmerClient; @@ -146,6 +144,6 @@ pub async fn wait_package( } } - spinner_ok!(pb, "Package is available!"); + pb.finish_and_clear(); Ok(()) } diff --git a/lib/cli/src/commands/package/tag.rs b/lib/cli/src/commands/package/tag.rs index b1c9c717e54..5bc63ada6d3 100644 --- a/lib/cli/src/commands/package/tag.rs +++ b/lib/cli/src/commands/package/tag.rs @@ -373,7 +373,7 @@ impl PackageTag { spinner_ok!( pb, format!( - "Found package {} in the registry", + "Found package in the registry ({})", hash.to_string() .trim_start_matches("sha256:") .chars() From 4415490959227062a12badf00b1a04cfa4189b45 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 8 May 2024 13:32:59 +0200 Subject: [PATCH 52/55] chore(cli): Make linter happy --- lib/cli/src/commands/package/publish.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cli/src/commands/package/publish.rs b/lib/cli/src/commands/package/publish.rs index 3a04dec0c50..9626b6bc1f0 100644 --- a/lib/cli/src/commands/package/publish.rs +++ b/lib/cli/src/commands/package/publish.rs @@ -11,9 +11,9 @@ use crate::{ }; use colored::Colorize; use is_terminal::IsTerminal; -use wasmer_api::WasmerClient; use std::path::PathBuf; -use wasmer_config::package::{PackageIdent, Manifest}; +use wasmer_api::WasmerClient; +use wasmer_config::package::{Manifest, PackageIdent}; /// Publish (push and tag) a package to the registry. #[derive(Debug, clap::Parser)] From d2389f8f23c94cf3c71282a7a43d0b1da04395c7 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 8 May 2024 14:03:21 +0200 Subject: [PATCH 53/55] chore(cli): Make linter happy(er) --- lib/cli/src/commands/app/deploy.rs | 4 +-- lib/cli/src/commands/package/common/mod.rs | 30 ++++++++++---------- lib/cli/src/commands/package/publish.rs | 24 ++++++++-------- lib/cli/src/commands/package/push.rs | 32 +++++++++++----------- lib/cli/src/commands/package/tag.rs | 14 +++++----- lib/emscripten/src/time.rs | 5 ++-- 6 files changed, 55 insertions(+), 54 deletions(-) diff --git a/lib/cli/src/commands/app/deploy.rs b/lib/cli/src/commands/app/deploy.rs index 80a7b89e977..6e3f457c18e 100644 --- a/lib/cli/src/commands/app/deploy.rs +++ b/lib/cli/src/commands/app/deploy.rs @@ -121,9 +121,7 @@ impl CmdAppDeploy { api: self.api.clone(), }; - publish_cmd - .publish(&client, &manifest_path, &manifest) - .await + publish_cmd.publish(client, &manifest_path, &manifest).await } async fn get_owner( diff --git a/lib/cli/src/commands/package/common/mod.rs b/lib/cli/src/commands/package/common/mod.rs index b54bbded786..4cea7746bca 100644 --- a/lib/cli/src/commands/package/common/mod.rs +++ b/lib/cli/src/commands/package/common/mod.rs @@ -6,7 +6,11 @@ use crate::{ use colored::Colorize; use dialoguer::Confirm; use semver::VersionReq; -use std::{collections::BTreeMap, path::PathBuf, str::FromStr}; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, + str::FromStr, +}; use wasmer_api::WasmerClient; use wasmer_config::package::{Manifest, NamedPackageIdent, PackageHash, PackageIdent}; use webc::wasmer_package::Package; @@ -35,9 +39,9 @@ pub(super) enum Tag { Hash(PackageHash), } -impl Into for PackageSpecifier { - fn into(self) -> PackageIdent { - match self { +impl From for PackageIdent { + fn from(value: PackageSpecifier) -> Self { + match value { PackageSpecifier::Hash { hash, .. } => PackageIdent::Hash(hash), PackageSpecifier::Named { namespace, @@ -75,7 +79,7 @@ pub(super) fn into_specifier( }, Some(n) => match &n.name { Some(name) => { - let named = NamedPackageIdent::from_str(&name)?; + let named = NamedPackageIdent::from_str(name)?; match &n.version { Some(v) => PackageSpecifier::Named { namespace, @@ -109,7 +113,7 @@ pub(super) fn on_error(e: anyhow::Error) -> anyhow::Error { // are cleaner ways to achieve this, but for now we're just going to // clear out the whole GraphQL query cache. // See https://github.com/wasmerio/wasmer/pull/3983 for more -pub(super) fn invalidate_graphql_query_cache(cache_dir: &PathBuf) -> Result<(), anyhow::Error> { +pub(super) fn invalidate_graphql_query_cache(cache_dir: &Path) -> Result<(), anyhow::Error> { let cache_dir = cache_dir.join("queries"); std::fs::remove_dir_all(cache_dir)?; @@ -129,9 +133,9 @@ pub(super) async fn upload( let url = { let default_timeout_secs = Some(60 * 30); let q = wasmer_api::query::get_signed_url_for_package_upload( - &client, + client, default_timeout_secs, - Some(&hash_str), + Some(hash_str), None, None, ); @@ -198,14 +202,12 @@ pub(super) async fn upload( tracing::info!("webc is {total_bytes} bytes long"); let chunk_size = 1_048_576; // 1MB - 315s / 100MB - let mut chunks = bytes.chunks(chunk_size); + let chunks = bytes.chunks(chunk_size); let mut total_bytes_sent = 0; let client = reqwest::Client::builder().build().unwrap(); - while let Some(chunk) = chunks.next() { - // TODO: add upload pbar. - + for chunk in chunks { let n = chunk.len(); let start = total_bytes_sent; @@ -236,8 +238,8 @@ pub(super) async fn upload( /// // The difference with the `load_package_manifest` is that // this function returns an error if no manifest is found. -pub(super) fn get_manifest(path: &PathBuf) -> anyhow::Result<(PathBuf, Manifest)> { - load_package_manifest(&path).and_then(|j| { +pub(super) fn get_manifest(path: &Path) -> anyhow::Result<(PathBuf, Manifest)> { + load_package_manifest(path).and_then(|j| { j.ok_or_else(|| anyhow::anyhow!("No valid manifest found in path '{}'", path.display())) }) } diff --git a/lib/cli/src/commands/package/publish.rs b/lib/cli/src/commands/package/publish.rs index 9626b6bc1f0..bb7617065ab 100644 --- a/lib/cli/src/commands/package/publish.rs +++ b/lib/cli/src/commands/package/publish.rs @@ -11,7 +11,7 @@ use crate::{ }; use colored::Colorize; use is_terminal::IsTerminal; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use wasmer_api::WasmerClient; use wasmer_config::package::{Manifest, PackageIdent}; @@ -85,7 +85,7 @@ impl PackagePublish { pub async fn publish( &self, client: &WasmerClient, - manifest_path: &PathBuf, + manifest_path: &Path, manifest: &Manifest, ) -> anyhow::Result { let (package_namespace, package_hash) = { @@ -95,31 +95,31 @@ impl PackagePublish { dry_run: self.dry_run, quiet: self.quiet, package_namespace: self.package_namespace.clone(), - timeout: self.timeout.clone(), - bump: self.bump.clone(), - non_interactive: self.non_interactive.clone(), - wait: self.wait.clone(), + timeout: self.timeout, + bump: self.bump, + non_interactive: self.non_interactive, + wait: self.wait, package_path: self.package_path.clone(), }; - push_cmd.push(&client, &manifest, &manifest_path).await? + push_cmd.push(client, manifest, manifest_path).await? }; PackageTag { api: self.api.clone(), env: self.env.clone(), - dry_run: self.dry_run.clone(), + dry_run: self.dry_run, quiet: self.quiet, package_namespace: Some(package_namespace), package_name: self.package_name.clone(), package_version: self.package_version.clone(), - timeout: self.timeout.clone(), - bump: self.bump.clone(), - non_interactive: self.non_interactive.clone(), + timeout: self.timeout, + bump: self.bump, + non_interactive: self.non_interactive, package_path: self.package_path.clone(), package_hash, } - .tag(&client, &manifest) + .tag(client, manifest) .await } } diff --git a/lib/cli/src/commands/package/push.rs b/lib/cli/src/commands/package/push.rs index 8081022c0e5..06e9fd8b4c3 100644 --- a/lib/cli/src/commands/package/push.rs +++ b/lib/cli/src/commands/package/push.rs @@ -5,7 +5,7 @@ use crate::{ }; use colored::Colorize; use is_terminal::IsTerminal; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use wasmer_api::WasmerClient; use wasmer_config::package::{Manifest, PackageHash}; use webc::wasmer_package::Package; @@ -79,7 +79,7 @@ impl PackagePush { if let Some(pkg) = &manifest.package { if let Some(ns) = &pkg.name { - if let Some(first) = ns.split("/").next() { + if let Some(first) = ns.split('/').next() { return Ok(first.to_string()); } } @@ -90,7 +90,7 @@ impl PackagePush { anyhow::bail!("No package namespace specified: use --namespace XXX"); } - let user = wasmer_api::query::current_user_with_namespaces(&client, None).await?; + let user = wasmer_api::query::current_user_with_namespaces(client, None).await?; let owner = crate::utils::prompts::prompt_for_namespace("Choose a namespace", None, Some(&user))?; @@ -107,7 +107,7 @@ impl PackagePush { async fn should_push(&self, client: &WasmerClient, hash: &PackageHash) -> anyhow::Result { let res = wasmer_api::query::get_package_release(client, &hash.to_string()).await; tracing::info!("{:?}", res); - res.map(|p| !p.is_some()) + res.map(|p| p.is_none()) } async fn do_push( @@ -120,7 +120,7 @@ impl PackagePush { ) -> anyhow::Result<()> { let pb = make_spinner!(self.quiet, "Uploading the package to the registry.."); - let signed_url = upload(client, package_hash, self.timeout.clone(), package).await?; + let signed_url = upload(client, package_hash, self.timeout, package).await?; let id = match wasmer_api::query::push_package_release( client, @@ -145,7 +145,7 @@ impl PackagePush { None => anyhow::bail!("An unidentified error occurred while publishing the package."), // <- This is extremely bad.. }; - wait_package(client, self.wait, id, &pb, self.timeout.clone()).await?; + wait_package(client, self.wait, id, &pb, self.timeout).await?; Ok(()) } @@ -153,31 +153,31 @@ impl PackagePush { &self, client: &WasmerClient, manifest: &Manifest, - manifest_path: &PathBuf, + manifest_path: &Path, ) -> anyhow::Result<(String, PackageHash)> { tracing::info!("Building package"); let pb = make_spinner!(self.quiet, "Creating the package locally..."); - let (package, hash) = PackageBuild::check(manifest_path.clone()).execute()?; + let (package, hash) = PackageBuild::check(manifest_path.to_path_buf()).execute()?; spinner_ok!(pb, "Correctly built package locally"); - tracing::info!("Package has hash: {hash}",); + tracing::info!("Package has hash: {hash}"); - let namespace = self.get_namespace(client, &manifest).await?; + let namespace = self.get_namespace(client, manifest).await?; - let private = self.get_privacy(&manifest); + let private = self.get_privacy(manifest); tracing::info!("If published, package privacy is {private}"); let pb = make_spinner!( self.quiet, "Checking if package is already in the registry.." ); - if self.should_push(&client, &hash).await.map_err(on_error)? { + if self.should_push(client, &hash).await.map_err(on_error)? { if !self.dry_run { tracing::info!("Package should be published"); pb.finish_and_clear(); // spinner_ok!(pb, "Package not in the registry yet!"); - self.do_push(&client, &namespace, &package, &hash, private) + self.do_push(client, &namespace, &package, &hash, private) .await .map_err(on_error)?; } else { @@ -234,7 +234,7 @@ impl AsyncCliCommand for PackagePush { "You can now tag your package with `{}`", format!( "{bin_name} package tag {}{}", - hash.to_string(), + hash, if manifest_path_dir.canonicalize()? == std::env::current_dir()? { String::new() } else { @@ -246,13 +246,13 @@ impl AsyncCliCommand for PackagePush { } else { eprintln!( "You can now run your package with `{}`", - format!("{bin_name} run {}", hash.to_string()).bold() + format!("{bin_name} run {}", hash).bold() ); } } else { eprintln!( "You can now run your package with `{}`", - format!("{bin_name} run {}", hash.to_string()).bold() + format!("{bin_name} run {}", hash).bold() ); } } diff --git a/lib/cli/src/commands/package/tag.rs b/lib/cli/src/commands/package/tag.rs index 5bc63ada6d3..1490466dd64 100644 --- a/lib/cli/src/commands/package/tag.rs +++ b/lib/cli/src/commands/package/tag.rs @@ -82,7 +82,7 @@ impl PackageTag { if let Some(pkg) = &manifest.package { if let Some(ns) = &pkg.name { - if let Some(first) = ns.split("/").next() { + if let Some(first) = ns.split('/').next() { return Ok(first.to_string()); } } @@ -93,7 +93,7 @@ impl PackageTag { anyhow::bail!("No package namespace specified: use --namespace XXX"); } - let user = wasmer_api::query::current_user_with_namespaces(&client, None).await?; + let user = wasmer_api::query::current_user_with_namespaces(client, None).await?; let owner = crate::utils::prompts::prompt_for_namespace("Choose a namespace", None, Some(&user))?; @@ -344,7 +344,7 @@ impl PackageTag { eprintln!("\n\nThe package with the required hash does not exist in the selected registry."); let bin_name = bin_name!(); let cli = std::env::args() - .filter(|s| !s.starts_with("-")) + .filter(|s| !s.starts_with('-')) .collect::>() .join(" "); @@ -390,19 +390,19 @@ impl PackageTag { client: &WasmerClient, manifest: &Manifest, ) -> anyhow::Result { - let namespace = self.get_namespace(client, &manifest).await?; + let namespace = self.get_namespace(client, manifest).await?; - let ident = into_specifier(&manifest, &self.package_hash, namespace)?; + let ident = into_specifier(manifest, &self.package_hash, namespace)?; tracing::info!("PackageIdent extracted from manifest is {:?}", ident); - let package_id = self.get_package_id(&client, &self.package_hash).await?; + let package_id = self.get_package_id(client, &self.package_hash).await?; tracing::info!( "The package identifier returned from the registry is {:?}", package_id ); let ident = self - .do_tag(&client, &ident, &manifest, &package_id) + .do_tag(client, &ident, manifest, &package_id) .await .map_err(on_error)?; diff --git a/lib/emscripten/src/time.rs b/lib/emscripten/src/time.rs index e77c704fdff..40a34752fd2 100644 --- a/lib/emscripten/src/time.rs +++ b/lib/emscripten/src/time.rs @@ -4,6 +4,7 @@ use libc::{c_char, c_int}; // use libc::{c_char, c_int, clock_getres, clock_settime}; use std::mem; use std::time::SystemTime; +use time::InstantExt; #[cfg(not(target_os = "windows"))] use libc::{clockid_t, time as libc_time, timegm as libc_timegm, tm as libc_tm}; @@ -100,10 +101,10 @@ pub fn _clock_gettime(ctx: FunctionEnvMut, clk_id: clockid_t, tp: c_int) CLOCK_MONOTONIC | CLOCK_MONOTONIC_COARSE => { lazy_static! { - static ref PRECISE0: time::Instant = time::Instant::now(); + static ref PRECISE0 = std::time::Instant::now(); }; let precise_ns = *PRECISE0; - (time::Instant::now() - precise_ns).whole_nanoseconds() + (std::time::Instant::now() - precise_ns).whole_nanoseconds() } _ => panic!("Clock with id \"{}\" is not supported.", clk_id), }; From 3b2328f5060221e5b8c4ebca379838faae7dc9a2 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 8 May 2024 14:09:08 +0200 Subject: [PATCH 54/55] fix(emscripten): revert back to previous times --- lib/emscripten/src/time.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/emscripten/src/time.rs b/lib/emscripten/src/time.rs index 40a34752fd2..e77c704fdff 100644 --- a/lib/emscripten/src/time.rs +++ b/lib/emscripten/src/time.rs @@ -4,7 +4,6 @@ use libc::{c_char, c_int}; // use libc::{c_char, c_int, clock_getres, clock_settime}; use std::mem; use std::time::SystemTime; -use time::InstantExt; #[cfg(not(target_os = "windows"))] use libc::{clockid_t, time as libc_time, timegm as libc_timegm, tm as libc_tm}; @@ -101,10 +100,10 @@ pub fn _clock_gettime(ctx: FunctionEnvMut, clk_id: clockid_t, tp: c_int) CLOCK_MONOTONIC | CLOCK_MONOTONIC_COARSE => { lazy_static! { - static ref PRECISE0 = std::time::Instant::now(); + static ref PRECISE0: time::Instant = time::Instant::now(); }; let precise_ns = *PRECISE0; - (std::time::Instant::now() - precise_ns).whole_nanoseconds() + (time::Instant::now() - precise_ns).whole_nanoseconds() } _ => panic!("Clock with id \"{}\" is not supported.", clk_id), }; From 903092329ae0ebe6b2c92e4cd2f1966c0998b85d Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 8 May 2024 14:25:06 +0200 Subject: [PATCH 55/55] fix(wasix): Add sysinfoapi to winapi create features --- lib/wasix/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index d586e898588..5c02e9ea0bb 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -120,7 +120,7 @@ libc = { version = "^0.2", default-features = false } termios = { version = "0.3" } [target.'cfg(windows)'.dependencies] -winapi = "0.3" +winapi = { version = "0.3", features = ["sysinfoapi"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] terminal_size = { version = "0.3.0" }