From 724d2cc48cbbca2a86a25be45072424c77363158 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 14:20:40 +0330 Subject: [PATCH 1/5] handle local package resolution --- lib/cli/src/commands/run/mod.rs | 11 +++---- lib/wasix/src/bin_factory/binary_package.rs | 33 ++++++++++++++++++++- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 468584cb9b6..2c7a226a8cc 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -673,15 +673,12 @@ impl ExecutableTarget { pb: &ProgressBar, ) -> Result { pb.set_message(format!("Loading \"{}\" into memory", dir.display())); - - let manifest_path = dir.join("wasmer.toml"); - let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?; - let container = Container::from(webc); - pb.set_message("Resolving dependencies"); let inner_runtime = runtime.clone(); - let pkg = runtime.task_manager().spawn_and_block_on(async move { - BinaryPackage::from_webc(&container, inner_runtime.as_ref()).await + let pkg = runtime.task_manager().spawn_and_block_on({ + let path = dir.to_path_buf(); + + async move { BinaryPackage::from_dir(&path, inner_runtime.as_ref()).await } })??; Ok(ExecutableTarget::Package(pkg)) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 6abe0f9bbc6..dbf2706ca7c 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -1,8 +1,9 @@ -use std::sync::Arc; +use std::{os::unix::ffi::OsStrExt, path::Path, sync::Arc}; use anyhow::Context; use derivative::*; use once_cell::sync::OnceCell; +use sha2::Digest; use virtual_fs::FileSystem; use wasmer_config::package::{PackageHash, PackageId, PackageSource}; use webc::{compat::SharedBytes, Container}; @@ -79,6 +80,36 @@ pub struct BinaryPackage { } impl BinaryPackage { + #[tracing::instrument(level = "debug", skip_all)] + pub async fn from_dir( + dir: &Path, + rt: &(dyn Runtime + Send + Sync), + ) -> Result { + let source = rt.source(); + + // since each package must be in its own directory, hash of the `dir` should provide a good enough + // unique identifier for the package + let hash = sha2::Sha256::digest(dir.as_os_str().as_bytes()).into(); + let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash)); + + let manifest_path = dir.join("wasmer.toml"); + let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?; + let container = Container::from(webc); + let manifest = container.manifest(); + + let root = PackageInfo::from_manifest(id, manifest, container.version())?; + let root_id = root.id.clone(); + + let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?; + let pkg = rt + .package_loader() + .load_package_tree(&container, &resolution) + .await + .map_err(|e| anyhow::anyhow!(e))?; + + Ok(pkg) + } + /// Load a [`webc::Container`] and all its dependencies into a /// [`BinaryPackage`]. #[tracing::instrument(level = "debug", skip_all)] From 41af2d8909d5b4d40d6c81bc4c5bf5453fb57bf3 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 14:39:14 +0330 Subject: [PATCH 2/5] fix platform dependent API usage --- lib/wasix/src/bin_factory/binary_package.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index dbf2706ca7c..e7d63c3e3d8 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -1,4 +1,4 @@ -use std::{os::unix::ffi::OsStrExt, path::Path, sync::Arc}; +use std::{path::Path, sync::Arc}; use anyhow::Context; use derivative::*; @@ -89,7 +89,7 @@ impl BinaryPackage { // since each package must be in its own directory, hash of the `dir` should provide a good enough // unique identifier for the package - let hash = sha2::Sha256::digest(dir.as_os_str().as_bytes()).into(); + let hash = sha2::Sha256::digest(dir.display().to_string().as_bytes()).into(); let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash)); let manifest_path = dir.join("wasmer.toml"); From ec7f2321c55536ad3eb30a4dc19d396a47179b39 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 14:48:06 +0330 Subject: [PATCH 3/5] use new webc version and fix publish code path --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/cli/src/commands/package/build.rs | 8 +++++--- lib/wasix/src/bin_factory/binary_package.rs | 4 +++- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d8396b8463..f1e70ea0006 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7145,9 +7145,9 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.0-alpha5" +version = "6.0.0-alpha6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10e8cca5fe3a8f85803e8cf67bb4f166c9fb326d4581c26a1a48e22f7922762" +checksum = "f90e7ef808f844f15d1680a7cea4a0c5082ccaae0dc09621855a655f71989eb1" dependencies = [ "anyhow", "base64 0.21.7", diff --git a/Cargo.toml b/Cargo.toml index f302186d9ee..487a8a9a5fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ wasmer-config = { path = "./lib/config" } wasmer-wasix = { path = "./lib/wasix" } # Wasmer-owned crates -webc = { version = "6.0.0-alpha5", default-features = false, features = ["package"] } +webc = { version = "6.0.0-alpha6", default-features = false, features = ["package"] } edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" diff --git a/lib/cli/src/commands/package/build.rs b/lib/cli/src/commands/package/build.rs index 67093906ce4..80db3505a4c 100644 --- a/lib/cli/src/commands/package/build.rs +++ b/lib/cli/src/commands/package/build.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use anyhow::Context; use dialoguer::console::{style, Emoji}; use indicatif::ProgressBar; +use sha2::Digest; use wasmer_config::package::PackageHash; use crate::utils::load_package_manifest; @@ -53,7 +54,10 @@ impl PackageBuild { ) }; let pkg = webc::wasmer_package::Package::from_manifest(manifest_path)?; - let pkg_hash = PackageHash::from_sha256_bytes(pkg.webc_hash()); + let data = pkg.serialize()?; + let hash = sha2::Sha256::digest(&data).into(); + + let pkg_hash = PackageHash::from_sha256_bytes(hash); let name = if let Some(manifest_pkg) = manifest.package { if let Some(name) = manifest_pkg.name { if let Some(version) = manifest_pkg.version { @@ -116,8 +120,6 @@ impl PackageBuild { ); } - let data = pkg.serialize()?; - pb.println(format!( "{} {}Writing package...", style("[3/3]").bold().dim(), diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index e7d63c3e3d8..6d729ead819 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -121,7 +121,9 @@ impl BinaryPackage { let manifest = container.manifest(); let id = PackageInfo::package_id_from_manifest(manifest)?.unwrap_or_else(|| { - PackageId::Hash(PackageHash::from_sha256_bytes(container.webc_hash())) + PackageId::Hash(PackageHash::from_sha256_bytes( + container.webc_hash().unwrap(), + )) }); let root = PackageInfo::from_manifest(id, manifest, container.version())?; From 41f380d89a10464ad2efe5e232627f6c90095f26 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 16:48:31 +0330 Subject: [PATCH 4/5] fix tests --- lib/wasix/src/bin_factory/binary_package.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 6d729ead819..dee299aefee 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -237,9 +237,6 @@ mod tests { std::fs::create_dir_all(&out).unwrap(); let file_txt = "Hello, World!"; std::fs::write(out.join("file.txt"), file_txt).unwrap(); - let webc: Container = webc::wasmer_package::Package::from_manifest(manifest) - .unwrap() - .into(); let tasks = task_manager(); let mut runtime = PluggableRuntime::new(tasks); runtime.set_package_loader( @@ -247,7 +244,9 @@ mod tests { .with_shared_http_client(runtime.http_client().unwrap().clone()), ); - let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap(); + let pkg = BinaryPackage::from_dir(&temp.path(), &runtime) + .await + .unwrap(); // We should have mapped "./out/file.txt" on the host to // "/public/file.txt" on the guest. @@ -290,7 +289,7 @@ mod tests { let atom_path = temp.path().join("foo.wasm"); std::fs::write(&atom_path, b"").unwrap(); - let webc: Container = webc::wasmer_package::Package::from_manifest(manifest) + let webc: Container = webc::wasmer_package::Package::from_manifest(&manifest) .unwrap() .into(); @@ -301,7 +300,9 @@ mod tests { .with_shared_http_client(runtime.http_client().unwrap().clone()), ); - let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap(); + let pkg = BinaryPackage::from_dir(&temp.path(), &runtime) + .await + .unwrap(); assert_eq!(pkg.commands.len(), 1); let command = pkg.get_command("cmd").unwrap(); From f5c5e427a813f96f298860dbc04b1c2a1c14b4da Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Mon, 6 May 2024 22:03:52 +0330 Subject: [PATCH 5/5] return error upon hash absence --- lib/wasix/src/bin_factory/binary_package.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index dee299aefee..d73755f1da5 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -120,11 +120,13 @@ impl BinaryPackage { let source = rt.source(); let manifest = container.manifest(); - let id = PackageInfo::package_id_from_manifest(manifest)?.unwrap_or_else(|| { - PackageId::Hash(PackageHash::from_sha256_bytes( - container.webc_hash().unwrap(), - )) - }); + let id = PackageInfo::package_id_from_manifest(manifest)? + .or_else(|| { + container + .webc_hash() + .map(|hash| PackageId::Hash(PackageHash::from_sha256_bytes(hash))) + }) + .ok_or_else(|| anyhow::Error::msg("webc file did not provide its hash"))?; let root = PackageInfo::from_manifest(id, manifest, container.version())?; let root_id = root.id.clone();