diff --git a/lib/wasi/src/bin_factory/binary_package.rs b/lib/wasi/src/bin_factory/binary_package.rs index 1ee1b0798a2..3bb3e929b92 100644 --- a/lib/wasi/src/bin_factory/binary_package.rs +++ b/lib/wasi/src/bin_factory/binary_package.rs @@ -1,4 +1,4 @@ -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use derivative::*; use once_cell::sync::OnceCell; @@ -6,7 +6,13 @@ use semver::Version; use virtual_fs::FileSystem; use webc::{compat::SharedBytes, Container}; -use crate::{runtime::module_cache::ModuleHash, WasiRuntime}; +use crate::{ + runtime::{ + module_cache::ModuleHash, + resolver::{PackageId, PackageInfo, PackageSpecifier, SourceId, SourceKind}, + }, + WasiRuntime, +}; #[derive(Derivative, Clone)] #[derivative(Debug)] @@ -53,7 +59,7 @@ pub struct BinaryPackage { pub entry: Option, pub hash: OnceCell, pub webc_fs: Arc, - pub commands: Arc>>, + pub commands: Vec, pub uses: Vec, pub version: Version, pub module_memory_footprint: u64, @@ -64,16 +70,47 @@ impl BinaryPackage { /// Load a [`webc::Container`] and all its dependencies into a /// [`BinaryPackage`]. pub async fn from_webc( - _container: &Container, - _rt: &dyn WasiRuntime, + container: &Container, + rt: &dyn WasiRuntime, ) -> Result { - // let summary = crate::runtime::resolver::extract_summary_from_manifest( - // container.manifest(), - // source, - // url, - // webc_sha256, - // )?; - todo!(); + let registry = rt.registry(); + let root = PackageInfo::from_manifest(container.manifest())?; + let root_id = PackageId { + package_name: root.name.clone(), + version: root.version.clone(), + source: SourceId::new( + SourceKind::LocalRegistry, + "http://localhost/".parse().unwrap(), + ), + }; + + let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*registry).await?; + let pkg = rt + .load_package_tree(container, &resolution) + .await + .map_err(|e| anyhow::anyhow!(e))?; + + Ok(pkg) + } + + /// Load a [`BinaryPackage`] and all its dependencies from a registry. + pub async fn from_specifier( + specifier: &PackageSpecifier, + runtime: &dyn WasiRuntime, + ) -> Result { + let registry = runtime.registry(); + let root_summary = registry.latest(specifier).await?; + let root = runtime.package_loader().load(&root_summary).await?; + let id = root_summary.package_id(); + + let resolution = + crate::runtime::resolver::resolve(&id, &root_summary.pkg, ®istry).await?; + let pkg = runtime + .load_package_tree(&root, &resolution) + .await + .map_err(|e| anyhow::anyhow!(e))?; + + Ok(pkg) } pub fn hash(&self) -> ModuleHash { diff --git a/lib/wasi/src/os/command/builtins/cmd_wasmer.rs b/lib/wasi/src/os/command/builtins/cmd_wasmer.rs index 009b007938a..2c906f3768d 100644 --- a/lib/wasi/src/os/command/builtins/cmd_wasmer.rs +++ b/lib/wasi/src/os/command/builtins/cmd_wasmer.rs @@ -71,7 +71,7 @@ impl CmdWasmer { state.args = args; env.state = Arc::new(state); - if let Ok(binary) = self.get_package(what.clone()).await { + if let Ok(binary) = self.get_package(&what).await { // Now run the module spawn_exec(binary, name, store, env, &self.runtime).await } else { @@ -91,18 +91,9 @@ impl CmdWasmer { } } - pub async fn get_package(&self, name: String) -> Result { - let registry = self.runtime.registry(); + pub async fn get_package(&self, name: &str) -> Result { let specifier = name.parse()?; - let root_package = registry.latest(&specifier).await?; - let resolution = crate::runtime::resolver::resolve(&root_package, ®istry).await?; - let pkg = self - .runtime - .load_package_tree(&resolution) - .await - .map_err(|e| anyhow::anyhow!(e))?; - - Ok(pkg) + BinaryPackage::from_specifier(&specifier, &*self.runtime).await } } diff --git a/lib/wasi/src/os/console/mod.rs b/lib/wasi/src/os/console/mod.rs index 028fbd4d91e..d2ee416d035 100644 --- a/lib/wasi/src/os/console/mod.rs +++ b/lib/wasi/src/os/console/mod.rs @@ -208,7 +208,8 @@ impl Console { } }; - let resolved_package = tasks.block_on(load_package(&webc_ident, env.runtime())); + let resolved_package = + tasks.block_on(BinaryPackage::from_specifier(&webc_ident, env.runtime())); let binary = match resolved_package { Ok(pkg) => pkg, @@ -278,15 +279,3 @@ impl Console { .ok(); } } - -async fn load_package( - specifier: &PackageSpecifier, - runtime: &dyn WasiRuntime, -) -> Result> { - let registry = runtime.registry(); - let root_package = registry.latest(specifier).await?; - let resolution = crate::runtime::resolver::resolve(&root_package, ®istry).await?; - let pkg = runtime.load_package_tree(&resolution).await?; - - Ok(pkg) -} diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index fd233d52b46..37bb650b568 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -135,12 +135,13 @@ impl crate::runners::Runner for WasiRunner { let wasi = metadata .annotation("wasi")? .unwrap_or_else(|| Wasi::new(command_name)); - let atom = pkg - .entry - .as_deref() - .context("The package doesn't contain an entrpoint")?; + let cmd = pkg + .commands + .iter() + .find(|cmd| cmd.name() == command_name) + .with_context(|| format!("The package doesn't contain a \"{command_name}\" command"))?; - let module = crate::runners::compile_module(atom, &*runtime)?; + let module = crate::runners::compile_module(cmd.atom(), &*runtime)?; let mut store = runtime.new_store(); self.prepare_webc_env(command_name, &wasi, Arc::clone(&pkg.webc_fs), runtime)? diff --git a/lib/wasi/src/runtime/mod.rs b/lib/wasi/src/runtime/mod.rs index 2b4d241fa2d..1d734f8be18 100644 --- a/lib/wasi/src/runtime/mod.rs +++ b/lib/wasi/src/runtime/mod.rs @@ -13,6 +13,7 @@ use std::{ use derivative::Derivative; use futures::future::BoxFuture; use virtual_net::{DynVirtualNetworking, VirtualNetworking}; +use webc::Container; use crate::{ bin_factory::BinaryPackage, @@ -28,29 +29,6 @@ use crate::{ /// Represents an implementation of the WASI runtime - by default everything is /// unimplemented. -/// -/// # Loading Packages -/// -/// Loading a package, complete with dependencies, can feel a bit involved -/// because it requires several non-trivial components. -/// -/// ```rust -/// use wasmer_wasix::{ -/// runtime::{ -/// WasiRuntime, -/// resolver::{PackageSpecifier, resolve}, -/// }, -/// bin_factory::BinaryPackage, -/// }; -/// -/// async fn with_runtime(runtime: &dyn WasiRuntime) -> Result<(), Box> { -/// let registry = runtime.registry(); -/// let specifier: PackageSpecifier = "python/python@3.10".parse()?; -/// let root_package = registry.latest(&specifier).await?; -/// let resolution = resolve(&root_package, ®istry).await?; -/// let pkg: BinaryPackage = runtime.load_package_tree(&resolution).await?; -/// Ok(()) -/// } #[allow(unused_variables)] pub trait WasiRuntime where @@ -108,12 +86,13 @@ where /// should be good enough for most applications. fn load_package_tree<'a>( &'a self, + root: &'a Container, resolution: &'a Resolution, ) -> BoxFuture<'a, Result>> { let package_loader = self.package_loader(); Box::pin(async move { - let pkg = package_loader::load_package_tree(&package_loader, resolution).await?; + let pkg = package_loader::load_package_tree(root, &package_loader, resolution).await?; Ok(pkg) }) } diff --git a/lib/wasi/src/runtime/package_loader/load_package_tree.rs b/lib/wasi/src/runtime/package_loader/load_package_tree.rs index 1eb03ad1951..541fa712b80 100644 --- a/lib/wasi/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasi/src/runtime/package_loader/load_package_tree.rs @@ -1,7 +1,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, path::Path, - sync::{Arc, RwLock}, + sync::Arc, }; use anyhow::{Context, Error}; @@ -14,17 +14,20 @@ use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, runtime::{ package_loader::PackageLoader, - resolver::{ItemLocation, PackageId, Resolution, ResolvedPackage, Summary}, + resolver::{ + DependencyGraph, ItemLocation, PackageId, Resolution, ResolvedPackage, Summary, + }, }, }; /// Given a fully resolved package, load it into memory for execution. pub async fn load_package_tree( + root: &Container, loader: &impl PackageLoader, resolution: &Resolution, ) -> Result { - let containers = - used_packages(loader, &resolution.package, &resolution.graph.summaries).await?; + let mut containers = used_packages(loader, &resolution.package, &resolution.graph).await?; + containers.insert(resolution.package.root_package.clone(), root.clone()); let fs = filesystem(&containers, &resolution.package)?; let root = &resolution.package.root_package; @@ -53,7 +56,7 @@ pub async fn load_package_tree( .map(|cmd| cmd.atom.clone()) }), webc_fs: Arc::new(fs), - commands: Arc::new(RwLock::new(commands)), + commands, uses: Vec::new(), module_memory_footprint, file_system_memory_footprint, @@ -191,10 +194,9 @@ fn legacy_atom_hack(webc: &Container, command_name: &str) -> Option, + graph: &DependencyGraph, ) -> Result, Error> { let mut packages = HashSet::new(); - packages.insert(pkg.root_package.clone()); for loc in pkg.commands.values() { packages.insert(loc.package.clone()); @@ -204,9 +206,18 @@ async fn used_packages( packages.insert(mapping.package.clone()); } + // We don't need to download the root package + packages.remove(&pkg.root_package); + let packages: FuturesUnordered<_> = packages .into_iter() - .map(|id| async { loader.load(&summaries[&id]).await.map(|webc| (id, webc)) }) + .map(|id| async { + let summary = Summary { + pkg: graph.package_info[&id].clone(), + dist: graph.distribution[&id].clone(), + }; + loader.load(&summary).await.map(|webc| (id, webc)) + }) .collect(); let packages: HashMap = packages.try_collect().await?; diff --git a/lib/wasi/src/runtime/resolver/outputs.rs b/lib/wasi/src/runtime/resolver/outputs.rs index 6d091f1753c..71099989427 100644 --- a/lib/wasi/src/runtime/resolver/outputs.rs +++ b/lib/wasi/src/runtime/resolver/outputs.rs @@ -6,7 +6,7 @@ use std::{ use semver::Version; -use crate::runtime::resolver::{SourceId, Summary}; +use crate::runtime::resolver::{DistributionInfo, PackageInfo, SourceId}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Resolution { @@ -58,7 +58,8 @@ impl Display for PackageId { pub struct DependencyGraph { pub root: PackageId, pub dependencies: HashMap>, - pub summaries: HashMap, + pub package_info: HashMap, + pub distribution: HashMap, } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/lib/wasi/src/runtime/resolver/resolve.rs b/lib/wasi/src/runtime/resolver/resolve.rs index 144b0984c82..edcd94119c9 100644 --- a/lib/wasi/src/runtime/resolver/resolve.rs +++ b/lib/wasi/src/runtime/resolver/resolve.rs @@ -1,15 +1,20 @@ use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use crate::runtime::resolver::{ - DependencyGraph, ItemLocation, PackageId, Registry, Resolution, ResolvedPackage, Summary, + DependencyGraph, ItemLocation, PackageId, PackageInfo, Registry, Resolution, ResolvedPackage, + Summary, }; use super::FileSystemMapping; /// Given the [`Summary`] for a root package, resolve its dependency graph and /// figure out how it could be executed. -pub async fn resolve(root: &Summary, registry: &impl Registry) -> Result { - let graph = resolve_dependency_graph(root, registry).await?; +pub async fn resolve( + root_id: &PackageId, + root: &PackageInfo, + registry: &dyn Registry, +) -> Result { + let graph = resolve_dependency_graph(root_id, root, registry).await?; let package = resolve_package(&graph)?; Ok(Resolution { graph, package }) @@ -48,22 +53,24 @@ fn print_cycle(packages: &[PackageId]) -> String { } async fn resolve_dependency_graph( - root: &Summary, - registry: &impl Registry, + root_id: &PackageId, + root: &PackageInfo, + registry: &dyn Registry, ) -> Result { let mut dependencies = HashMap::new(); - let mut summaries = HashMap::new(); + let mut package_info = HashMap::new(); + let mut distribution = HashMap::new(); - summaries.insert(root.package_id(), root.clone()); + package_info.insert(root_id.clone(), root.clone()); let mut to_visit = VecDeque::new(); - to_visit.push_back(root.clone()); + to_visit.push_back((root_id.clone(), root.clone())); - while let Some(summary) = to_visit.pop_front() { + while let Some((id, info)) = to_visit.pop_front() { let mut deps = HashMap::new(); - for dep in &summary.pkg.dependencies { + for dep in &info.dependencies { let dep_summary = registry .latest(&dep.pkg) .await @@ -71,25 +78,28 @@ async fn resolve_dependency_graph( deps.insert(dep.alias().to_string(), dep_summary.package_id()); let dep_id = dep_summary.package_id(); - if summaries.contains_key(&dep_id) { + if dependencies.contains_key(&dep_id) { // We don't need to visit this dependency again continue; } - summaries.insert(dep_id, dep_summary.clone()); - to_visit.push_back(dep_summary); + let Summary { pkg, dist } = dep_summary; + + to_visit.push_back((dep_id.clone(), pkg.clone())); + package_info.insert(dep_id.clone(), pkg); + distribution.insert(dep_id, dist); } - dependencies.insert(summary.package_id(), deps); + dependencies.insert(id, deps); } - let root = root.package_id(); - check_for_cycles(&dependencies, &root)?; + check_for_cycles(&dependencies, root_id)?; Ok(DependencyGraph { - root, + root: root_id.clone(), dependencies, - summaries, + package_info, + distribution, }) } @@ -149,20 +159,20 @@ fn resolve_package(dependency_graph: &DependencyGraph) -> Result &mut Self { + self.summary.pkg.entrypoint = Some(name.to_string()); + self + } } impl<'builder> Drop for AddPackageVersion<'builder> { @@ -378,7 +393,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph.insert("root", "1.0.0"); @@ -401,7 +418,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph.insert("root", "1.0.0"); @@ -432,7 +451,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph @@ -464,7 +485,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("first", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph @@ -498,7 +521,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph @@ -538,7 +563,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph @@ -580,7 +607,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph @@ -622,7 +651,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let resolution = resolve(root, ®istry).await.unwrap(); + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); let mut dependency_graph = builder.start_dependency_graph(); dependency_graph @@ -658,7 +689,9 @@ mod tests { let registry = builder.finish(); let root = builder.get("root", "1.0.0"); - let err = resolve(root, ®istry).await.unwrap_err(); + let err = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap_err(); let cycle = err.as_cycle().unwrap().to_vec(); assert_eq!( @@ -671,6 +704,39 @@ mod tests { ); } + #[tokio::test] + async fn entrypoint_is_inherited() { + let mut builder = RegistryBuilder::new(); + builder + .register("root", "1.0.0") + .with_dependency("dep", "=1.0.0"); + builder + .register("dep", "1.0.0") + .with_command("entry") + .with_entrypoint("entry"); + let registry = builder.finish(); + let root = builder.get("root", "1.0.0"); + + let resolution = resolve(&root.package_id(), &root.pkg, ®istry) + .await + .unwrap(); + + assert_eq!( + resolution.package, + ResolvedPackage { + root_package: root.package_id(), + commands: map! { + "entry" => ItemLocation { + name: "entry".to_string(), + package: builder.get("dep", "1.0.0").package_id(), + }, + }, + entrypoint: Some("entry".to_string()), + filesystem: Vec::new(), + } + ); + } + #[test] fn cyclic_error_message() { let source = SourceId::new( diff --git a/lib/wasi/src/state/env.rs b/lib/wasi/src/state/env.rs index 8887785a9b3..8e6d39cf65b 100644 --- a/lib/wasi/src/state/env.rs +++ b/lib/wasi/src/state/env.rs @@ -830,7 +830,7 @@ impl WasiEnv { while let Some(use_package) = use_packages.pop_back() { match cmd_wasmer .ok_or_else(|| anyhow::anyhow!("Unable to get /bin/wasmer")) - .and_then(|cmd| tasks.block_on(cmd.get_package(use_package.clone()))) + .and_then(|cmd| tasks.block_on(cmd.get_package(&use_package))) { Ok(package) => { // If its already been added make sure the version is correct @@ -857,7 +857,7 @@ impl WasiEnv { // Add all the commands as binaries in the bin folder - let commands = package.commands.read().unwrap(); + let commands = &package.commands; if !commands.is_empty() { let _ = root_fs.create_dir(Path::new("/bin")); for command in commands.iter() { diff --git a/lib/wasi/src/wapm/mod.rs b/lib/wasi/src/wapm/mod.rs index f38ec2fcea5..1137244c93b 100644 --- a/lib/wasi/src/wapm/mod.rs +++ b/lib/wasi/src/wapm/mod.rs @@ -1,10 +1,6 @@ use anyhow::Context; use once_cell::sync::OnceCell; -use std::{ - collections::HashMap, - path::Path, - sync::{Arc, RwLock}, -}; +use std::{collections::HashMap, path::Path, sync::Arc}; use virtual_fs::{FileSystem, WebcVolumeFileSystem}; use wasmer_wasix_types::wasi::Snapshot0Clockid; @@ -69,7 +65,7 @@ pub(crate) fn parse_webc(webc: &Container) -> Result = commands + let commands: BTreeMap<&str, &[u8]> = pkg + .commands .iter() .map(|cmd| (cmd.name(), cmd.atom())) .collect(); @@ -258,8 +254,8 @@ mod tests { assert_eq!(pkg.module_memory_footprint, 0); assert_eq!(pkg.file_system_memory_footprint, 44); assert_eq!(pkg.entry, None); - let commands = pkg.commands.read().unwrap(); - let commands: BTreeMap<&str, &[u8]> = commands + let commands: BTreeMap<&str, &[u8]> = pkg + .commands .iter() .map(|cmd| (cmd.name(), cmd.atom())) .collect(); @@ -388,8 +384,8 @@ mod tests { assert_eq!(pkg.uses, &["sharrattj/coreutils@1.0.11"]); assert_eq!(pkg.module_memory_footprint, 0); assert_eq!(pkg.file_system_memory_footprint, 0); - let commands = pkg.commands.read().unwrap(); - let commands: BTreeMap<&str, &[u8]> = commands + let commands: BTreeMap<&str, &[u8]> = pkg + .commands .iter() .map(|cmd| (cmd.name(), cmd.atom())) .collect(); @@ -405,8 +401,7 @@ mod tests { assert_eq!(pkg.package_name, "wasmer/hello"); assert_eq!(pkg.version.to_string(), "0.1.0"); - let commands = pkg.commands.read().unwrap(); - assert!(commands.is_empty()); + assert!(pkg.commands.is_empty()); assert!(pkg.entry.is_none()); assert_eq!(pkg.uses, ["sharrattj/static-web-server@1"]); } diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index 5f341c16dd6..99c6a9af0e7 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -53,12 +53,11 @@ mod wasi { ) }); let err = handle.join().unwrap().unwrap_err(); - dbg!(&err); let runtime_error = err .chain() .find_map(|e| e.downcast_ref::()) - .unwrap(); + .expect("Couldn't find a WasiError"); let exit_code = match runtime_error { WasiError::Exit(code) => *code, other => unreachable!("Something else went wrong: {:?}", other), @@ -104,7 +103,7 @@ mod wcgi { use futures::{channel::mpsc::Sender, future::AbortHandle, SinkExt, StreamExt}; use rand::Rng; use tokio::runtime::Handle; - use wasmer_wasix::{runners::wcgi::WcgiRunner, bin_factory::BinaryPackage}; + use wasmer_wasix::{bin_factory::BinaryPackage, runners::wcgi::WcgiRunner}; use super::*;