From 75420682ece6216205cedb7da408d3b79ab63c68 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 25 May 2023 22:00:21 +0800 Subject: [PATCH 01/16] Add filesystem mappings to the resolver --- .../runtime/package_loader/builtin_loader.rs | 1 + .../src/runtime/resolver/in_memory_source.rs | 1 + lib/wasix/src/runtime/resolver/inputs.rs | 26 +++ lib/wasix/src/runtime/resolver/mod.rs | 3 +- lib/wasix/src/runtime/resolver/outputs.rs | 5 +- lib/wasix/src/runtime/resolver/resolve.rs | 195 +++++++++++++++++- lib/wasix/src/runtime/resolver/wapm_source.rs | 1 + 7 files changed, 220 insertions(+), 12 deletions(-) diff --git a/lib/wasix/src/runtime/package_loader/builtin_loader.rs b/lib/wasix/src/runtime/package_loader/builtin_loader.rs index 749bf5a8b9b..a9fce3c6295 100644 --- a/lib/wasix/src/runtime/package_loader/builtin_loader.rs +++ b/lib/wasix/src/runtime/package_loader/builtin_loader.rs @@ -372,6 +372,7 @@ mod tests { dependencies: Vec::new(), commands: Vec::new(), entrypoint: Some("asdf".to_string()), + filesystem: Vec::new(), }, dist: DistributionInfo { webc: "https://wapm.io/python/python".parse().unwrap(), diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/wasix/src/runtime/resolver/in_memory_source.rs index c8a7e34b6a0..9aaa56ef3b9 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/wasix/src/runtime/resolver/in_memory_source.rs @@ -168,6 +168,7 @@ mod tests { name: "bash".to_string(), }], entrypoint: Some("bash".to_string()), + filesystem: Vec::new(), }, dist: DistributionInfo { webc: crate::runtime::resolver::utils::url_from_file_path( diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/wasix/src/runtime/resolver/inputs.rs index e289277830b..51fddcb781f 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/wasix/src/runtime/resolver/inputs.rs @@ -182,6 +182,7 @@ pub struct PackageInfo { pub entrypoint: Option, /// Any dependencies this package may have. pub dependencies: Vec, + pub filesystem: Vec, } impl PackageInfo { @@ -209,12 +210,15 @@ impl PackageInfo { }) .collect(); + let filesystem = filesystem_mapping_from_manifest(manifest); + Ok(PackageInfo { name, version: version.parse()?, dependencies, commands, entrypoint: manifest.entrypoint.clone(), + filesystem, }) } @@ -226,6 +230,28 @@ impl PackageInfo { } } +fn filesystem_mapping_from_manifest(_manifest: &Manifest) -> Vec { + // FIXME(Michael-F-Bryan): wapm2pirita never added filesystem mappings to the manifest + // That means we need to + // - Figure out whether filesystem mappings belong to the whole package or + // if each command has their own set of mappings + // - Update wapm-targz-to-pirita to copy the [fs] table across + // - Re-generate all packages on WAPM + // - Update this function to copy metadata into our internal datastructures + Vec::new() +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileSystemMapping { + /// The volume to be mounted. + pub volume_name: String, + /// Where the volume should be mounted within the resulting filesystem. + pub mount_path: String, + /// The name of the package this volume comes from (current package if + /// `None`). + pub dependency_name: Option, +} + fn url_or_manifest_to_specifier(value: &UrlOrManifest) -> Result { match value { UrlOrManifest::Url(url) => Ok(PackageSpecifier::Url(url.clone())), diff --git a/lib/wasix/src/runtime/resolver/mod.rs b/lib/wasix/src/runtime/resolver/mod.rs index 6ab6a533cf0..089741e36f9 100644 --- a/lib/wasix/src/runtime/resolver/mod.rs +++ b/lib/wasix/src/runtime/resolver/mod.rs @@ -18,7 +18,8 @@ pub use self::{ }, multi_source_registry::MultiSource, outputs::{ - DependencyGraph, FileSystemMapping, ItemLocation, PackageId, Resolution, ResolvedPackage, + DependencyGraph, ItemLocation, PackageId, Resolution, ResolvedFileSystemMapping, + ResolvedPackage, }, resolve::resolve, source::Source, diff --git a/lib/wasix/src/runtime/resolver/outputs.rs b/lib/wasix/src/runtime/resolver/outputs.rs index e2289531e7a..a161e9a35b1 100644 --- a/lib/wasix/src/runtime/resolver/outputs.rs +++ b/lib/wasix/src/runtime/resolver/outputs.rs @@ -62,7 +62,8 @@ impl DependencyGraph { } #[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileSystemMapping { +pub struct ResolvedFileSystemMapping { + // TODO: Change this to a new type that isn't coupled to the OS pub mount_path: PathBuf, pub volume_name: String, pub package: PackageId, @@ -75,5 +76,5 @@ pub struct ResolvedPackage { pub commands: BTreeMap, pub entrypoint: Option, /// A mapping from paths to the volumes that should be mounted there. - pub filesystem: Vec, + pub filesystem: Vec, } diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/wasix/src/runtime/resolver/resolve.rs index 0b7650efab2..fccbbec6104 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/wasix/src/runtime/resolver/resolve.rs @@ -1,4 +1,7 @@ -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::{ + collections::{BTreeMap, HashMap, HashSet, VecDeque}, + path::PathBuf, +}; use semver::Version; @@ -7,7 +10,7 @@ use crate::runtime::resolver::{ ResolvedPackage, Source, }; -use super::FileSystemMapping; +use super::ResolvedFileSystemMapping; /// Given the [`PackageInfo`] for a root package, resolve its dependency graph /// and figure out how it could be executed. @@ -293,18 +296,33 @@ fn resolve_package(dependency_graph: &DependencyGraph) -> Result Result, ResolveError> { - // TODO: Add filesystem mappings to summary and figure out the final mapping - // for this dependency graph. - // See for more. - Ok(Vec::new()) + graph: &DependencyGraph, +) -> Result, ResolveError> { + let mut mappings = Vec::new(); + + for (id, pkg) in &graph.package_info { + for mapping in &pkg.filesystem { + let dep = match &mapping.dependency_name { + Some(name) => graph.dependencies[id].get(name).unwrap(), + None => id, + }; + mappings.push(ResolvedFileSystemMapping { + mount_path: PathBuf::from(&mapping.mount_path), + volume_name: mapping.volume_name.clone(), + package: dep.clone(), + }) + } + } + + Ok(mappings) } #[cfg(test)] mod tests { + use std::path::PathBuf; + use crate::runtime::resolver::{ - inputs::{DistributionInfo, PackageInfo}, + inputs::{DistributionInfo, FileSystemMapping, PackageInfo}, Dependency, InMemorySource, MultiSource, PackageSpecifier, }; @@ -324,6 +342,7 @@ mod tests { dependencies: Vec::new(), commands: Vec::new(), entrypoint: None, + filesystem: Vec::new(), }; let dist = DistributionInfo { webc: format!("http://localhost/{name}@{version}") @@ -402,6 +421,29 @@ mod tests { self.summary.pkg.entrypoint = Some(name.to_string()); self } + + fn with_fs_mapping(&mut self, volume_name: &str, mount_path: &str) -> &mut Self { + self.summary.pkg.filesystem.push(FileSystemMapping { + volume_name: volume_name.to_string(), + mount_path: mount_path.to_string(), + dependency_name: None, + }); + self + } + + fn with_fs_mapping_from_dependency( + &mut self, + volume_name: &str, + mount_path: &str, + dependency: &str, + ) -> &mut Self { + self.summary.pkg.filesystem.push(FileSystemMapping { + volume_name: volume_name.to_string(), + mount_path: mount_path.to_string(), + dependency_name: Some(dependency.to_string()), + }); + self + } } impl<'builder> Drop for AddPackageVersion<'builder> { @@ -897,4 +939,139 @@ mod tests { assert_eq!(message, "root@1.0.0 → dep@1.0.0 → root@1.0.0"); } + + #[test] + fn filesystem_with_one_package_and_no_fs_tables() { + let mut builder = RegistryBuilder::new(); + builder.register("root", "1.0.0"); + let mut dep_builder = builder.start_dependency_graph(); + dep_builder.insert("root", "1.0.0"); + let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); + let root_id = pkg.id(); + let graph = DependencyGraph { + root: root_id.clone(), + dependencies: dep_builder.finish(), + package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), + distribution: vec![(root_id, dist)].into_iter().collect(), + }; + + let mappings = resolve_filesystem_mapping(&graph).unwrap(); + assert_eq!(mappings, Vec::new()); + } + + #[test] + fn filesystem_with_one_package_and_one_fs_tables() { + let mut builder = RegistryBuilder::new(); + builder + .register("root", "1.0.0") + .with_fs_mapping("lib", "/lib"); + let mut dep_builder = builder.start_dependency_graph(); + dep_builder.insert("root", "1.0.0"); + let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); + let root_id = pkg.id(); + let graph = DependencyGraph { + root: root_id.clone(), + dependencies: dep_builder.finish(), + package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), + distribution: vec![(root_id, dist)].into_iter().collect(), + }; + + let mappings = resolve_filesystem_mapping(&graph).unwrap(); + assert_eq!( + mappings, + vec![ResolvedFileSystemMapping { + mount_path: PathBuf::from("/lib"), + volume_name: "lib".to_string(), + package: builder.get("root", "1.0.0").package_id(), + }] + ); + } + + #[test] + fn merge_fs_mappings_from_multiple_packages() { + let mut builder = RegistryBuilder::new(); + builder + .register("root", "1.0.0") + .with_dependency("first", "=1.0.0") + .with_dependency("second", "=1.0.0") + .with_fs_mapping("lib", "/root"); + builder + .register("first", "1.0.0") + .with_fs_mapping("main", "/usr/local/lib/first"); + builder + .register("second", "1.0.0") + .with_fs_mapping("main", "/usr/local/lib/second"); + let mut dep_builder = builder.start_dependency_graph(); + dep_builder + .insert("root", "1.0.0") + .with_dependency("first", "1.0.0") + .with_dependency("second", "1.0.0"); + dep_builder.insert("first", "1.0.0"); + dep_builder.insert("second", "1.0.0"); + let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); + let root_id = pkg.id(); + let graph = DependencyGraph { + root: root_id.clone(), + dependencies: dep_builder.finish(), + package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), + distribution: vec![(root_id, dist)].into_iter().collect(), + }; + + let mappings = resolve_filesystem_mapping(&graph).unwrap(); + + assert_eq!( + mappings, + vec![ + ResolvedFileSystemMapping { + mount_path: PathBuf::from("/usr/local/lib/second"), + volume_name: "main".to_string(), + package: builder.get("second", "1.0.0").package_id(), + }, + ResolvedFileSystemMapping { + mount_path: PathBuf::from("/usr/local/lib/first"), + volume_name: "main".to_string(), + package: builder.get("first", "1.0.0").package_id(), + }, + ResolvedFileSystemMapping { + mount_path: PathBuf::from("/root"), + volume_name: "lib".to_string(), + package: builder.get("root", "1.0.0").package_id(), + } + ] + ); + } + + #[test] + fn use_fs_mapping_from_dependency() { + let mut builder = RegistryBuilder::new(); + builder + .register("root", "1.0.0") + .with_dependency("dep", "=1.0.0") + .with_fs_mapping_from_dependency("dep-volume", "/root", "dep"); + builder.register("dep", "1.0.0"); + let mut dep_builder = builder.start_dependency_graph(); + dep_builder + .insert("root", "1.0.0") + .with_dependency("dep", "1.0.0"); + dep_builder.insert("dep", "1.0.0"); + let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); + let root_id = pkg.id(); + let graph = DependencyGraph { + root: root_id.clone(), + dependencies: dep_builder.finish(), + package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), + distribution: vec![(root_id, dist)].into_iter().collect(), + }; + + let mappings = resolve_filesystem_mapping(&graph).unwrap(); + + assert_eq!( + mappings, + vec![ResolvedFileSystemMapping { + mount_path: PathBuf::from("/root"), + volume_name: "dep-volume".to_string(), + package: builder.get("dep", "1.0.0").package_id(), + }] + ); + } } diff --git a/lib/wasix/src/runtime/resolver/wapm_source.rs b/lib/wasix/src/runtime/resolver/wapm_source.rs index 71029ff5d45..ee3d5cbc1f1 100644 --- a/lib/wasix/src/runtime/resolver/wapm_source.rs +++ b/lib/wasix/src/runtime/resolver/wapm_source.rs @@ -274,6 +274,7 @@ mod tests { }, ], entrypoint: Some("wasmer-pack".to_string()), + filesystem: Vec::new(), }, dist: DistributionInfo { webc: "https://registry-cdn.wapm.io/packages/wasmer/wasmer-pack-cli/wasmer-pack-cli-0.6.0-654a2ed8-875f-11ed-90e2-c6aeb50490de.webc".parse().unwrap(), From 584baaf9fdcbfcef3a7dbe21724ba0c8880aef22 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 2 Jun 2023 18:58:36 +0800 Subject: [PATCH 02/16] Switched the dependency graph over to petgraph --- Cargo.lock | 17 + lib/wasix/Cargo.toml | 16 +- .../package_loader/load_package_tree.rs | 10 +- lib/wasix/src/runtime/resolver/mod.rs | 4 +- lib/wasix/src/runtime/resolver/outputs.rs | 132 +++++- lib/wasix/src/runtime/resolver/resolve.rs | 432 ++++++++++-------- 6 files changed, 382 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c8ebc060152..0adde1037b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1471,6 +1471,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.0.26" @@ -2980,6 +2986,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "petgraph" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pin-project" version = "1.1.0" @@ -6087,6 +6103,7 @@ dependencies = [ "libc", "linked_hash_set", "once_cell", + "petgraph", "pin-project", "pretty_assertions", "rand", diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index fd287221e76..0cd094cec20 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -65,6 +65,7 @@ wcgi-host = { version = "0.1.2", optional = true } tower-http = { version = "0.4.0", features = ["trace", "util", "catch-panic", "cors"], optional = true } tower = { version = "0.4.13", features = ["make", "util"], optional = true } url = "2.3.1" +petgraph = "0.6.3" [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest] version = "0.11" @@ -116,7 +117,7 @@ webc_runner_rt_wcgi = ["hyper", "wcgi", "wcgi-host", "tower", "tower-http"] webc_runner_rt_emscripten = ["wasmer-emscripten"] sys = ["webc/mmap", "time"] -sys-default = ["sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest" ] +sys-default = ["sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest"] sys-poll = [] sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread"] @@ -127,21 +128,14 @@ js = ["virtual-fs/no-time", "getrandom/js", "chrono"] js-default = ["js"] test-js = ["js", "wasmer/wat"] -host-vnet = [ "virtual-net/host-net" ] +host-vnet = ["virtual-net/host-net"] host-threads = [] host-reqwest = ["reqwest"] host-fs = ["virtual-fs/host-fs"] logging = ["tracing/log"] -disable-all-logging = [ - "tracing/release_max_level_off", - "tracing/max_level_off" -] -enable-serde = [ - "typetag", - "virtual-fs/enable-serde", - "wasmer-wasix-types/enable-serde", -] +disable-all-logging = ["tracing/release_max_level_off", "tracing/max_level_off"] +enable-serde = ["typetag", "virtual-fs/enable-serde", "wasmer-wasix-types/enable-serde"] [package.metadata.docs.rs] features = ["wasmer/sys"] diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 36199b8dea2..104ab4162b4 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -215,13 +215,15 @@ async fn fetch_dependencies( let packages: FuturesUnordered<_> = packages .into_iter() - .map(|id| async { + .filter_map(|id| { + let crate::runtime::resolver::Node { pkg, dist, .. } = &graph[&id]; let summary = PackageSummary { - pkg: graph.package_info[&id].clone(), - dist: graph.distribution[&id].clone(), + pkg: pkg.clone(), + dist: dist.clone()?, }; - loader.load(&summary).await.map(|webc| (id, webc)) + Some((id, summary)) }) + .map(|(id, summary)| async move { loader.load(&summary).await.map(|webc| (id, webc)) }) .collect(); let packages: HashMap = packages.try_collect().await?; diff --git a/lib/wasix/src/runtime/resolver/mod.rs b/lib/wasix/src/runtime/resolver/mod.rs index 089741e36f9..6f5a16738e1 100644 --- a/lib/wasix/src/runtime/resolver/mod.rs +++ b/lib/wasix/src/runtime/resolver/mod.rs @@ -18,8 +18,8 @@ pub use self::{ }, multi_source_registry::MultiSource, outputs::{ - DependencyGraph, ItemLocation, PackageId, Resolution, ResolvedFileSystemMapping, - ResolvedPackage, + DependencyGraph, Edge, ItemLocation, Node, PackageId, Resolution, + ResolvedFileSystemMapping, ResolvedPackage, }, resolve::resolve, source::Source, diff --git a/lib/wasix/src/runtime/resolver/outputs.rs b/lib/wasix/src/runtime/resolver/outputs.rs index a161e9a35b1..ef9c9672693 100644 --- a/lib/wasix/src/runtime/resolver/outputs.rs +++ b/lib/wasix/src/runtime/resolver/outputs.rs @@ -1,15 +1,16 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::BTreeMap, fmt::{self, Display, Formatter}, + ops::Index, path::PathBuf, - unreachable, }; +use petgraph::{graph::NodeIndex, visit::EdgeRef}; use semver::Version; use crate::runtime::resolver::{DistributionInfo, PackageInfo}; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct Resolution { pub package: ResolvedPackage, pub graph: DependencyGraph, @@ -24,7 +25,7 @@ pub struct ItemLocation { } /// An identifier for a package within a dependency graph. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct PackageId { pub package_name: String, pub version: Version, @@ -40,25 +41,122 @@ impl Display for PackageId { } } -/// A dependency graph. -#[derive(Debug, Clone, PartialEq, Eq)] +/// An acyclic, directed dependency graph. +#[derive(Debug, Clone)] pub struct DependencyGraph { - pub root: PackageId, - pub dependencies: HashMap>, - pub package_info: HashMap, - pub distribution: HashMap, + root: NodeIndex, + graph: petgraph::graph::DiGraph, + packages: BTreeMap, } impl DependencyGraph { - pub fn root_info(&self) -> &PackageInfo { - match self.package_info.get(&self.root) { - Some(info) => info, - None => unreachable!( - "The dependency graph should always have package info for the root package, {}", - self.root - ), + pub(crate) fn new( + root: NodeIndex, + graph: petgraph::graph::DiGraph, + packages: BTreeMap, + ) -> Self { + DependencyGraph { + root, + graph, + packages, } } + + pub fn root_info(&self) -> &PackageInfo { + let Node { pkg, .. } = &self.graph[self.root]; + pkg + } + + pub fn root(&self) -> NodeIndex { + self.root + } + + pub fn graph(&self) -> &petgraph::graph::DiGraph { + &self.graph + } + + /// Get a mapping from [`PackageId`]s to [`NodeIndex`]s. + pub fn packages(&self) -> &BTreeMap { + &self.packages + } + + /// Get an iterator over all the packages in this dependency graph and their + /// dependency mappings. + pub fn iter_dependencies( + &self, + ) -> impl Iterator)> + '_ { + self.packages.iter().map(move |(id, index)| { + let dependencies: BTreeMap<_, _> = self + .graph + .edges(*index) + .map(|edge_ref| { + ( + edge_ref.weight().alias.as_str(), + &self.graph[edge_ref.target()].id, + ) + }) + .collect(); + (id, dependencies) + }) + } +} + +impl Index for DependencyGraph { + type Output = Node; + + #[track_caller] + fn index(&self, index: NodeIndex) -> &Self::Output { + &self.graph[index] + } +} + +impl Index<&NodeIndex> for DependencyGraph { + type Output = Node; + + #[track_caller] + fn index(&self, index: &NodeIndex) -> &Self::Output { + &self[*index] + } +} + +impl Index<&PackageId> for DependencyGraph { + type Output = Node; + + #[track_caller] + fn index(&self, index: &PackageId) -> &Self::Output { + let index = self.packages[index]; + &self[index] + } +} + +impl PartialEq for DependencyGraph { + fn eq(&self, other: &Self) -> bool { + let DependencyGraph { + root, + graph, + packages, + } = self; + + root.index() == other.root.index() + && *packages == other.packages + && petgraph::algo::is_isomorphic_matching(graph, &other.graph, Node::eq, Edge::eq) + } +} + +impl Eq for DependencyGraph {} + +/// A node in the [`DependencyGraph`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Node { + pub id: PackageId, + pub pkg: PackageInfo, + pub dist: Option, +} + +/// An edge in the [`DependencyGraph`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Edge { + pub alias: String, } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/wasix/src/runtime/resolver/resolve.rs index fccbbec6104..64043a7e906 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/wasix/src/runtime/resolver/resolve.rs @@ -1,11 +1,16 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet, VecDeque}, + collections::{BTreeMap, HashSet, VecDeque}, path::PathBuf, }; +use petgraph::{ + graph::{DiGraph, NodeIndex}, + visit::EdgeRef, +}; use semver::Version; use crate::runtime::resolver::{ + outputs::{Edge, Node}, DependencyGraph, ItemLocation, PackageId, PackageInfo, PackageSummary, Resolution, ResolvedPackage, Source, }; @@ -71,20 +76,48 @@ async fn resolve_dependency_graph( root: &PackageInfo, source: &dyn Source, ) -> Result { - let mut dependencies = HashMap::new(); - let mut package_info = HashMap::new(); - let mut distribution = HashMap::new(); + let DiscoveredPackages { + root, + graph, + indices, + packages, + } = discover_dependencies(root_id, root, source).await?; + + check_for_duplicate_versions(indices.iter().copied().map(|ix| &graph[ix].id))?; + log_dependencies(&graph, root); + + let graph = DependencyGraph::new(root, graph, packages); + + Ok(graph) +} + +async fn discover_dependencies( + root_id: &PackageId, + root: &PackageInfo, + source: &dyn Source, +) -> Result { + let mut nodes: BTreeMap = BTreeMap::new(); + let mut graph: DiGraph = DiGraph::new(); - package_info.insert(root_id.clone(), root.clone()); + let root_index = graph.add_node(Node { + id: root_id.clone(), + pkg: root.clone(), + dist: None, + }); let mut to_visit = VecDeque::new(); + to_visit.push_back(root_index); - to_visit.push_back((root_id.clone(), root.clone())); + while let Some(index) = to_visit.pop_front() { + let mut deps = BTreeMap::new(); - while let Some((id, info)) = to_visit.pop_front() { - let mut deps = HashMap::new(); + let mut to_add = Vec::new(); - for dep in &info.dependencies { + for dep in &graph[index].pkg.dependencies { + // Get the latest version that satisfies our requirement. If we were + // doing this more rigorously, we would be narrowing the version + // down using existing requirements and trying to reuse the same + // dependency when possible. let dep_summary = source .latest(&dep.pkg) .await @@ -92,67 +125,85 @@ async fn resolve_dependency_graph( deps.insert(dep.alias().to_string(), dep_summary.package_id()); let dep_id = dep_summary.package_id(); - if dependencies.contains_key(&dep_id) { + if nodes.contains_key(&dep_id) { // We don't need to visit this dependency again continue; } + // Otherwise, add it to the graph and schedule its dependencies to + // be retrieved + let PackageSummary { pkg, dist } = dep_summary; - to_visit.push_back((dep_id.clone(), pkg.clone())); - package_info.insert(dep_id.clone(), pkg); - distribution.insert(dep_id, dist); + to_add.push(Node { + id: dep_id, + pkg, + dist: Some(dist), + }); } - dependencies.insert(id, deps); + for node in to_add { + let id = node.id.clone(); + let index = graph.add_node(node); + nodes.insert(id, index); + } } - let graph = DependencyGraph { - root: root_id.clone(), - dependencies, - package_info, - distribution, - }; + let sorted_indices = + petgraph::algo::toposort(&graph, None).map_err(|cycle| cycle_error(&graph, cycle))?; - check_for_cycles(&graph.dependencies, &graph.root)?; - check_for_duplicate_versions(graph.dependencies.keys())?; - log_dependencies(&graph); + Ok(DiscoveredPackages { + root: root_index, + graph, + indices: sorted_indices, + packages: nodes, + }) +} - Ok(graph) +fn cycle_error( + graph: &petgraph::Graph, + cycle: petgraph::algo::Cycle, +) -> ResolveError { + let cyclic_node = cycle.node_id(); + let cycle: Vec<_> = + petgraph::algo::simple_paths::all_simple_paths(graph, cyclic_node, cyclic_node, 1, None) + .next() + .expect("We know there is at least one cycle"); + + let package_ids = cycle.into_iter().map(|ix| graph[ix].pkg.id()).collect(); + + ResolveError::Cycle(package_ids) } -#[tracing::instrument(level = "debug", name = "dependencies", skip_all)] -fn log_dependencies(graph: &DependencyGraph) { - let DependencyGraph { - root, dependencies, .. - } = graph; +#[derive(Debug)] +struct DiscoveredPackages { + root: NodeIndex, + graph: DiGraph, + /// All node indices, in topologically sorted order. + indices: Vec, + packages: BTreeMap, +} +#[tracing::instrument(level = "debug", name = "dependencies", skip_all)] +fn log_dependencies(graph: &DiGraph, root: NodeIndex) { tracing::debug!( - %root, - dependency_count=dependencies.len(), + root = root.index(), + dependency_count = graph.node_count(), "Resolved dependencies", ); if tracing::enabled!(tracing::Level::TRACE) { - let mut to_print = VecDeque::new(); - let mut visited = HashSet::new(); - to_print.push_back(root); - while let Some(next) = to_print.pop_front() { - visited.insert(next); - - let deps = &dependencies[next]; - let pretty: BTreeMap<_, _> = deps - .iter() - .map(|(name, pkg_id)| (name, pkg_id.to_string())) - .collect(); - - tracing::trace!( - package=%next, - dependencies=?pretty, - ); - - to_print.extend(deps.values().filter(|pkg| !visited.contains(pkg))); - } + petgraph::visit::depth_first_search(graph, [root], |event| { + if let petgraph::visit::DfsEvent::Discover(n, _) = event { + let package = &graph[n].id; + let dependencies: BTreeMap<_, _> = graph + .edges(n) + .map(|edge_ref| (&edge_ref.weight().alias, &graph[edge_ref.target()].id)) + .collect(); + + tracing::trace!( %package, ?dependencies); + } + }); } } @@ -164,7 +215,7 @@ fn check_for_duplicate_versions<'a, I>(package_ids: I) -> Result<(), ResolveErro where I: Iterator, { - let mut package_versions: HashMap<&str, HashSet<&Version>> = HashMap::new(); + let mut package_versions: BTreeMap<&str, HashSet<&Version>> = BTreeMap::new(); for PackageId { package_name, @@ -191,74 +242,30 @@ where Ok(()) } -/// Check for dependency cycles by doing a Depth First Search of the graph, -/// starting at the root. -fn check_for_cycles( - dependencies: &HashMap>, - root: &PackageId, -) -> Result<(), ResolveError> { - fn search<'a>( - dependencies: &'a HashMap>, - id: &'a PackageId, - visited: &mut HashSet<&'a PackageId>, - stack: &mut Vec<&'a PackageId>, - ) -> Result<(), ResolveError> { - if let Some(index) = stack.iter().position(|item| *item == id) { - // we've detected a cycle! - let mut cycle: Vec<_> = stack.drain(index..).cloned().collect(); - cycle.push(id.clone()); - return Err(ResolveError::Cycle(cycle)); - } - - if visited.contains(&id) { - // We already know this dependency is fine - return Ok(()); - } - - stack.push(id); - for dep in dependencies[id].values() { - search(dependencies, dep, visited, stack)?; - } - stack.pop(); - - Ok(()) - } - - let mut visited = HashSet::new(); - let mut stack = Vec::new(); - - search(dependencies, root, &mut visited, &mut stack) -} - -/// Given a [`DependencyGraph`], figure out how the resulting "package" would -/// look when loaded at runtime. +/// Given some [`DiscoveredPackages`], figure out how the resulting "package" +/// would look when loaded at runtime. fn resolve_package(dependency_graph: &DependencyGraph) -> Result { // FIXME: This code is all super naive and will break the moment there // are any conflicts or duplicate names. tracing::trace!("Resolving the package"); let mut commands = BTreeMap::new(); - - let filesystem = resolve_filesystem_mapping(dependency_graph)?; - - let mut to_check = VecDeque::new(); - let mut visited = HashSet::new(); - - to_check.push_back(&dependency_graph.root); + let mut filesystem = Vec::new(); let mut entrypoint = dependency_graph.root_info().entrypoint.clone(); - while let Some(next) = to_check.pop_front() { - visited.insert(next); - let pkg = &dependency_graph.package_info[next]; + for index in petgraph::algo::toposort(dependency_graph.graph(), None).expect("acyclic") { + let node = &dependency_graph[index]; + let id = &node.id; + let pkg = &node.pkg; - // set the entrypoint, if necessary + // update the entrypoint, if necessary if entrypoint.is_none() { if let Some(entry) = &pkg.entrypoint { tracing::trace!( entrypoint = entry.as_str(), - parent.name=next.package_name.as_str(), - parent.version=%next.version, + parent.name=id.package_name.as_str(), + parent.version=%id.version, "Inheriting the entrypoint", ); @@ -266,47 +273,50 @@ fn resolve_package(dependency_graph: &DependencyGraph) -> Result { + let resolved = ItemLocation { + name: cmd.name.clone(), + package: id.clone(), + }; + entry.insert(resolved); + tracing::trace!( + command.name=cmd.name.as_str(), + pkg.name=id.package_name.as_str(), + pkg.version=%id.version, + "Discovered command", + ); + } + std::collections::btree_map::Entry::Occupied(_) => { + tracing::trace!( + command.name=cmd.name.as_str(), + pkg.name=id.package_name.as_str(), + pkg.version=%id.version, + "Ignoring duplicate command", + ); + } + } } - let remaining_dependencies = dependency_graph.dependencies[next] - .values() - .filter(|id| !visited.contains(id)); - to_check.extend(remaining_dependencies); - } - - Ok(ResolvedPackage { - root_package: dependency_graph.root.clone(), - commands, - entrypoint, - filesystem, - }) -} - -fn resolve_filesystem_mapping( - graph: &DependencyGraph, -) -> Result, ResolveError> { - let mut mappings = Vec::new(); - - for (id, pkg) in &graph.package_info { for mapping in &pkg.filesystem { let dep = match &mapping.dependency_name { - Some(name) => graph.dependencies[id].get(name).unwrap(), + Some(name) => { + let dep_index = dependency_graph + .graph() + .edges(index) + .find(|edge| edge.weight().alias == *name) + .unwrap() + .target(); + &dependency_graph[dep_index].id + } None => id, }; - mappings.push(ResolvedFileSystemMapping { + filesystem.push(ResolvedFileSystemMapping { mount_path: PathBuf::from(&mapping.mount_path), volume_name: mapping.volume_name.clone(), package: dep.clone(), @@ -314,7 +324,12 @@ fn resolve_filesystem_mapping( } } - Ok(mappings) + Ok(ResolvedPackage { + root_package: dependency_graph.root_info().id(), + commands, + entrypoint, + filesystem, + }) } #[cfg(test)] @@ -371,7 +386,7 @@ mod tests { fn start_dependency_graph(&self) -> DependencyGraphBuilder<'_> { DependencyGraphBuilder { - dependencies: HashMap::new(), + dependencies: BTreeMap::new(), source: &self.0, } } @@ -455,7 +470,7 @@ mod tests { #[derive(Debug)] struct DependencyGraphBuilder<'source> { - dependencies: HashMap>, + dependencies: BTreeMap>, source: &'source InMemorySource, } @@ -470,20 +485,59 @@ mod tests { DependencyGraphEntryBuilder { builder: self, pkg_id, - dependencies: HashMap::new(), + dependencies: BTreeMap::new(), } } - fn finish(self) -> HashMap> { + fn finish(self) -> BTreeMap> { self.dependencies } + + /// Using the dependency mapping that we've been building up, construct + /// a dependency graph using the specified root package. + fn graph(self, root_name: &str, version: &str) -> DependencyGraph { + let version = version.parse().unwrap(); + let root_id = self.source.get(root_name, &version).unwrap().package_id(); + + let mut graph = DiGraph::new(); + let mut nodes = BTreeMap::new(); + + for id in self.dependencies.keys() { + let PackageSummary { pkg, dist } = + self.source.get(&id.package_name, &id.version).unwrap(); + let index = graph.add_node(Node { + id: pkg.id(), + pkg: pkg.clone(), + dist: Some(dist.clone()), + }); + nodes.insert(id.clone(), index); + } + + for (id, deps) in &self.dependencies { + let index = nodes[id]; + for (dep_name, dep_id) in deps { + let dep_index = nodes[dep_id]; + graph.add_edge( + index, + dep_index, + Edge { + alias: dep_name.clone(), + }, + ); + } + } + + let root_index = nodes[&root_id]; + + DependencyGraph::new(root_index, graph, nodes) + } } #[derive(Debug)] struct DependencyGraphEntryBuilder<'source, 'builder> { builder: &'builder mut DependencyGraphBuilder<'source>, pkg_id: PackageId, - dependencies: HashMap, + dependencies: BTreeMap, } impl<'source, 'builder> DependencyGraphEntryBuilder<'source, 'builder> { @@ -527,6 +581,20 @@ mod tests { } } + fn deps(resolution: &Resolution) -> BTreeMap> { + resolution + .graph + .iter_dependencies() + .map(|(id, deps)| { + let deps = deps + .into_iter() + .map(|(name, dep_id)| (name.to_string(), dep_id.clone())) + .collect(); + (id.clone(), deps) + }) + .collect() + } + #[tokio::test] async fn no_deps_and_no_commands() { let mut builder = RegistryBuilder::new(); @@ -540,7 +608,7 @@ mod tests { let mut dependency_graph = builder.start_dependency_graph(); dependency_graph.insert("root", "1.0.0"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -565,7 +633,7 @@ mod tests { let mut dependency_graph = builder.start_dependency_graph(); dependency_graph.insert("root", "1.0.0"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -601,7 +669,7 @@ mod tests { .insert("root", "1.0.0") .with_dependency("dep", "1.0.0"); dependency_graph.insert("dep", "1.0.0"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -638,7 +706,7 @@ mod tests { .insert("second", "1.0.0") .with_dependency("third", "1.0.0"); dependency_graph.insert("third", "1.0.0"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -671,7 +739,7 @@ mod tests { .insert("root", "1.0.0") .with_dependency("dep", "1.0.2"); dependency_graph.insert("dep", "1.0.2"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -760,7 +828,7 @@ mod tests { .insert("second", "1.0.0") .with_dependency("common", "1.2.0"); dependency_graph.insert("common", "1.2.0"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -799,7 +867,7 @@ mod tests { .with_dependency("second", "1.0.0"); dependency_graph.insert("first", "1.0.0"); dependency_graph.insert("second", "1.0.0"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -841,7 +909,7 @@ mod tests { .insert("root", "1.0.0") .with_dependency("dep", "1.0.0"); dependency_graph.insert("dep", "1.0.0"); - assert_eq!(resolution.graph.dependencies, dependency_graph.finish()); + assert_eq!(deps(&resolution), dependency_graph.finish()); assert_eq!( resolution.package, ResolvedPackage { @@ -946,17 +1014,11 @@ mod tests { builder.register("root", "1.0.0"); let mut dep_builder = builder.start_dependency_graph(); dep_builder.insert("root", "1.0.0"); - let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); - let root_id = pkg.id(); - let graph = DependencyGraph { - root: root_id.clone(), - dependencies: dep_builder.finish(), - package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), - distribution: vec![(root_id, dist)].into_iter().collect(), - }; - - let mappings = resolve_filesystem_mapping(&graph).unwrap(); - assert_eq!(mappings, Vec::new()); + let graph = dep_builder.graph("root", "1.0.0"); + + let pkg = resolve_package(&graph).unwrap(); + + assert!(pkg.filesystem.is_empty()); } #[test] @@ -967,18 +1029,12 @@ mod tests { .with_fs_mapping("lib", "/lib"); let mut dep_builder = builder.start_dependency_graph(); dep_builder.insert("root", "1.0.0"); - let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); - let root_id = pkg.id(); - let graph = DependencyGraph { - root: root_id.clone(), - dependencies: dep_builder.finish(), - package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), - distribution: vec![(root_id, dist)].into_iter().collect(), - }; - - let mappings = resolve_filesystem_mapping(&graph).unwrap(); + let graph = dep_builder.graph("root", "1.0.0"); + + let pkg = resolve_package(&graph).unwrap(); + assert_eq!( - mappings, + pkg.filesystem, vec![ResolvedFileSystemMapping { mount_path: PathBuf::from("/lib"), volume_name: "lib".to_string(), @@ -1008,19 +1064,12 @@ mod tests { .with_dependency("second", "1.0.0"); dep_builder.insert("first", "1.0.0"); dep_builder.insert("second", "1.0.0"); - let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); - let root_id = pkg.id(); - let graph = DependencyGraph { - root: root_id.clone(), - dependencies: dep_builder.finish(), - package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), - distribution: vec![(root_id, dist)].into_iter().collect(), - }; + let graph = dep_builder.graph("root", "1.0.0"); - let mappings = resolve_filesystem_mapping(&graph).unwrap(); + let pkg = resolve_package(&graph).unwrap(); assert_eq!( - mappings, + pkg.filesystem, vec![ ResolvedFileSystemMapping { mount_path: PathBuf::from("/usr/local/lib/second"), @@ -1054,19 +1103,12 @@ mod tests { .insert("root", "1.0.0") .with_dependency("dep", "1.0.0"); dep_builder.insert("dep", "1.0.0"); - let PackageSummary { pkg, dist } = builder.get("root", "1.0.0").clone(); - let root_id = pkg.id(); - let graph = DependencyGraph { - root: root_id.clone(), - dependencies: dep_builder.finish(), - package_info: vec![(root_id.clone(), pkg)].into_iter().collect(), - distribution: vec![(root_id, dist)].into_iter().collect(), - }; + let graph = dep_builder.graph("root", "1.0.0"); - let mappings = resolve_filesystem_mapping(&graph).unwrap(); + let pkg = resolve_package(&graph).unwrap(); assert_eq!( - mappings, + pkg.filesystem, vec![ResolvedFileSystemMapping { mount_path: PathBuf::from("/root"), volume_name: "dep-volume".to_string(), From be897c8d4985ce85cd6c065ea7834cc121a9f3e5 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 6 Jun 2023 21:55:15 +0800 Subject: [PATCH 03/16] Switched load_package_tree's over to using the filesystem annotations --- Cargo.lock | 29 ++- lib/cli/src/commands/create_exe.rs | 2 +- lib/wasix/Cargo.toml | 2 +- .../package_loader/load_package_tree.rs | 174 ++++++++++++++++-- lib/wasix/src/runtime/resolver/inputs.rs | 47 +++-- 5 files changed, 221 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0adde1037b8..fb7582c8a86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1317,7 +1317,7 @@ checksum = "64fba5a42bd76a17cad4bfa00de168ee1cbfa06a5e8ce992ae880218c05641a9" dependencies = [ "byteorder", "dynasm", - "memmap2", + "memmap2 0.5.10", ] [[package]] @@ -2484,6 +2484,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -4027,6 +4036,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shared-buffer" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a4987984c21cbd48a683c00975a0a9652d1882e8e986c42223f65440f4668f" +dependencies = [ + "bytes", + "memmap2 0.6.2", +] + [[package]] name = "shell-words" version = "1.1.0" @@ -5661,7 +5680,7 @@ dependencies = [ "hashbrown 0.11.2", "lazy_static", "leb128", - "memmap2", + "memmap2 0.5.10", "more-asserts", "region", "serde", @@ -6446,9 +6465,9 @@ dependencies = [ [[package]] name = "webc" -version = "5.0.2" +version = "5.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42af14e63ed784e4f813bd5fb35bc84016fa8b245879809547247da6051107f0" +checksum = "2041e22d0f634bec4f43f1fb77620fd68fb1adaa3f0a721fa682dcdb2e916840" dependencies = [ "anyhow", "base64", @@ -6457,7 +6476,6 @@ dependencies = [ "indexmap", "leb128", "lexical-sort", - "memmap2", "once_cell", "path-clean", "rand", @@ -6465,6 +6483,7 @@ dependencies = [ "serde_cbor", "serde_json", "sha2", + "shared-buffer", "thiserror", "url", "walkdir", diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index d21a57d874a..babd148d2e2 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -514,7 +514,7 @@ fn serialize_volume_to_webc_v1(volume: &WebcVolume) -> Vec { if let Some(contents) = volume.read_file(&*path) { files.insert( webc::v1::DirOrFile::File(path.to_string().into()), - contents.into(), + contents.to_vec(), ); } } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 0cd094cec20..f008378eda6 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -29,7 +29,7 @@ bincode = { version = "1.3" } chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" -webc = { version = "5.0.2", default-features = false } +webc = { version = "5.0.3", default-features = false } serde_cbor = { version = "0.11.2", optional = true } anyhow = { version = "1.0.66" } lazy_static = "1.4" diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 104ab4162b4..4320e6ced97 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -1,21 +1,22 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, - path::Path, + path::{Path, PathBuf}, sync::Arc, }; use anyhow::{Context, Error}; use futures::{stream::FuturesUnordered, TryStreamExt}; use once_cell::sync::OnceCell; -use virtual_fs::{FileSystem, WebcVolumeFileSystem}; -use webc::compat::Container; +use virtual_fs::{FileSystem, OverlayFileSystem, WebcVolumeFileSystem}; +use webc::compat::{Container, Volume}; use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, runtime::{ package_loader::PackageLoader, resolver::{ - DependencyGraph, ItemLocation, PackageId, PackageSummary, Resolution, ResolvedPackage, + DependencyGraph, ItemLocation, PackageId, PackageSummary, Resolution, + ResolvedFileSystemMapping, ResolvedPackage, }, }, }; @@ -231,17 +232,7 @@ async fn fetch_dependencies( Ok(packages) } -fn filesystem( - packages: &HashMap, - pkg: &ResolvedPackage, -) -> Result { - // FIXME: Take the [fs] table into account - // See for more - let root = &packages[&pkg.root_package]; - let fs = WebcVolumeFileSystem::mount_all(root); - Ok(fs) -} - +/// How many bytes worth of files does a directory contain? fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 { let mut total = 0; @@ -263,3 +254,156 @@ fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 { total } + +/// Given a set of [`ResolvedFileSystemMapping`]s and the [`Container`] for each +/// package in a dependency tree, construct the resulting filesystem. +/// +/// # Note to future readers +/// +/// Sooo... this code is a bit convoluted because we're constrained by the +/// filesystem implementations we've got available. +/// +/// Ideally, we would create a WebcVolumeFileSystem for each volume we're +/// using, then we'd have a single "union" filesystem which lets you mount +/// filesystem objects under various paths and can deal with conflicts. +/// +/// The OverlayFileSystem lets us make files from multiple filesystem +/// implementations available at the same time, however all of the +/// filesystems will be mounted at "/", when the user wants to mount volumes +/// at arbitrary locations. +/// +/// The TmpFileSystem *does* allow mounting at non-root paths, however it can't +/// handle nested paths (e.g. mounting to "/lib" and "/lib/python3.10" - see +/// https://github.com/wasmerio/wasmer/issues/3678 for more) and you aren't +/// allowed to mount to "/" because it's a special directory that already +/// exists. +/// +/// As a result, we'll duct-tape things together and hope for the best 🤞 +fn filesystem( + packages: &HashMap, + pkg: &ResolvedPackage, +) -> Result { + let mut filesystems = Vec::new(); + let mut volumes: HashMap<&PackageId, BTreeMap> = HashMap::new(); + + let mut mountings: Vec<_> = pkg.filesystem.iter().collect(); + mountings.sort_by_key(|m| std::cmp::Reverse(m.mount_path.as_path())); + + for ResolvedFileSystemMapping { + mount_path, + volume_name, + package, + } in &pkg.filesystem + { + // Note: We want to reuse existing Volume instances if we can. That way + // we can keep the memory usage down. A webc::compat::Volume is + // reference-counted, anyway. + let container_volumes = match volumes.entry(package) { + std::collections::hash_map::Entry::Occupied(entry) => &*entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => { + // looks like we need to insert it + let container = packages.get(package) + .with_context(|| format!( + "\"{}\" wants to use the \"{}\" package, but it isn't in the dependency tree", + pkg.root_package, + package, + ))?; + &*entry.insert(container.volumes()) + } + }; + + let volume = container_volumes.get(volume_name).with_context(|| { + format!("The \"{package}\" package doesn't have a \"{volume_name}\" volume") + })?; + + let fs = NestedFileSystem::new( + mount_path.clone(), + WebcVolumeFileSystem::new(volume.clone()), + ); + filesystems.push(fs); + } + + let fs = OverlayFileSystem::new(virtual_fs::EmptyFileSystem::default(), filesystems); + + Ok(fs) +} + +/// A [`FileSystem`] implementation that exposes some other filesystem under a +/// nested directory. +#[derive(Debug, Clone, PartialEq)] +struct NestedFileSystem { + path: PathBuf, + inner: F, +} + +impl NestedFileSystem { + fn new(path: PathBuf, inner: F) -> Self { + NestedFileSystem { path, inner } + } + + fn strip_prefix(&self, path: &Path) -> Result { + let path = path + .strip_prefix(&self.path) + .map_err(|_| virtual_fs::FsError::BaseNotDirectory)?; + + // Don't forget to make the path absolute again. + Ok(Path::new("/").join(path)) + } +} + +impl FileSystem for NestedFileSystem +where + F: FileSystem, +{ + fn read_dir(&self, path: &Path) -> virtual_fs::Result { + let path = self.strip_prefix(path)?; + self.inner.read_dir(&path) + } + + fn create_dir(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.strip_prefix(path)?; + self.inner.create_dir(&path) + } + + fn remove_dir(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.strip_prefix(path)?; + self.inner.remove_dir(&path) + } + + fn rename(&self, from: &Path, to: &Path) -> virtual_fs::Result<()> { + let from = self.strip_prefix(from)?; + let to = self.strip_prefix(to)?; + self.inner.rename(&from, &to) + } + + fn metadata(&self, path: &Path) -> virtual_fs::Result { + let path = self.strip_prefix(path)?; + self.inner.metadata(&path) + } + + fn remove_file(&self, path: &Path) -> virtual_fs::Result<()> { + let path = self.strip_prefix(path)?; + self.inner.remove_file(&path) + } + + fn new_open_options(&self) -> virtual_fs::OpenOptions { + virtual_fs::OpenOptions::new(self) + } +} + +impl virtual_fs::FileOpener for NestedFileSystem +where + F: FileSystem, +{ + fn open( + &self, + path: &Path, + conf: &virtual_fs::OpenOptionsConfig, + ) -> virtual_fs::Result> { + let path = self.strip_prefix(path)?; + self.inner + .new_open_options() + .options(conf.clone()) + .open(path) + } +} diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/wasix/src/runtime/resolver/inputs.rs index 51fddcb781f..c5fd83a7d1b 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/wasix/src/runtime/resolver/inputs.rs @@ -188,7 +188,7 @@ pub struct PackageInfo { impl PackageInfo { pub fn from_manifest(manifest: &Manifest) -> Result { let WapmAnnotations { name, version, .. } = manifest - .package_annotation("wapm")? + .wapm()? .context("Unable to find the \"wapm\" annotations")?; let dependencies = manifest @@ -210,7 +210,7 @@ impl PackageInfo { }) .collect(); - let filesystem = filesystem_mapping_from_manifest(manifest); + let filesystem = filesystem_mapping_from_manifest(manifest)?; Ok(PackageInfo { name, @@ -230,15 +230,40 @@ impl PackageInfo { } } -fn filesystem_mapping_from_manifest(_manifest: &Manifest) -> Vec { - // FIXME(Michael-F-Bryan): wapm2pirita never added filesystem mappings to the manifest - // That means we need to - // - Figure out whether filesystem mappings belong to the whole package or - // if each command has their own set of mappings - // - Update wapm-targz-to-pirita to copy the [fs] table across - // - Re-generate all packages on WAPM - // - Update this function to copy metadata into our internal datastructures - Vec::new() +fn filesystem_mapping_from_manifest( + manifest: &Manifest, +) -> Result, serde_cbor::Error> { + match manifest.filesystem()? { + Some(webc::metadata::annotations::FileSystemMappings(mappings)) => { + let mappings = mappings + .into_iter() + .map(|mapping| FileSystemMapping { + volume_name: mapping.volume_name, + mount_path: mapping.mount_path, + dependency_name: mapping.from, + }) + .collect(); + + Ok(mappings) + } + None => { + // A "fs" annotation hasn't been attached to this package. This was + // the case when *.webc files were generated by wapm2pirita version + // 1.0.29 and earlier. + // + // To maintain compatibility with those older packages, we'll say + // that the "atom" volume from the current package is mounted to "/" + // and contains all files in the package. + tracing::debug!( + "No \"fs\" package annotations found. Mounting the \"atom\" volume to \"/\" for compatibility." + ); + Ok(vec![FileSystemMapping { + volume_name: "atom".to_string(), + mount_path: "/".to_string(), + dependency_name: None, + }]) + } + } } #[derive(Debug, Clone, PartialEq, Eq)] From d6ad768459252e23dd3aebc2f8002b1d63721894 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 6 Jun 2023 22:55:40 +0800 Subject: [PATCH 04/16] Made serde_cbor a required dependency --- lib/wasix/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index f008378eda6..9117d448728 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -30,7 +30,7 @@ chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", derivative = { version = "^2" } bytes = "1" webc = { version = "5.0.3", default-features = false } -serde_cbor = { version = "0.11.2", optional = true } +serde_cbor = { version = "0.11.2" } anyhow = { version = "1.0.66" } lazy_static = "1.4" sha2 = { version = "0.10" } @@ -111,7 +111,7 @@ default = ["sys-default"] time = ["tokio/time"] -webc_runner = ["serde_cbor"] +webc_runner = [] webc_runner_rt_wasi = [] webc_runner_rt_wcgi = ["hyper", "wcgi", "wcgi-host", "tower", "tower-http"] webc_runner_rt_emscripten = ["wasmer-emscripten"] From 13239a32ac4fc567eb23e1d1c26da77ba434b3c8 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 6 Jun 2023 23:51:21 +0800 Subject: [PATCH 05/16] Fixed up the PartialEq implementation for DependencyGraph --- lib/wasix/src/runtime/resolver/outputs.rs | 42 +++++++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/lib/wasix/src/runtime/resolver/outputs.rs b/lib/wasix/src/runtime/resolver/outputs.rs index ef9c9672693..e9e8b6302b0 100644 --- a/lib/wasix/src/runtime/resolver/outputs.rs +++ b/lib/wasix/src/runtime/resolver/outputs.rs @@ -5,7 +5,10 @@ use std::{ path::PathBuf, }; -use petgraph::{graph::NodeIndex, visit::EdgeRef}; +use petgraph::{ + graph::{DiGraph, NodeIndex}, + visit::EdgeRef, +}; use semver::Version; use crate::runtime::resolver::{DistributionInfo, PackageInfo}; @@ -45,16 +48,25 @@ impl Display for PackageId { #[derive(Debug, Clone)] pub struct DependencyGraph { root: NodeIndex, - graph: petgraph::graph::DiGraph, + graph: DiGraph, packages: BTreeMap, } impl DependencyGraph { pub(crate) fn new( root: NodeIndex, - graph: petgraph::graph::DiGraph, + graph: DiGraph, packages: BTreeMap, ) -> Self { + if cfg!(debug_assertions) { + // Note: We assume the packages table correctly maps PackageIds to + // node indices as part of the PartialEq implementation. + for (id, index) in &packages { + let node = &graph[*index]; + assert_eq!(*id, node.id, "Mismatch for node {index:?}"); + } + } + DependencyGraph { root, graph, @@ -71,7 +83,7 @@ impl DependencyGraph { self.root } - pub fn graph(&self) -> &petgraph::graph::DiGraph { + pub fn graph(&self) -> &DiGraph { &self.graph } @@ -137,9 +149,24 @@ impl PartialEq for DependencyGraph { packages, } = self; - root.index() == other.root.index() - && *packages == other.packages - && petgraph::algo::is_isomorphic_matching(graph, &other.graph, Node::eq, Edge::eq) + // Make sure their roots are the same package + let this_root = graph.node_weight(*root); + let other_root = other.graph.node_weight(other.root); + + match (this_root, other_root) { + (Some(lhs), Some(rhs)) if lhs == rhs => {} + _ => return false, + } + + // the packages table *should* just be an optimisation. We've checked + // it is valid as part of DependencyGraph::new() and the entire graph + // is immutable, so it's fine to ignore. + let _ = packages; + + // Most importantly, the graphs should be "the same" (i.e. if a node + // in one graph is a + // nodes are connected to the same nodes in both) + petgraph::algo::is_isomorphic_matching(graph, &other.graph, Node::eq, Edge::eq) } } @@ -156,6 +183,7 @@ pub struct Node { /// An edge in the [`DependencyGraph`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Edge { + /// The name used by the package when referring to this dependency. pub alias: String, } From 73cc213a4d3c66ff30073861d041d90ad45839b9 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 7 Jun 2023 12:18:55 +0800 Subject: [PATCH 06/16] Use BufferUnordered instead of FuturesUnordered --- .../package_loader/load_package_tree.rs | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 4320e6ced97..ae3019d8c4d 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -5,7 +5,7 @@ use std::{ }; use anyhow::{Context, Error}; -use futures::{stream::FuturesUnordered, TryStreamExt}; +use futures::{StreamExt, TryStreamExt}; use once_cell::sync::OnceCell; use virtual_fs::{FileSystem, OverlayFileSystem, WebcVolumeFileSystem}; use webc::compat::{Container, Volume}; @@ -21,6 +21,9 @@ use crate::{ }, }; +/// The maximum number of packages that will be loaded in parallel. +const MAX_PARALLEL_DOWNLOADS: usize = 32; + /// Given a fully resolved package, load it into memory for execution. #[tracing::instrument(level = "debug", skip_all)] pub async fn load_package_tree( @@ -214,20 +217,24 @@ async fn fetch_dependencies( // We don't need to download the root package packages.remove(&pkg.root_package); - let packages: FuturesUnordered<_> = packages - .into_iter() - .filter_map(|id| { - let crate::runtime::resolver::Node { pkg, dist, .. } = &graph[&id]; - let summary = PackageSummary { - pkg: pkg.clone(), - dist: dist.clone()?, - }; - Some((id, summary)) + let packages = packages.into_iter().filter_map(|id| { + let crate::runtime::resolver::Node { pkg, dist, .. } = &graph[&id]; + let summary = PackageSummary { + pkg: pkg.clone(), + dist: dist.clone()?, + }; + Some((id, summary)) + }); + let packages: HashMap = futures::stream::iter(packages) + .map(|(id, s)| async move { + match loader.load(&s).await { + Ok(webc) => Ok((id, webc)), + Err(e) => Err(e), + } }) - .map(|(id, summary)| async move { loader.load(&summary).await.map(|webc| (id, webc)) }) - .collect(); - - let packages: HashMap = packages.try_collect().await?; + .buffer_unordered(MAX_PARALLEL_DOWNLOADS) + .try_collect() + .await?; Ok(packages) } From 7d999a9af0323cbe8b0039ec42b65c68e8fa84ed Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 7 Jun 2023 15:48:47 +0800 Subject: [PATCH 07/16] Forgot to add edges --- .../src/runtime/resolver/in_memory_source.rs | 8 ++- lib/wasix/src/runtime/resolver/outputs.rs | 13 ++++ lib/wasix/src/runtime/resolver/resolve.rs | 72 +++++++++++++------ lib/wasix/src/runtime/resolver/wapm_source.rs | 8 ++- 4 files changed, 74 insertions(+), 27 deletions(-) diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/wasix/src/runtime/resolver/in_memory_source.rs index 9aaa56ef3b9..de3db68afa2 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/wasix/src/runtime/resolver/in_memory_source.rs @@ -121,7 +121,7 @@ mod tests { use tempfile::TempDir; use crate::runtime::resolver::{ - inputs::{DistributionInfo, PackageInfo}, + inputs::{DistributionInfo, FileSystemMapping, PackageInfo}, Dependency, WebcHash, }; @@ -168,7 +168,11 @@ mod tests { name: "bash".to_string(), }], entrypoint: Some("bash".to_string()), - filesystem: Vec::new(), + filesystem: vec![FileSystemMapping { + volume_name: "atom".to_string(), + mount_path: "/".to_string(), + dependency_name: None, + }], }, dist: DistributionInfo { webc: crate::runtime::resolver::utils::url_from_file_path( diff --git a/lib/wasix/src/runtime/resolver/outputs.rs b/lib/wasix/src/runtime/resolver/outputs.rs index e9e8b6302b0..b549dcd2d40 100644 --- a/lib/wasix/src/runtime/resolver/outputs.rs +++ b/lib/wasix/src/runtime/resolver/outputs.rs @@ -66,6 +66,10 @@ impl DependencyGraph { assert_eq!(*id, node.id, "Mismatch for node {index:?}"); } } + debug_assert!( + packages.values().any(|ix| *ix == root), + "The packages mapping doesn't contain the root node" + ); DependencyGraph { root, @@ -111,6 +115,12 @@ impl DependencyGraph { (id, dependencies) }) } + + /// Visualise this graph as a DOT program. + pub fn visualise(&self) -> String { + let graph = self.graph.map(|_, node| &node.id, |_, edge| &edge.alias); + petgraph::dot::Dot::new(&graph).to_string() + } } impl Index for DependencyGraph { @@ -177,6 +187,9 @@ impl Eq for DependencyGraph {} pub struct Node { pub id: PackageId, pub pkg: PackageInfo, + /// Information about how the package is distributed. + /// + /// This will only ever be missing for the root package. pub dist: Option, } diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/wasix/src/runtime/resolver/resolve.rs index 64043a7e906..2d8011ac5b3 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/wasix/src/runtime/resolver/resolve.rs @@ -104,13 +104,13 @@ async fn discover_dependencies( pkg: root.clone(), dist: None, }); + nodes.insert(root_id.clone(), root_index); let mut to_visit = VecDeque::new(); to_visit.push_back(root_index); while let Some(index) = to_visit.pop_front() { - let mut deps = BTreeMap::new(); - + eprintln!("Checking {}", graph[index].id); let mut to_add = Vec::new(); for dep in &graph[index].pkg.dependencies { @@ -122,35 +122,49 @@ async fn discover_dependencies( .latest(&dep.pkg) .await .map_err(ResolveError::Registry)?; - deps.insert(dep.alias().to_string(), dep_summary.package_id()); let dep_id = dep_summary.package_id(); - if nodes.contains_key(&dep_id) { - // We don't need to visit this dependency again - continue; - } - - // Otherwise, add it to the graph and schedule its dependencies to - // be retrieved - let PackageSummary { pkg, dist } = dep_summary; - to_add.push(Node { + let alias = dep.alias().to_string(); + let node = Node { id: dep_id, pkg, dist: Some(dist), - }); + }; + // Note: We can't add the node to the graph directly because we're + // still iterating over it. + to_add.push((alias, node)); } - for node in to_add { - let id = node.id.clone(); - let index = graph.add_node(node); - nodes.insert(id, index); + for (alias, node) in to_add { + let dep_id = node.id.clone(); + eprintln!("{} -> {} (as {alias})", &graph[index].id, dep_id); + + let dep_index = match nodes.get(&dep_id) { + Some(&ix) => ix, + None => { + // Create a new node and schedule its dependencies to be + // retrieved + let ix = graph.add_node(node); + eprintln!("Scheduling to check \"{dep_id}\""); + nodes.insert(dep_id, ix); + to_visit.push_back(ix); + ix + } + }; + + graph.add_edge(index, dep_index, Edge { alias }); } } let sorted_indices = - petgraph::algo::toposort(&graph, None).map_err(|cycle| cycle_error(&graph, cycle))?; + dbg!(petgraph::algo::toposort(&graph, None)).map_err(|cycle| cycle_error(&graph, cycle))?; + dbg!(sorted_indices + .iter() + .copied() + .map(|ix| &graph[ix].id) + .collect::>()); Ok(DiscoveredPackages { root: root_index, @@ -164,6 +178,12 @@ fn cycle_error( graph: &petgraph::Graph, cycle: petgraph::algo::Cycle, ) -> ResolveError { + eprintln!( + "{}", + petgraph::dot::Dot::new( + &graph.map(|_, node| node.id.to_string(), |_, edge| edge.alias.clone()), + ) + ); let cyclic_node = cycle.node_id(); let cycle: Vec<_> = petgraph::algo::simple_paths::all_simple_paths(graph, cyclic_node, cyclic_node, 1, None) @@ -201,7 +221,7 @@ fn log_dependencies(graph: &DiGraph, root: NodeIndex) { .map(|edge_ref| (&edge_ref.weight().alias, &graph[edge_ref.target()].id)) .collect(); - tracing::trace!( %package, ?dependencies); + tracing::trace!(%package, ?dependencies); } }); } @@ -324,6 +344,12 @@ fn resolve_package(dependency_graph: &DependencyGraph) -> Result Date: Wed, 7 Jun 2023 15:49:31 +0800 Subject: [PATCH 08/16] Added a test for mapping directories to different names --- Cargo.lock | 1 + lib/wasix/Cargo.toml | 1 + lib/wasix/src/bin_factory/binary_package.rs | 85 +++++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index fb7582c8a86..6c2993191a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6152,6 +6152,7 @@ dependencies = [ "virtual-net", "wai-bindgen-wasmer", "waker-fn", + "wapm-targz-to-pirita", "wasm-bindgen", "wasm-bindgen-test", "wasmer", diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 9117d448728..39198ff0460 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -97,6 +97,7 @@ wasm-bindgen = ">= 0.2.74, < 0.2.85" wasmer = { path = "../api", version = "=4.0.0-beta.1", default-features = false, features = ["wat", "js-serializable-module"] } tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } pretty_assertions = "1.3.0" +wapm-targz-to-pirita = "0.2.1" [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = "0.3.0" diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 3677a3db943..235466a7a33 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -136,3 +136,88 @@ impl BinaryPackage { }) } } + +#[cfg(test)] +mod tests { + use std::{collections::BTreeMap, path::Path}; + + use tempfile::TempDir; + use virtual_fs::AsyncReadExt; + + use crate::{runtime::task_manager::tokio::TokioTaskManager, PluggableRuntime}; + + use super::*; + + #[tokio::test] + async fn fs_table_can_map_directories_to_different_names() { + let temp = TempDir::new().unwrap(); + let wasmer_toml = r#" + [package] + name = "some/package" + version = "0.0.0" + description = "a dummy package" + + [fs] + "/public" = "./out" + "#; + std::fs::write(temp.path().join("wasmer.toml"), wasmer_toml).unwrap(); + let out = temp.path().join("out"); + std::fs::create_dir_all(&out).unwrap(); + let file_txt = "Hello, World!"; + std::fs::write(out.join("file.txt"), file_txt).unwrap(); + let webc = construct_webc_in_memory(temp.path()); + let webc = Container::from_bytes(webc).unwrap(); + let tasks = TokioTaskManager::new(tokio::runtime::Handle::current()); + let runtime = PluggableRuntime::new(Arc::new(tasks)); + + let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap(); + + // We should have mapped "./out/file.txt" on the host to + // "/public/file.txt" on the guest. + let mut f = pkg + .webc_fs + .new_open_options() + .read(true) + .open("/public/file.txt") + .unwrap(); + let mut buffer = String::new(); + f.read_to_string(&mut buffer).await.unwrap(); + assert_eq!(buffer, file_txt); + } + + fn construct_webc_in_memory(dir: &Path) -> Vec { + let mut files = BTreeMap::new(); + load_files_from_disk(&mut files, dir, dir); + + let wasmer_toml = webc::v1::DirOrFile::File("wasmer.toml".into()); + if let Some(toml_data) = files.remove(&wasmer_toml) { + // HACK(Michael-F-Bryan): The version of wapm-targz-to-pirita we are + // using doesn't know we renamed "wapm.toml" to "wasmer.toml", so we + // manually patch things up if people have already migrated their + // projects. + files + .entry(webc::v1::DirOrFile::File("wapm.toml".into())) + .or_insert(toml_data); + } + + let functions = wapm_targz_to_pirita::TransformManifestFunctions::default(); + wapm_targz_to_pirita::generate_webc_file(files, dir, None, &functions).unwrap() + } + + fn load_files_from_disk(files: &mut wapm_targz_to_pirita::FileMap, dir: &Path, base: &Path) { + let entries = dir.read_dir().unwrap(); + + for entry in entries { + let path = entry.unwrap().path(); + let relative_path = path.strip_prefix(base).unwrap().to_path_buf(); + + if path.is_dir() { + load_files_from_disk(files, &path, base); + files.insert(webc::v1::DirOrFile::Dir(relative_path), Vec::new()); + } else if path.is_file() { + let data = std::fs::read(&path).unwrap(); + files.insert(webc::v1::DirOrFile::File(relative_path), data); + } + } + } +} From abb06311d69a26717426f148ac022c5c79c0e665 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 8 Jun 2023 14:58:20 +0800 Subject: [PATCH 09/16] Fixed the cycle error bug --- lib/wasix/src/runtime/resolver/resolve.rs | 46 +++++++++---------- lib/wasix/src/runtime/resolver/wapm_source.rs | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/wasix/src/runtime/resolver/resolve.rs index 2d8011ac5b3..8d43940b19f 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/wasix/src/runtime/resolver/resolve.rs @@ -158,13 +158,7 @@ async fn discover_dependencies( } } - let sorted_indices = - dbg!(petgraph::algo::toposort(&graph, None)).map_err(|cycle| cycle_error(&graph, cycle))?; - dbg!(sorted_indices - .iter() - .copied() - .map(|ix| &graph[ix].id) - .collect::>()); + let sorted_indices = petgraph::algo::toposort(&graph, None).map_err(|_| cycle_error(&graph))?; Ok(DiscoveredPackages { root: root_index, @@ -174,24 +168,30 @@ async fn discover_dependencies( }) } -fn cycle_error( - graph: &petgraph::Graph, - cycle: petgraph::algo::Cycle, -) -> ResolveError { - eprintln!( - "{}", - petgraph::dot::Dot::new( - &graph.map(|_, node| node.id.to_string(), |_, edge| edge.alias.clone()), - ) - ); - let cyclic_node = cycle.node_id(); - let cycle: Vec<_> = - petgraph::algo::simple_paths::all_simple_paths(graph, cyclic_node, cyclic_node, 1, None) - .next() - .expect("We know there is at least one cycle"); +fn cycle_error(graph: &petgraph::Graph) -> ResolveError { + // We know the graph has at least one cycle, so use SCC to find it. + let mut cycle = petgraph::algo::kosaraju_scc(graph) + .into_iter() + .find(|cycle| cycle.len() > 1) + .expect("We know there is at least one cycle"); + + // we want the loop's starting node to be deterministic (for tests), and + // nodes with lower indices are normally closer to the root of the + // dependency tree. + let lowest_index_node = cycle.iter().copied().min().expect("Cycle is non-empty"); + + // We want the cycle vector to start with that node, so let's do a bit of + // shuffling + let offset = cycle + .iter() + .position(|&node| node == lowest_index_node) + .unwrap(); + cycle.rotate_left(offset); - let package_ids = cycle.into_iter().map(|ix| graph[ix].pkg.id()).collect(); + // Don't forget to make the cycle start and end with the same node + cycle.push(lowest_index_node); + let package_ids = cycle.into_iter().map(|ix| graph[ix].pkg.id()).collect(); ResolveError::Cycle(package_ids) } diff --git a/lib/wasix/src/runtime/resolver/wapm_source.rs b/lib/wasix/src/runtime/resolver/wapm_source.rs index 8813d51fcbb..9bb3ac8a05e 100644 --- a/lib/wasix/src/runtime/resolver/wapm_source.rs +++ b/lib/wasix/src/runtime/resolver/wapm_source.rs @@ -200,7 +200,7 @@ mod tests { use crate::{ http::HttpResponse, - runtime::resolver::inputs::{DistributionInfo, PackageInfo, FileSystemMapping}, + runtime::resolver::inputs::{DistributionInfo, FileSystemMapping, PackageInfo}, }; use super::*; From e2e76d285c113e7d49d57cb1e9e2e9e05fdf9ea1 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 8 Jun 2023 16:43:09 +0800 Subject: [PATCH 10/16] Add wapm-targz-to-pirita's BUSL license as an exception --- deny.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index e8382d45698..fc715e92829 100644 --- a/deny.toml +++ b/deny.toml @@ -111,7 +111,8 @@ confidence-threshold = 0.8 exceptions = [ # Each entry is the crate and version constraint, and its specific allow # list - { name = "webc", allow = ["BUSL-1.1"] } + { name = "webc", allow = ["BUSL-1.1"] }, + { name = "wapm-targz-to-pirita", allow = ["BUSL-1.1"] }, ] From 683e47e363f19dd41ca8f4d1ac1062af7a629284 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 8 Jun 2023 17:13:27 +0800 Subject: [PATCH 11/16] Wired up the "original_path" field from FileSystemMapping --- Cargo.lock | 4 +- lib/wasix/Cargo.toml | 2 +- .../package_loader/load_package_tree.rs | 78 ++++++++++++------- lib/wasix/src/runtime/resolver/inputs.rs | 4 + lib/wasix/src/runtime/resolver/outputs.rs | 1 + lib/wasix/src/runtime/resolver/resolve.rs | 38 ++++++--- 6 files changed, 88 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c2993191a5..1b6d0edfa98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6466,9 +6466,9 @@ dependencies = [ [[package]] name = "webc" -version = "5.0.3" +version = "5.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2041e22d0f634bec4f43f1fb77620fd68fb1adaa3f0a721fa682dcdb2e916840" +checksum = "02ba1b6e7ad252e691a86d727aa422fbc5ed95e9bca4ec8547869e9b5779f8f3" dependencies = [ "anyhow", "base64", diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 39198ff0460..8868a845edf 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -29,7 +29,7 @@ bincode = { version = "1.3" } chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" -webc = { version = "5.0.3", default-features = false } +webc = { version = "5.0.4", default-features = false } serde_cbor = { version = "0.11.2" } anyhow = { version = "1.0.66" } lazy_static = "1.4" diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index ae3019d8c4d..5feefdae4bb 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, + fmt::Debug, path::{Path, PathBuf}, sync::Arc, }; @@ -300,6 +301,7 @@ fn filesystem( mount_path, volume_name, package, + original_path, } in &pkg.filesystem { // Note: We want to reuse existing Volume instances if we can. That way @@ -323,10 +325,19 @@ fn filesystem( format!("The \"{package}\" package doesn't have a \"{volume_name}\" volume") })?; - let fs = NestedFileSystem::new( - mount_path.clone(), - WebcVolumeFileSystem::new(volume.clone()), - ); + let original_path = PathBuf::from(original_path); + let mount_path = mount_path.clone(); + // Get a filesystem which will map "$mount_dir/some-path" to + // "$original_path/some-path" on the original volume + let fs = + MappedPathFileSystem::new(WebcVolumeFileSystem::new(volume.clone()), move |path| { + let without_mount_dir = path + .strip_prefix(&mount_path) + .map_err(|_| virtual_fs::FsError::BaseNotDirectory)?; + let path_on_original_volume = original_path.join(without_mount_dir); + Ok(path_on_original_volume) + }); + filesystems.push(fs); } @@ -335,61 +346,63 @@ fn filesystem( Ok(fs) } -/// A [`FileSystem`] implementation that exposes some other filesystem under a -/// nested directory. -#[derive(Debug, Clone, PartialEq)] -struct NestedFileSystem { - path: PathBuf, +/// A [`FileSystem`] implementation that lets you map the [`Path`] to something +/// else. +#[derive(Clone, PartialEq)] +struct MappedPathFileSystem { inner: F, + map: M, } -impl NestedFileSystem { - fn new(path: PathBuf, inner: F) -> Self { - NestedFileSystem { path, inner } +impl MappedPathFileSystem +where + M: Fn(&Path) -> Result + Send + Sync + 'static, +{ + fn new(inner: F, map: M) -> Self { + MappedPathFileSystem { inner, map } } - fn strip_prefix(&self, path: &Path) -> Result { - let path = path - .strip_prefix(&self.path) - .map_err(|_| virtual_fs::FsError::BaseNotDirectory)?; + fn path(&self, path: &Path) -> Result { + let path = (self.map)(path)?; // Don't forget to make the path absolute again. Ok(Path::new("/").join(path)) } } -impl FileSystem for NestedFileSystem +impl FileSystem for MappedPathFileSystem where F: FileSystem, + M: Fn(&Path) -> Result + Send + Sync + 'static, { fn read_dir(&self, path: &Path) -> virtual_fs::Result { - let path = self.strip_prefix(path)?; + let path = self.path(path)?; self.inner.read_dir(&path) } fn create_dir(&self, path: &Path) -> virtual_fs::Result<()> { - let path = self.strip_prefix(path)?; + let path = self.path(path)?; self.inner.create_dir(&path) } fn remove_dir(&self, path: &Path) -> virtual_fs::Result<()> { - let path = self.strip_prefix(path)?; + let path = self.path(path)?; self.inner.remove_dir(&path) } fn rename(&self, from: &Path, to: &Path) -> virtual_fs::Result<()> { - let from = self.strip_prefix(from)?; - let to = self.strip_prefix(to)?; + let from = self.path(from)?; + let to = self.path(to)?; self.inner.rename(&from, &to) } fn metadata(&self, path: &Path) -> virtual_fs::Result { - let path = self.strip_prefix(path)?; + let path = self.path(path)?; self.inner.metadata(&path) } fn remove_file(&self, path: &Path) -> virtual_fs::Result<()> { - let path = self.strip_prefix(path)?; + let path = self.path(path)?; self.inner.remove_file(&path) } @@ -398,19 +411,32 @@ where } } -impl virtual_fs::FileOpener for NestedFileSystem +impl virtual_fs::FileOpener for MappedPathFileSystem where F: FileSystem, + M: Fn(&Path) -> Result + Send + Sync + 'static, { fn open( &self, path: &Path, conf: &virtual_fs::OpenOptionsConfig, ) -> virtual_fs::Result> { - let path = self.strip_prefix(path)?; + let path = self.path(path)?; self.inner .new_open_options() .options(conf.clone()) .open(path) } } + +impl Debug for MappedPathFileSystem +where + F: Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MappedPathFileSystem") + .field("inner", &self.inner) + .field("map", &std::any::type_name::()) + .finish() + } +} diff --git a/lib/wasix/src/runtime/resolver/inputs.rs b/lib/wasix/src/runtime/resolver/inputs.rs index c5fd83a7d1b..64bca33e6c5 100644 --- a/lib/wasix/src/runtime/resolver/inputs.rs +++ b/lib/wasix/src/runtime/resolver/inputs.rs @@ -241,6 +241,7 @@ fn filesystem_mapping_from_manifest( volume_name: mapping.volume_name, mount_path: mapping.mount_path, dependency_name: mapping.from, + original_path: mapping.original_path, }) .collect(); @@ -260,6 +261,7 @@ fn filesystem_mapping_from_manifest( Ok(vec![FileSystemMapping { volume_name: "atom".to_string(), mount_path: "/".to_string(), + original_path: "/".to_string(), dependency_name: None, }]) } @@ -272,6 +274,8 @@ pub struct FileSystemMapping { pub volume_name: String, /// Where the volume should be mounted within the resulting filesystem. pub mount_path: String, + /// The path of the mapped item within its original volume. + pub original_path: String, /// The name of the package this volume comes from (current package if /// `None`). pub dependency_name: Option, diff --git a/lib/wasix/src/runtime/resolver/outputs.rs b/lib/wasix/src/runtime/resolver/outputs.rs index b549dcd2d40..a52b9fa6d0a 100644 --- a/lib/wasix/src/runtime/resolver/outputs.rs +++ b/lib/wasix/src/runtime/resolver/outputs.rs @@ -205,6 +205,7 @@ pub struct ResolvedFileSystemMapping { // TODO: Change this to a new type that isn't coupled to the OS pub mount_path: PathBuf, pub volume_name: String, + pub original_path: String, pub package: PackageId, } diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/wasix/src/runtime/resolver/resolve.rs index 8d43940b19f..f2b4c938ff1 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/wasix/src/runtime/resolver/resolve.rs @@ -338,6 +338,7 @@ fn resolve_package(dependency_graph: &DependencyGraph) -> Result &mut Self { + fn with_fs_mapping( + &mut self, + volume_name: &str, + original_path: &str, + mount_path: &str, + ) -> &mut Self { self.summary.pkg.filesystem.push(FileSystemMapping { volume_name: volume_name.to_string(), mount_path: mount_path.to_string(), + original_path: original_path.to_string(), dependency_name: None, }); self @@ -476,11 +483,13 @@ mod tests { &mut self, volume_name: &str, mount_path: &str, + original_path: &str, dependency: &str, ) -> &mut Self { self.summary.pkg.filesystem.push(FileSystemMapping { volume_name: volume_name.to_string(), mount_path: mount_path.to_string(), + original_path: original_path.to_string(), dependency_name: Some(dependency.to_string()), }); self @@ -1052,7 +1061,7 @@ mod tests { let mut builder = RegistryBuilder::new(); builder .register("root", "1.0.0") - .with_fs_mapping("lib", "/lib"); + .with_fs_mapping("lib", "/lib", "/other/lib"); let mut dep_builder = builder.start_dependency_graph(); dep_builder.insert("root", "1.0.0"); let graph = dep_builder.graph("root", "1.0.0"); @@ -1063,6 +1072,7 @@ mod tests { pkg.filesystem, vec![ResolvedFileSystemMapping { mount_path: PathBuf::from("/lib"), + original_path: "/other/lib".to_string(), volume_name: "lib".to_string(), package: builder.get("root", "1.0.0").package_id(), }] @@ -1076,13 +1086,17 @@ mod tests { .register("root", "1.0.0") .with_dependency("first", "=1.0.0") .with_dependency("second", "=1.0.0") - .with_fs_mapping("lib", "/root"); - builder - .register("first", "1.0.0") - .with_fs_mapping("main", "/usr/local/lib/first"); - builder - .register("second", "1.0.0") - .with_fs_mapping("main", "/usr/local/lib/second"); + .with_fs_mapping("lib", "/root", "/root"); + builder.register("first", "1.0.0").with_fs_mapping( + "main", + "/usr/local/lib/first", + "/usr/local/lib/first", + ); + builder.register("second", "1.0.0").with_fs_mapping( + "main", + "/usr/local/lib/second", + "/usr/local/lib/second", + ); let mut dep_builder = builder.start_dependency_graph(); dep_builder .insert("root", "1.0.0") @@ -1100,15 +1114,18 @@ mod tests { ResolvedFileSystemMapping { mount_path: PathBuf::from("/usr/local/lib/first"), volume_name: "main".to_string(), + original_path: "/usr/local/lib/first".to_string(), package: builder.get("first", "1.0.0").package_id(), }, ResolvedFileSystemMapping { mount_path: PathBuf::from("/usr/local/lib/second"), + original_path: "/usr/local/lib/second".to_string(), volume_name: "main".to_string(), package: builder.get("second", "1.0.0").package_id(), }, ResolvedFileSystemMapping { mount_path: PathBuf::from("/root"), + original_path: "/root".to_string(), volume_name: "lib".to_string(), package: builder.get("root", "1.0.0").package_id(), } @@ -1122,7 +1139,7 @@ mod tests { builder .register("root", "1.0.0") .with_dependency("dep", "=1.0.0") - .with_fs_mapping_from_dependency("dep-volume", "/root", "dep"); + .with_fs_mapping_from_dependency("dep-volume", "/root", "/root", "dep"); builder.register("dep", "1.0.0"); let mut dep_builder = builder.start_dependency_graph(); dep_builder @@ -1137,6 +1154,7 @@ mod tests { pkg.filesystem, vec![ResolvedFileSystemMapping { mount_path: PathBuf::from("/root"), + original_path: "/root".to_string(), volume_name: "dep-volume".to_string(), package: builder.get("dep", "1.0.0").package_id(), }] From d0bf95f2f6e49a91f2e1e6ae4d49708fb7cf24eb Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 8 Jun 2023 21:35:04 +0800 Subject: [PATCH 12/16] Fixed a compile error in tests --- .../src/runtime/resolver/in_memory_source.rs | 1 + lib/wasix/src/runtime/resolver/resolve.rs | 18 +++++++++--------- lib/wasix/src/runtime/resolver/wapm_source.rs | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/wasix/src/runtime/resolver/in_memory_source.rs b/lib/wasix/src/runtime/resolver/in_memory_source.rs index de3db68afa2..35456220aa5 100644 --- a/lib/wasix/src/runtime/resolver/in_memory_source.rs +++ b/lib/wasix/src/runtime/resolver/in_memory_source.rs @@ -171,6 +171,7 @@ mod tests { filesystem: vec![FileSystemMapping { volume_name: "atom".to_string(), mount_path: "/".to_string(), + original_path: "/".to_string(), dependency_name: None, }], }, diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/wasix/src/runtime/resolver/resolve.rs index f2b4c938ff1..f4b79235d48 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/wasix/src/runtime/resolver/resolve.rs @@ -1061,7 +1061,7 @@ mod tests { let mut builder = RegistryBuilder::new(); builder .register("root", "1.0.0") - .with_fs_mapping("lib", "/lib", "/other/lib"); + .with_fs_mapping("atom", "/publisher/lib", "/lib"); let mut dep_builder = builder.start_dependency_graph(); dep_builder.insert("root", "1.0.0"); let graph = dep_builder.graph("root", "1.0.0"); @@ -1072,8 +1072,8 @@ mod tests { pkg.filesystem, vec![ResolvedFileSystemMapping { mount_path: PathBuf::from("/lib"), - original_path: "/other/lib".to_string(), - volume_name: "lib".to_string(), + original_path: "/publisher/lib".to_string(), + volume_name: "atom".to_string(), package: builder.get("root", "1.0.0").package_id(), }] ); @@ -1086,14 +1086,14 @@ mod tests { .register("root", "1.0.0") .with_dependency("first", "=1.0.0") .with_dependency("second", "=1.0.0") - .with_fs_mapping("lib", "/root", "/root"); + .with_fs_mapping("atom", "/root", "/root"); builder.register("first", "1.0.0").with_fs_mapping( - "main", + "atom", "/usr/local/lib/first", "/usr/local/lib/first", ); builder.register("second", "1.0.0").with_fs_mapping( - "main", + "atom", "/usr/local/lib/second", "/usr/local/lib/second", ); @@ -1113,20 +1113,20 @@ mod tests { vec![ ResolvedFileSystemMapping { mount_path: PathBuf::from("/usr/local/lib/first"), - volume_name: "main".to_string(), + volume_name: "atom".to_string(), original_path: "/usr/local/lib/first".to_string(), package: builder.get("first", "1.0.0").package_id(), }, ResolvedFileSystemMapping { mount_path: PathBuf::from("/usr/local/lib/second"), original_path: "/usr/local/lib/second".to_string(), - volume_name: "main".to_string(), + volume_name: "atom".to_string(), package: builder.get("second", "1.0.0").package_id(), }, ResolvedFileSystemMapping { mount_path: PathBuf::from("/root"), original_path: "/root".to_string(), - volume_name: "lib".to_string(), + volume_name: "atom".to_string(), package: builder.get("root", "1.0.0").package_id(), } ] diff --git a/lib/wasix/src/runtime/resolver/wapm_source.rs b/lib/wasix/src/runtime/resolver/wapm_source.rs index 9bb3ac8a05e..47b90f5b054 100644 --- a/lib/wasix/src/runtime/resolver/wapm_source.rs +++ b/lib/wasix/src/runtime/resolver/wapm_source.rs @@ -277,6 +277,7 @@ mod tests { filesystem: vec![FileSystemMapping { volume_name: "atom".to_string(), mount_path: "/".to_string(), + original_path: "/".to_string(), dependency_name: None, }], }, From 683c039d5ba77af66596b1307f3a277cb233f5a2 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 8 Jun 2023 23:53:04 +0800 Subject: [PATCH 13/16] Most tests work again --- Cargo.lock | 411 +++++++++--------- Cargo.toml | 5 + lib/cli/Cargo.toml | 6 +- lib/cli/src/commands/run.rs | 6 +- lib/wasix/Cargo.toml | 4 +- lib/wasix/src/bin_factory/binary_package.rs | 15 +- .../package_loader/load_package_tree.rs | 2 +- lib/wasix/src/runtime/resolver/resolve.rs | 3 - 8 files changed, 225 insertions(+), 227 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b6d0edfa98..789032382ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,13 +41,19 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -213,7 +219,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -253,7 +259,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "miniz_oxide 0.6.2", - "object 0.30.3", + "object 0.30.4", "rustc-demangle", ] @@ -458,9 +464,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cbindgen" -version = "0.24.3" +version = "0.24.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6358dedf60f4d9b8db43ad187391afe959746101346fe51bb978126bec61dfb" +checksum = "4b922faaf31122819ec80c4047cc684c6979a087366c069611e33649bf98e18d" dependencies = [ "clap 3.2.25", "heck 0.4.1", @@ -498,13 +504,13 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.24" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "wasm-bindgen", "winapi", @@ -548,9 +554,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.0" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" +checksum = "401a4694d2bf92537b6867d94de48c4842089645fdcdf6c71865b175d836e9c2" dependencies = [ "clap_builder", "clap_derive", @@ -563,15 +569,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eef05769009513df2eb1c3b4613e7fad873a14c600ff025b08f250f59fee7de" dependencies = [ - "clap 4.3.0", + "clap 4.3.2", "log", ] [[package]] name = "clap_builder" -version = "4.3.0" +version = "4.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" +checksum = "72394f3339a76daf211e57d4bcb374410f3965dcc606dd0e03738c7888766980" dependencies = [ "anstream", "anstyle", @@ -582,14 +588,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.0" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -650,9 +656,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "6.1.4" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7b787b0dc42e8111badfdbe4c3059158ccb2db8780352fa1b01e8ccf45cc4d" +checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba" dependencies = [ "crossterm", "strum", @@ -971,9 +977,9 @@ dependencies = [ [[package]] name = "crossterm" -version = "0.25.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" +checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" dependencies = [ "bitflags", "crossterm_winapi", @@ -1006,9 +1012,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ "csv-core", "itoa", @@ -1125,7 +1131,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -1147,7 +1153,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core 0.20.1", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -1160,7 +1166,7 @@ dependencies = [ "hashbrown 0.12.3", "lock_api", "once_cell", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.8", ] [[package]] @@ -1176,13 +1182,13 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cdeb9ec472d588e539a818b2dee436825730da08ad0017c4b1a17676bdc8b7" +checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.18", ] [[package]] @@ -1275,9 +1281,9 @@ checksum = "6d9d8664cf849d7d0f3114a3a387d2f5e4303176d746d5a951aaddc66dfe9240" [[package]] name = "dlib" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1b7517328c04c2aa68422fc60a41b92208182142ed04a25879c26c8f878794" +checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" dependencies = [ "libloading", ] @@ -1385,7 +1391,7 @@ dependencies = [ "darling 0.20.1", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -1451,11 +1457,11 @@ dependencies = [ [[package]] name = "field-offset" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3cf3a800ff6e860c863ca6d4b16fd999db8b752819c1606884047b73e468535" +checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" dependencies = [ - "memoffset 0.8.0", + "memoffset 0.9.0", "rustc_version 0.4.0", ] @@ -1519,9 +1525,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -1594,7 +1600,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -1657,9 +1663,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1676,7 +1682,7 @@ checksum = "e77ac7b51b8e6313251737fcef4b1c01a2ea102bde68415b62c0ee9268fec357" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -2019,9 +2025,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2065,9 +2071,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -2086,13 +2092,14 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.17.3" +version = "0.17.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" +checksum = "8ff8cc23a7393a397ed1d7f56e6365cba772aba9f9912ab968b03043c395d057" dependencies = [ "console", + "instant", "number_prefix", - "portable-atomic 0.3.20", + "portable-atomic", "unicode-width", ] @@ -2315,9 +2322,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.144" +version = "0.2.146" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "libfuzzer-sys" @@ -2332,12 +2339,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "d580318f95776505201b28cf98eb1fa5e4be3b689633ba6a3e6cd880ff22d8cb" dependencies = [ "cfg-if 1.0.0", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -2374,9 +2381,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "llvm-sys" -version = "140.1.0" +version = "140.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a867ea93540d92ffc0edb127c9d1c20e29c64222f2e52259e0d8f32960f577" +checksum = "fe48b87b95ad1b3eeca563010aadbb427b3d3e89c1f78fe21acd39846be5c7d4" dependencies = [ "cc", "lazy_static", @@ -2387,9 +2394,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -2397,12 +2404,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if 1.0.0", -] +checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" [[package]] name = "lzma-rs" @@ -2511,6 +2515,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2561,9 +2574,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "minisign" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23ef13ff1d745b1e52397daaa247e333c607f3cff96d4df2b798dc252db974b" +checksum = "d2b6f58413c6cee060115673578e47271838f3c87cb9322c61a3bcd6d740b7d2" dependencies = [ "getrandom", "rpassword", @@ -2590,14 +2603,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", "wasi", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -2702,16 +2715,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6842d8099b88d19a64158a6cfdc3e9ad82c738c041dab98280ef7ba98d64fa" -[[package]] -name = "num-integer" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.15" @@ -2772,9 +2775,9 @@ dependencies = [ [[package]] name = "object" -version = "0.30.3" +version = "0.30.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" dependencies = [ "flate2", "memchr", @@ -2782,9 +2785,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "oorandom" @@ -2794,9 +2797,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.52" +version = "0.10.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +checksum = "69b3f656a17a6cbc115b5c7a40c616947d213ba182135b014d6051b73ab6f019" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -2815,7 +2818,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -2826,9 +2829,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.87" +version = "0.9.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +checksum = "c2ce0f250f34a308dcfdbb351f511359857d4ed2134ba715a4eadd46e1ffd617" dependencies = [ "cc", "libc", @@ -2887,7 +2890,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.7", + "parking_lot_core 0.9.8", ] [[package]] @@ -2906,15 +2909,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.45.0", + "windows-targets 0.48.0", ] [[package]] @@ -2947,9 +2950,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" @@ -2981,7 +2984,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -3022,7 +3025,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -3073,18 +3076,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e30165d31df606f5726b090ec7592c308a0eaf61721ff64c9a3018e344a8753e" -dependencies = [ - "portable-atomic 1.3.2", -] - -[[package]] -name = "portable-atomic" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc59d1bcc64fc5d021d67521f818db868368028108d37f0e98d74e33f68297b5" +checksum = "767eb9f07d4a5ebcb39bbf2d452058a93c011373abf6832e24194a1c3f004794" [[package]] name = "ppv-lite86" @@ -3202,9 +3196,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ "unicode-ident", ] @@ -3266,9 +3260,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -3395,9 +3389,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81ca098a9821bd52d6b24fd8b10bd081f47d39c22778cafaa75a2857a62c6390" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", @@ -3865,9 +3859,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.163" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] @@ -3904,13 +3898,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -4038,9 +4032,9 @@ dependencies = [ [[package]] name = "shared-buffer" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a4987984c21cbd48a683c00975a0a9652d1882e8e986c42223f65440f4668f" +checksum = "2cf61602ee61e2f83dd016b3e6387245291cf728ea071c378b35088125b4d995" dependencies = [ "bytes", "memmap2 0.6.2", @@ -4278,9 +4272,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.16" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -4318,15 +4312,16 @@ checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" [[package]] name = "tempfile" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if 1.0.0", "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -4438,7 +4433,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -4468,9 +4463,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" +checksum = "ea9e1b3cf1243ae005d9e74085d4d542f3125458f3a81af210d901dcd7411efd" dependencies = [ "itoa", "serde", @@ -4557,9 +4552,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.28.1" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -4581,7 +4576,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -4730,7 +4725,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -4949,12 +4944,12 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", - "idna 0.3.0", + "idna 0.4.0", "percent-encoding", "serde", ] @@ -4983,9 +4978,9 @@ dependencies = [ [[package]] name = "validator" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f07b0a1390e01c0fc35ebb26b28ced33c9a3808f7f9fbe94d3cc01e233bfeed5" +checksum = "32ad5bf234c7d3ad1042e5252b7eddb2c4669ee23f32c7dd0e9b7705f07ef591" dependencies = [ "idna 0.2.3", "lazy_static", @@ -5205,9 +5200,9 @@ dependencies = [ [[package]] name = "wapm-targz-to-pirita" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac443765a5a1866a73b845afb3261d1ea8138f884603047d56e3e431fe37e34" +checksum = "7cedbf3ceb9070e8237e822134cc1c0162dd7b31cbd1c4ca586fb4d6be33f51b" dependencies = [ "anyhow", "base64", @@ -5221,31 +5216,15 @@ dependencies = [ "serde_json", "sha2", "tar", - "toml 0.5.11", + "toml 0.7.4", "tracing", "url", "validator", - "wapm-toml", - "wasmer-registry 4.5.0", + "wasmer-registry 4.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmer-toml", "webc", ] -[[package]] -name = "wapm-toml" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "994ef26447f3158955d2e3fca96021d1f1c47b830e2053569177673dca1447a9" -dependencies = [ - "anyhow", - "semver 1.0.17", - "serde", - "serde_cbor", - "serde_json", - "serde_yaml 0.9.21", - "thiserror", - "toml 0.5.11", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -5388,9 +5367,9 @@ dependencies = [ [[package]] name = "wasm-coredump-builder" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eebd6aa4d9af1aec6368d253af21687b7c42cd4cae188577d89b877633e28b5" +checksum = "bcb7050c67c77c4fe0c1197565c2e24ad6e5f66318f32dcbbe8f07b00d8d1502" dependencies = [ "wasm-coredump-encoder", "wasm-coredump-types", @@ -5399,9 +5378,9 @@ dependencies = [ [[package]] name = "wasm-coredump-encoder" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0130624de91e13a6be21fc8595f7351425e365641d9664ec79d659e1bd2a8eb6" +checksum = "6b78901e352d547edcaa8e5ee9e77bcdf354f7b67c327225839243c6c24c6747" dependencies = [ "leb128", "wasm-coredump-types", @@ -5409,9 +5388,9 @@ dependencies = [ [[package]] name = "wasm-coredump-types" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cbea8ddb142d1899854b060b57c79c2b9576b845ff32e9311b624fe0aad8b8f" +checksum = "6b3ba1c7b0d578f4733643ec18f35bd81a57197debb1d35be631787630ae615b" [[package]] name = "wasm-encoder" @@ -5433,9 +5412,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c94f464d50e31da425794a02da1a82d4b96a657dcb152a6664e8aa915be517" +checksum = "18c41dbd92eaebf3612a39be316540b8377c871cb9bde6b064af962984912881" dependencies = [ "leb128", ] @@ -5514,10 +5493,10 @@ dependencies = [ "reqwest", "serde", "serde_json", - "time 0.3.21", + "time 0.3.22", "tracing", "url", - "wasmer-deploy-schema", + "wasmer-deploy-schema 0.0.3", "webc", ] @@ -5611,7 +5590,7 @@ dependencies = [ "cargo_metadata", "cfg-if 1.0.0", "chrono", - "clap 4.3.0", + "clap 4.3.2", "clap-verbosity-flag", "colored 2.0.0", "dialoguer", @@ -5623,7 +5602,7 @@ dependencies = [ "isatty", "libc", "log", - "object 0.30.3", + "object 0.30.4", "once_cell", "pathdiff", "prettytable-rs", @@ -5702,7 +5681,7 @@ dependencies = [ "atty", "bytesize", "cfg-if 1.0.0", - "clap 4.3.0", + "clap 4.3.2", "colored 2.0.0", "distance", "fern", @@ -5777,12 +5756,12 @@ dependencies = [ [[package]] name = "wasmer-deploy-cli" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf454c05ebc34014b4a76dea8fc54ba80feff850fc10996f4d28bc02e19044d" +checksum = "6a0dbfd10317322e230aff6caf7ace1479fc5cb839ebd297b6bedb7e30c7130a" dependencies = [ "anyhow", - "clap 4.3.0", + "clap 4.3.2", "clap-verbosity-flag", "comfy-table", "dialoguer", @@ -5796,7 +5775,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.8.26", - "time 0.3.21", + "time 0.3.22", "tokio", "toml 0.7.4", "tracing", @@ -5804,9 +5783,9 @@ dependencies = [ "url", "uuid", "wasmer-api", - "wasmer-deploy-schema", + "wasmer-deploy-schema 0.0.4", "wasmer-deploy-util", - "wasmer-registry 4.5.0", + "wasmer-registry 4.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "wasmer-toml", "webc", ] @@ -5821,7 +5800,23 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", - "time 0.3.21", + "time 0.3.22", + "url", + "uuid", + "wcgi-host", +] + +[[package]] +name = "wasmer-deploy-schema" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08851e586b75833e53162b7bdef8ec0e04f0ed04e2623d6d1846d5ca92d42e37" +dependencies = [ + "bytesize", + "serde", + "serde_json", + "serde_path_to_error", + "time 0.3.22", "url", "uuid", "wcgi-host", @@ -5834,7 +5829,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed7ea0a8f5f5efcddba21e4ea251d90579309deef1cfa013a52a11421cfd635" dependencies = [ "serde", - "wasmer-deploy-schema", + "wasmer-deploy-schema 0.0.3", "wasmparser 0.95.0", ] @@ -5905,7 +5900,7 @@ dependencies = [ "hex", "insta", "md5", - "object 0.30.3", + "object 0.30.4", "once_cell", "predicates 2.1.5", "pretty_assertions", @@ -5943,16 +5938,13 @@ dependencies = [ [[package]] name = "wasmer-registry" -version = "4.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "734862f2b8d92b3640e7d2a487c2d59857c53854a3312eacf5df786655aae819" +version = "4.7.0" dependencies = [ "anyhow", "console", "dirs", "filetime", "flate2", - "fs_extra", "futures-util", "graphql_client", "hex", @@ -5962,6 +5954,7 @@ dependencies = [ "log", "lzma-rs", "minisign", + "pretty_assertions", "regex", "reqwest", "rpassword", @@ -5972,27 +5965,29 @@ dependencies = [ "tar", "tempfile", "thiserror", - "time 0.3.21", + "time 0.3.22", "tldextract", "tokio", "toml 0.5.11", "url", "wasmer-toml", - "wasmer-wasm-interface 3.3.0", + "wasmer-wasm-interface 4.0.0-beta.1", "wasmparser 0.51.4", - "webc", "whoami", ] [[package]] name = "wasmer-registry" version = "4.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc018188be9bb86e1e548ad3e5eedc33e6a0fca703172b1dd4a605d35e82086" dependencies = [ "anyhow", "console", "dirs", "filetime", "flate2", + "fs_extra", "futures-util", "graphql_client", "hex", @@ -6002,7 +5997,6 @@ dependencies = [ "log", "lzma-rs", "minisign", - "pretty_assertions", "regex", "reqwest", "rpassword", @@ -6013,14 +6007,15 @@ dependencies = [ "tar", "tempfile", "thiserror", - "time 0.3.21", + "time 0.3.22", "tldextract", "tokio", "toml 0.5.11", "url", "wasmer-toml", - "wasmer-wasm-interface 4.0.0-beta.1", + "wasmer-wasm-interface 4.0.0-beta.1 (registry+https://github.com/rust-lang/crates.io-index)", "wasmparser 0.51.4", + "webc", "whoami", ] @@ -6205,26 +6200,26 @@ dependencies = [ [[package]] name = "wasmer-wasm-interface" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b31bad82289ef8ca59b098682c562f03a3b1bcd02e2110e4cce64bece47dbfba" +version = "4.0.0-beta.1" dependencies = [ + "bincode", "either", "nom 5.1.3", "serde", "wasmparser 0.51.4", + "wat", ] [[package]] name = "wasmer-wasm-interface" version = "4.0.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beebc2fea7a6f5a5b2ed15b2f437d3923657f6a95dc9c57bdf8ad48b5a310233" dependencies = [ - "bincode", "either", "nom 5.1.3", "serde", "wasmparser 0.51.4", - "wat", ] [[package]] @@ -6298,22 +6293,22 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.106.0" +version = "0.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d014e33793cab91655fa6349b0bc974984de106b2e0f6b0dfe6f6594b260624d" +checksum = "29e3ac9b780c7dda0cac7a52a5d6d2d6707cc6e3451c9db209b6c758f40d7acb" dependencies = [ "indexmap", - "url", + "semver 1.0.17", ] [[package]] name = "wasmprinter" -version = "0.2.58" +version = "0.2.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c520299a0b5999adef4f063add9689e4559b3e4eb2688dbd63cc36ebb595841" +checksum = "cc960b30b84abca377768f3c62cff3a1c74db8c0f6759ed581827da0bd3a3fed" dependencies = [ "anyhow", - "wasmparser 0.106.0", + "wasmparser 0.107.0", ] [[package]] @@ -6336,23 +6331,23 @@ dependencies = [ [[package]] name = "wast" -version = "59.0.0" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38462178c91e3f990df95f12bf48abe36018e03550a58a65c53975f4e704fc35" +checksum = "bd06cc744b536e30387e72a48fdd492105b9c938bb4f415c39c616a7a0a697ad" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder 0.28.0", + "wasm-encoder 0.29.0", ] [[package]] name = "wat" -version = "1.0.65" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936a025be0417a94d6e9bf92bfdf9e06dbf63debf187b650d9c73a5add701f1" +checksum = "5abe520f0ab205366e9ac7d3e6b2fc71de44e32a2b58f2ec871b6b575bdcea3b" dependencies = [ - "wast 59.0.0", + "wast 60.0.0", ] [[package]] @@ -6813,9 +6808,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d8f380ae16a37b30e6a2cf67040608071384b1450c189e61bea3ff57cde922d" +checksum = "52839dc911083a8ef63efa4d039d1f58b5e409f923e44c80828f206f66e5541c" [[package]] name = "yaml-rust" diff --git a/Cargo.toml b/Cargo.toml index e63e6db2b24..61dcde6f238 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,11 @@ repository = "https://github.com/wasmerio/wasmer" rust-version = "1.67" version = "4.0.0-beta.1" +[workspace.dependencies] +webc = { version = "5.0.4", default-features = false } +wapm-targz-to-pirita = "0.3.0" +wasmer-toml = "0.6.0" + [build-dependencies] test-generator = { path = "tests/lib/test-generator" } build-deps = "0.1.4" diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 9d16b3126c4..8aa9563c366 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -45,8 +45,8 @@ virtual-fs = { version = "0.4.0", path = "../virtual-fs", default-features = fa virtual-net = { version = "0.3.0", path = "../virtual-net" } # Wasmer-owned dependencies. -webc = { version = "5.0" } -wapm-targz-to-pirita = "0.2.1" +webc = { workspace = true } +wapm-targz-to-pirita = { workspace = true } wasmer-deploy-cli = { version = "0.1.9", default-features = false } # Third-party dependencies. @@ -67,7 +67,7 @@ dirs = { version = "4.0" } serde_json = { version = "1.0" } target-lexicon = { version = "0.12", features = ["std"] } prettytable-rs = "0.10.0" -wasmer-toml = "0.6.0" +wasmer-toml = { workspace = true } indexmap = "1.9.2" walkdir = "2.3.2" regex = "1.6.0" diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 1faea65462a..85505b18da1 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -22,7 +22,7 @@ use sha2::{Digest, Sha256}; use tempfile::NamedTempFile; use tokio::runtime::Handle; use url::Url; -use wapm_targz_to_pirita::{FileMap, TransformManifestFunctions}; +use wapm_targz_to_pirita::{webc::v1::DirOrFile, FileMap, TransformManifestFunctions}; use wasmer::{ DeserializeError, Engine, Function, Imports, Instance, Module, Store, Type, TypedFunction, Value, @@ -44,7 +44,7 @@ use wasmer_wasix::{ }, Runtime, }; -use webc::{metadata::Manifest, v1::DirOrFile, Container}; +use webc::{metadata::Manifest, Container}; use crate::{commands::run::wasi::Wasi, error::PrettyError, store::StoreOptions}; @@ -610,7 +610,7 @@ fn construct_webc_in_memory(dir: &Path) -> Result, Error> { } let functions = TransformManifestFunctions::default(); - let webc = wapm_targz_to_pirita::generate_webc_file(files, dir, None, &functions)?; + let webc = wapm_targz_to_pirita::generate_webc_file(files, dir, &functions)?; Ok(webc) } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 8868a845edf..6d74be955d4 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -29,7 +29,7 @@ bincode = { version = "1.3" } chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" -webc = { version = "5.0.4", default-features = false } +webc = { workspace = true } serde_cbor = { version = "0.11.2" } anyhow = { version = "1.0.66" } lazy_static = "1.4" @@ -97,7 +97,7 @@ wasm-bindgen = ">= 0.2.74, < 0.2.85" wasmer = { path = "../api", version = "=4.0.0-beta.1", default-features = false, features = ["wat", "js-serializable-module"] } tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } pretty_assertions = "1.3.0" -wapm-targz-to-pirita = "0.2.1" +wapm-targz-to-pirita = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = "0.3.0" diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 235466a7a33..47528d5c629 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -143,6 +143,7 @@ mod tests { use tempfile::TempDir; use virtual_fs::AsyncReadExt; + use wapm_targz_to_pirita::{webc::v1::DirOrFile, FileMap, TransformManifestFunctions}; use crate::{runtime::task_manager::tokio::TokioTaskManager, PluggableRuntime}; @@ -189,22 +190,22 @@ mod tests { let mut files = BTreeMap::new(); load_files_from_disk(&mut files, dir, dir); - let wasmer_toml = webc::v1::DirOrFile::File("wasmer.toml".into()); + let wasmer_toml = DirOrFile::File("wasmer.toml".into()); if let Some(toml_data) = files.remove(&wasmer_toml) { // HACK(Michael-F-Bryan): The version of wapm-targz-to-pirita we are // using doesn't know we renamed "wapm.toml" to "wasmer.toml", so we // manually patch things up if people have already migrated their // projects. files - .entry(webc::v1::DirOrFile::File("wapm.toml".into())) + .entry(DirOrFile::File("wapm.toml".into())) .or_insert(toml_data); } - let functions = wapm_targz_to_pirita::TransformManifestFunctions::default(); - wapm_targz_to_pirita::generate_webc_file(files, dir, None, &functions).unwrap() + let functions = TransformManifestFunctions::default(); + wapm_targz_to_pirita::generate_webc_file(files, dir, &functions).unwrap() } - fn load_files_from_disk(files: &mut wapm_targz_to_pirita::FileMap, dir: &Path, base: &Path) { + fn load_files_from_disk(files: &mut FileMap, dir: &Path, base: &Path) { let entries = dir.read_dir().unwrap(); for entry in entries { @@ -213,10 +214,10 @@ mod tests { if path.is_dir() { load_files_from_disk(files, &path, base); - files.insert(webc::v1::DirOrFile::Dir(relative_path), Vec::new()); + files.insert(DirOrFile::Dir(relative_path), Vec::new()); } else if path.is_file() { let data = std::fs::read(&path).unwrap(); - files.insert(webc::v1::DirOrFile::File(relative_path), data); + files.insert(DirOrFile::File(relative_path), data); } } } diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 5feefdae4bb..7fa9c105b54 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -282,7 +282,7 @@ fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 { /// /// The TmpFileSystem *does* allow mounting at non-root paths, however it can't /// handle nested paths (e.g. mounting to "/lib" and "/lib/python3.10" - see -/// https://github.com/wasmerio/wasmer/issues/3678 for more) and you aren't +/// for more) and you aren't /// allowed to mount to "/" because it's a special directory that already /// exists. /// diff --git a/lib/wasix/src/runtime/resolver/resolve.rs b/lib/wasix/src/runtime/resolver/resolve.rs index f4b79235d48..c3e9776da16 100644 --- a/lib/wasix/src/runtime/resolver/resolve.rs +++ b/lib/wasix/src/runtime/resolver/resolve.rs @@ -110,7 +110,6 @@ async fn discover_dependencies( to_visit.push_back(root_index); while let Some(index) = to_visit.pop_front() { - eprintln!("Checking {}", graph[index].id); let mut to_add = Vec::new(); for dep in &graph[index].pkg.dependencies { @@ -139,7 +138,6 @@ async fn discover_dependencies( for (alias, node) in to_add { let dep_id = node.id.clone(); - eprintln!("{} -> {} (as {alias})", &graph[index].id, dep_id); let dep_index = match nodes.get(&dep_id) { Some(&ix) => ix, @@ -147,7 +145,6 @@ async fn discover_dependencies( // Create a new node and schedule its dependencies to be // retrieved let ix = graph.add_node(node); - eprintln!("Scheduling to check \"{dep_id}\""); nodes.insert(dep_id, ix); to_visit.push_back(ix); ix From 4d9248cf800bb46663edeb0c02a90c7084374ef5 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 9 Jun 2023 00:48:11 +0800 Subject: [PATCH 14/16] Pick up the fix for wasmerio/pirita#134 --- Cargo.lock | 5 ++--- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 789032382ac..3dd5b02fa6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5200,9 +5200,9 @@ dependencies = [ [[package]] name = "wapm-targz-to-pirita" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cedbf3ceb9070e8237e822134cc1c0162dd7b31cbd1c4ca586fb4d6be33f51b" +checksum = "8d816eaf89496a0eefecb79f43463b07205701165c17790fa6f0eab5ae72bc39" dependencies = [ "anyhow", "base64", @@ -5220,7 +5220,6 @@ dependencies = [ "tracing", "url", "validator", - "wasmer-registry 4.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "wasmer-toml", "webc", ] diff --git a/Cargo.toml b/Cargo.toml index 61dcde6f238..093ee2df7c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,7 @@ version = "4.0.0-beta.1" [workspace.dependencies] webc = { version = "5.0.4", default-features = false } -wapm-targz-to-pirita = "0.3.0" +wapm-targz-to-pirita = "0.3.1" wasmer-toml = "0.6.0" [build-dependencies] From 47d031a39c08725f6a9b2ef1dc6a217c48fbfdc5 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 9 Jun 2023 01:04:30 +0800 Subject: [PATCH 15/16] Made "make test-js" work again --- lib/wasix/src/bin_factory/binary_package.rs | 20 +++++++++++++++++--- lib/wasix/src/runtime/resolver/utils.rs | 4 +++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 47528d5c629..6aec1baf771 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -145,11 +145,25 @@ mod tests { use virtual_fs::AsyncReadExt; use wapm_targz_to_pirita::{webc::v1::DirOrFile, FileMap, TransformManifestFunctions}; - use crate::{runtime::task_manager::tokio::TokioTaskManager, PluggableRuntime}; + use crate::{runtime::task_manager::VirtualTaskManager, PluggableRuntime}; use super::*; + fn task_manager() -> Arc { + cfg_if::cfg_if! { + if #[cfg(feature = "sys-threads")] { + Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::new(tokio::runtime::Handle::current())) + } else { + unimplemented!("Unable to get the task manager") + } + } + } + #[tokio::test] + #[cfg_attr( + feature = "sys-threads", + ignore = "The tokio task manager isn't available on this platform" + )] async fn fs_table_can_map_directories_to_different_names() { let temp = TempDir::new().unwrap(); let wasmer_toml = r#" @@ -168,8 +182,8 @@ mod tests { std::fs::write(out.join("file.txt"), file_txt).unwrap(); let webc = construct_webc_in_memory(temp.path()); let webc = Container::from_bytes(webc).unwrap(); - let tasks = TokioTaskManager::new(tokio::runtime::Handle::current()); - let runtime = PluggableRuntime::new(Arc::new(tasks)); + let tasks = task_manager(); + let runtime = PluggableRuntime::new(tasks); let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap(); diff --git a/lib/wasix/src/runtime/resolver/utils.rs b/lib/wasix/src/runtime/resolver/utils.rs index 03c018cc5f4..34cc5b5d72b 100644 --- a/lib/wasix/src/runtime/resolver/utils.rs +++ b/lib/wasix/src/runtime/resolver/utils.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use anyhow::{Context, Error}; +use anyhow::Error; use http::{HeaderMap, StatusCode}; use url::Url; @@ -62,6 +62,8 @@ pub(crate) fn file_path_from_url(url: &Url) -> Result { // Note: The Url::to_file_path() method is platform-specific cfg_if::cfg_if! { if #[cfg(any(unix, windows, target_os = "redox", target_os = "wasi"))] { + use anyhow::Context; + if let Ok(path) = url.to_file_path() { return Ok(path); } From 9a4c1733e1408aac6f866e8a79edbc0350d6e2a1 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 9 Jun 2023 02:32:48 +0800 Subject: [PATCH 16/16] The cfg_attr logic was around the wrong way --- lib/wasix/src/bin_factory/binary_package.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 6aec1baf771..f2f10508277 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -161,7 +161,7 @@ mod tests { #[tokio::test] #[cfg_attr( - feature = "sys-threads", + not(feature = "sys-threads"), ignore = "The tokio task manager isn't available on this platform" )] async fn fs_table_can_map_directories_to_different_names() {