Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Mount [fs] entries as additional mapped directories when running a lo… #4982

Merged
merged 5 commits into from
Aug 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions lib/cli/src/commands/auth/login/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,9 @@ impl Login {
}
#[cfg(test)]
{
// prevent unused binding warning
_ = user;

false
}
} else {
Expand Down
13 changes: 11 additions & 2 deletions lib/cli/src/commands/run/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ impl Run {
}

#[tracing::instrument(level = "debug", name = "wasmer_run", skip_all)]
fn execute_inner(self, output: Output) -> Result<(), Error> {
fn execute_inner(mut self, output: Output) -> Result<(), Error> {
let pb = ProgressBar::new_spinner();
pb.set_draw_target(output.draw_target());
pb.enable_steady_tick(TICK);
Expand Down Expand Up @@ -157,6 +157,12 @@ impl Run {

let target = self.input.resolve_target(&monitoring_runtime, &pb)?;

if let ExecutableTarget::Package(ref pkg) = target {
self.wasi
.mapped_dirs
.extend(pkg.additional_host_mapped_directories.clone());
}

pb.finish_and_clear();

// push the TTY state so we can restore it after the program finishes
Expand Down Expand Up @@ -1005,7 +1011,10 @@ impl wasmer_wasix::runtime::package_loader::PackageLoader for MonitoringPackageL
&self,
root: &Container,
resolution: &wasmer_wasix::runtime::resolver::Resolution,
root_is_local_dir: bool,
) -> Result<BinaryPackage, Error> {
self.inner.load_package_tree(root, resolution).await
self.inner
.load_package_tree(root, resolution, root_is_local_dir)
.await
}
}
1 change: 1 addition & 0 deletions lib/wasix/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ web-sys = { version = "0.3.64", features = [
ahash = "0.8.11"
hyper-util = { version = "0.1.5", features = ["server", "server-graceful", "tokio", "service", "client"], optional = true }
http-body-util = { version="0.1.1", optional = true }
toml = "0.8"

[target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest]
workspace = true
Expand Down
38 changes: 32 additions & 6 deletions lib/wasix/src/bin_factory/binary_package.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use wasmer_config::package::{PackageHash, PackageId, PackageSource};
use webc::{compat::SharedBytes, Container};

use crate::{
runners::MappedDirectory,
runtime::resolver::{PackageInfo, ResolveError},
Runtime,
};
Expand Down Expand Up @@ -75,6 +76,8 @@ pub struct BinaryPackage {
pub commands: Vec<BinaryPackageCommand>,
pub uses: Vec<String>,
pub file_system_memory_footprint: u64,

pub additional_host_mapped_directories: Vec<MappedDirectory>,
}

impl BinaryPackage {
Expand All @@ -91,20 +94,38 @@ impl BinaryPackage {
let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash));

let manifest_path = dir.join("wasmer.toml");
let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?;
let webc = webc::wasmer_package::Package::from_manifest(&manifest_path)?;
let container = Container::from(webc);
let manifest = container.manifest();

let root = PackageInfo::from_manifest(id, manifest, container.version())?;
let root_id = root.id.clone();

let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?;
let pkg = rt
let mut pkg = rt
.package_loader()
.load_package_tree(&container, &resolution)
.load_package_tree(&container, &resolution, true)
.await
.map_err(|e| anyhow::anyhow!(e))?;

// HACK: webc has no way to return its deserialized manifest to us, so we need to do it again here
// We already read and parsed the manifest once, so it'll succeed again. Unwrapping is safe at this point.
let wasmer_toml = std::fs::read_to_string(&manifest_path).unwrap();
let wasmer_toml: wasmer_config::package::Manifest = toml::from_str(&wasmer_toml).unwrap();
pkg.additional_host_mapped_directories.extend(
wasmer_toml
.fs
.into_iter()
.map(|(guest, host)| {
anyhow::Ok(MappedDirectory {
host: dir.join(host).canonicalize()?,
guest,
})
})
.collect::<Result<Vec<_>, _>>()?
.into_iter(),
);

Ok(pkg)
}

Expand Down Expand Up @@ -132,7 +153,7 @@ impl BinaryPackage {
let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?;
let pkg = rt
.package_loader()
.load_package_tree(container, &resolution)
.load_package_tree(container, &resolution, false)
.await
.map_err(|e| anyhow::anyhow!(e))?;

Expand Down Expand Up @@ -162,7 +183,7 @@ impl BinaryPackage {
.context("Dependency resolution failed")?;
let pkg = runtime
.package_loader()
.load_package_tree(&root, &resolution)
.load_package_tree(&root, &resolution, false)
.await
.map_err(|e| anyhow::anyhow!(e))?;

Expand Down Expand Up @@ -255,7 +276,12 @@ mod tests {
.with_shared_http_client(runtime.http_client().unwrap().clone()),
);

let pkg = BinaryPackage::from_dir(temp.path(), &runtime)
let pkg = webc::wasmer_package::Package::from_manifest(&manifest).unwrap();
let data = pkg.serialize().unwrap();
let webc_path = temp.path().join("package.webc");
std::fs::write(&webc_path, data).unwrap();

let pkg = BinaryPackage::from_webc(&Container::from_disk(&webc_path).unwrap(), &runtime)
.await
.unwrap();

Expand Down
3 changes: 2 additions & 1 deletion lib/wasix/src/runtime/package_loader/builtin_loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -362,8 +362,9 @@ impl PackageLoader for BuiltinPackageLoader {
&self,
root: &Container,
resolution: &Resolution,
root_is_local_dir: bool,
) -> Result<BinaryPackage, Error> {
super::load_package_tree(root, self, resolution).await
super::load_package_tree(root, self, resolution, root_is_local_dir).await
}
}

Expand Down
20 changes: 17 additions & 3 deletions lib/wasix/src/runtime/package_loader/load_package_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,11 @@ pub async fn load_package_tree(
root: &Container,
loader: &dyn PackageLoader,
resolution: &Resolution,
root_is_local_dir: bool,
) -> Result<BinaryPackage, Error> {
let mut containers = fetch_dependencies(loader, &resolution.package, &resolution.graph).await?;
containers.insert(resolution.package.root_package.clone(), root.clone());
let fs = filesystem(&containers, &resolution.package)?;
let fs = filesystem(&containers, &resolution.package, root_is_local_dir)?;

let root = &resolution.package.root_package;
let commands: Vec<BinaryPackageCommand> =
Expand All @@ -61,6 +62,8 @@ pub async fn load_package_tree(
commands,
uses: Vec::new(),
file_system_memory_footprint,

additional_host_mapped_directories: vec![],
};

Ok(loaded)
Expand Down Expand Up @@ -316,6 +319,7 @@ fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 {
fn filesystem(
packages: &HashMap<PackageId, Container>,
pkg: &ResolvedPackage,
root_is_local_dir: bool,
) -> Result<Box<dyn FileSystem + Send + Sync>, Error> {
if pkg.filesystem.is_empty() {
return Ok(Box::new(OverlayFileSystem::<
Expand All @@ -342,16 +346,17 @@ fn filesystem(
}

if found_v3 && !found_v2 {
filesystem_v3(packages, pkg)
filesystem_v3(packages, pkg, root_is_local_dir)
} else {
filesystem_v2(packages, pkg)
filesystem_v2(packages, pkg, root_is_local_dir)
}
}

/// Build the filesystem for webc v3 packages.
fn filesystem_v3(
packages: &HashMap<PackageId, Container>,
pkg: &ResolvedPackage,
root_is_local_dir: bool,
) -> Result<Box<dyn FileSystem + Send + Sync>, Error> {
let mut volumes: HashMap<&PackageId, BTreeMap<String, Volume>> = HashMap::new();

Expand All @@ -367,6 +372,10 @@ fn filesystem_v3(
..
} in &pkg.filesystem
{
if *package == pkg.root_package && root_is_local_dir {
continue;
}

// Note: We want to reuse existing Volume instances if we can. That way
// we can keep the memory usage down. A webc::compat::Volume is
// reference-counted, anyway.
Expand Down Expand Up @@ -427,6 +436,7 @@ fn filesystem_v3(
fn filesystem_v2(
packages: &HashMap<PackageId, Container>,
pkg: &ResolvedPackage,
root_is_local_dir: bool,
) -> Result<Box<dyn FileSystem + Send + Sync>, Error> {
let mut filesystems = Vec::new();
let mut volumes: HashMap<&PackageId, BTreeMap<String, Volume>> = HashMap::new();
Expand All @@ -441,6 +451,10 @@ fn filesystem_v2(
original_path,
} in &pkg.filesystem
{
if *package == pkg.root_package && root_is_local_dir {
continue;
}

// Note: We want to reuse existing Volume instances if we can. That way
// we can keep the memory usage down. A webc::compat::Volume is
// reference-counted, anyway.
Expand Down
6 changes: 5 additions & 1 deletion lib/wasix/src/runtime/package_loader/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ pub trait PackageLoader: Send + Sync + Debug {
&self,
root: &Container,
resolution: &Resolution,
root_is_local_dir: bool,
) -> Result<BinaryPackage, Error>;
}

Expand All @@ -37,7 +38,10 @@ where
&self,
root: &Container,
resolution: &Resolution,
root_is_local_dir: bool,
) -> Result<BinaryPackage, Error> {
(**self).load_package_tree(root, resolution).await
(**self)
.load_package_tree(root, resolution, root_is_local_dir)
.await
}
}
1 change: 1 addition & 0 deletions lib/wasix/src/runtime/package_loader/unsupported.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ impl PackageLoader for UnsupportedPackageLoader {
&self,
_root: &Container,
_resolution: &Resolution,
_root_is_local_dir: bool,
) -> Result<BinaryPackage, Error> {
Err(Error::new(Unsupported))
}
Expand Down
49 changes: 48 additions & 1 deletion tests/integration/cli/tests/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ fn test_wasmer_run_works_with_dir() {
let temp_dir = tempfile::TempDir::new().unwrap();
let qjs_path = temp_dir.path().join("qjs.wasm");

std::fs::copy(fixtures::qjs(), &qjs_path).unwrap();
std::fs::copy(fixtures::qjs(), qjs_path).unwrap();
std::fs::copy(
fixtures::qjs_wasmer_toml(),
temp_dir.path().join("wasmer.toml"),
Expand Down Expand Up @@ -945,6 +945,53 @@ fn run_a_package_that_uses_an_atom_from_a_dependency() {
assert.success().stdout(contains("Hello, World!"));
}

#[test]
fn local_package_has_write_access_to_its_volumes() {
let temp = tempfile::tempdir().unwrap();

std::fs::write(
temp.path().join("wasmer.toml"),
r#"
[dependencies]
"python/python" = "*"

[fs]
"/mounted" = "."

[[command]]
name = "run"
module = "python/python:python"
runner = "wasi"

[command.annotations.wasi]
main-args = ["/mounted/script.py"]

"#,
)
.unwrap();

std::fs::write(
temp.path().join("script.py"),
r#"
file = open("/mounted/hello.txt", "w")
file.write("Hello, world!")
"#,
)
.unwrap();

Command::new(get_wasmer_path())
.arg("run")
.arg(temp.path())
.arg("--registry=wasmer.io")
.env("RUST_LOG", &*RUST_LOG)
.assert()
.success();

let file_contents =
String::from_utf8(std::fs::read(temp.path().join("hello.txt")).unwrap()).unwrap();
assert_eq!(file_contents, "Hello, world!");
}

fn project_root() -> &'static Path {
Path::new(env!("CARGO_MANIFEST_DIR"))
.ancestors()
Expand Down
Loading