Skip to content

Commit

Permalink
Ripped out the union fs stuff
Browse files Browse the repository at this point in the history 
The WasiRunner's filesystem now only has access to volumes from its container.

The WcgiRunner's filesystem now only has access to mapped directories.
  • Loading branch information
Michael-F-Bryan committed Mar 17, 2023
1 parent e53956a commit db9753b
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 159 deletions.
150 changes: 2 additions & 148 deletions lib/wasi/src/runners/wasi.rs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
//! WebC container support for running WASI modules
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::Arc,
};
use std::{collections::HashMap, sync::Arc};

use crate::{
runners::{MappedDirectory, WapmContainer},
Expand All @@ -13,9 +9,6 @@ use crate::{
use crate::{WasiEnv, WasiEnvBuilder};
use anyhow::{Context, Error};
use serde::{Deserialize, Serialize};
use virtual_fs::{
FileSystem, FsError, OverlayFileSystem, PassthruFileSystem, RootFileSystemBuilder,
};
use wasmer::{Module, Store};
use webc::metadata::{annotations::Wasi, Command};

Expand Down Expand Up @@ -132,7 +125,7 @@ impl WasiRunner {
container: &WapmContainer,
command: &str,
) -> Result<WasiEnvBuilder, anyhow::Error> {
let (fs, preopen_dirs) = unioned_filesystem(&self.mapped_dirs, container)?;
let (fs, preopen_dirs) = container.container_fs();

let mut builder = WasiEnv::builder(command).args(&self.args);

Expand Down Expand Up @@ -192,142 +185,3 @@ impl crate::runners::Runner for WasiRunner {
Ok(())
}
}

/// Create a [`FileSystem`] which merges the WAPM container's volumes with any
/// directories that were mapped from the host.
pub(crate) fn unioned_filesystem(
mapped_dirs: &[MappedDirectory],
container: &WapmContainer,
) -> Result<(impl FileSystem, Vec<PathBuf>), Error> {
// We start with the root filesystem so we get things like "/dev/"
let primary = RootFileSystemBuilder::new().build();

let mut preopen_dirs = Vec::new();

// Now, let's merge in the host volumes.
if !mapped_dirs.is_empty() {
let host_fs: Arc<dyn FileSystem + Send + Sync> =
Arc::new(PassthruFileSystem::new(crate::default_fs_backing()));

for MappedDirectory { host, guest } in mapped_dirs.iter() {
let guest = match guest.starts_with('/') {
true => PathBuf::from(guest),
false => Path::new("/").join(guest),
};
tracing::debug!(
host=%host.display(),
guest=%guest.display(),
"mounting host directory",
);

if let Some(parent) = guest.parent() {
create_dir_all(&primary, parent.as_ref())?;
}

primary
.mount(guest.clone(), &host_fs, host.clone())
.with_context(|| {
format!(
"Unable to mount \"{}\" to \"{}\"",
host.display(),
guest.display()
)
})?;

preopen_dirs.push(guest);
}
}

let (container_fs, top_level_dirs) = container.container_fs();

preopen_dirs.extend(top_level_dirs.into_iter().map(|p| Path::new("/").join(p)));

// Once we've set up the primary filesystem, make sure it is overlayed with
// the WEBC container's files
let fs = OverlayFileSystem::new(primary, [container_fs]);

Ok((fs, preopen_dirs))
}

fn create_dir_all(fs: &(impl FileSystem + ?Sized), path: &Path) -> Result<(), Error> {
match fs.metadata(path) {
Ok(meta) if meta.is_dir() => return Ok(()),
Ok(_) => anyhow::bail!(FsError::BaseNotDirectory),
Err(FsError::EntryNotFound) => {}
Err(e) => anyhow::bail!(e),
}

if let Some(parent) = path.parent() {
create_dir_all(fs, parent)?;
}

fs.create_dir(path)?;

Ok(())
}

#[cfg(test)]
mod tests {
use tempfile::TempDir;
use tokio::io::AsyncReadExt;

use super::*;

#[track_caller]
async fn read_file(fs: &dyn FileSystem, path: impl AsRef<Path>) -> String {
let mut f = fs.new_open_options().read(true).open(path).unwrap();
let mut contents = String::new();
f.read_to_string(&mut contents).await.unwrap();

contents
}

#[track_caller]
fn read_dir(fs: &dyn FileSystem, path: impl AsRef<Path>) -> Vec<PathBuf> {
fs.read_dir(path.as_ref())
.unwrap()
.filter_map(|result| result.ok())
.map(|entry| entry.path)
.collect()
}

#[tokio::test]
async fn construct_the_unioned_fs() {
let temp = TempDir::new().unwrap();
std::fs::write(temp.path().join("file.txt"), b"Hello, World!").unwrap();
let webc: &[u8] =
include_bytes!("../../../../lib/c-api/examples/assets/python-0.1.0.wasmer");
let container = WapmContainer::from_bytes(webc.into()).unwrap();
let mapped_dirs = [MappedDirectory {
guest: "/path/to/".to_string(),
host: temp.path().to_path_buf(),
}];

let (fs, _) = unioned_filesystem(&mapped_dirs, &container).unwrap();

// Files that were mounted on the host
let path_contents = read_dir(&fs, "/path/to/");
// FIXME: We can't use the commented-out version because of a bug in
// memfs. For more, see https://github.com/wasmerio/wasmer/issues/3685
assert_eq!(path_contents.len(), 1);
assert!(path_contents[0].ends_with("file.txt"));
// assert_eq!(
// read_dir(&fs, "/path/to/"),
// vec![PathBuf::from("/path/to/file.txt")]
// );
assert_eq!(read_file(&fs, "/path/to/file.txt").await, "Hello, World!");
// Files from the Python WEBC file's volumes
assert_eq!(
read_dir(&fs, "/lib/python3.6/collections/"),
vec![
PathBuf::from("/lib/python3.6/collections/__init__.py"),
PathBuf::from("/lib/python3.6/collections/abc.py"),
]
);
let abc = read_file(&fs, "/lib/python3.6/collections/abc.py").await;
assert_eq!(
abc,
"from _collections_abc import *\nfrom _collections_abc import __all__\n"
);
}
}
67 changes: 58 additions & 9 deletions lib/wasi/src/runners/wcgi/handler.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use std::{
task::Poll,
};

use anyhow::Error;
use anyhow::{Context, Error};
use futures::{Future, FutureExt, StreamExt, TryFutureExt};
use http::{Request, Response};
use hyper::{service::Service, Body};
Expand All @@ -16,7 +16,7 @@ use tokio::{
runtime::Handle,
};
use tracing::Instrument;
use virtual_fs::FileSystem;
use virtual_fs::{FileSystem, PassthruFileSystem, RootFileSystemBuilder, TmpFileSystem};
use wasmer::Module;
use wcgi_host::CgiDialect;

Expand Down Expand Up @@ -54,8 +54,6 @@ impl Handler {
self.dialect
.prepare_environment_variables(parts, &mut request_specific_env);

let (fs, preopen_dirs) = self.fs()?;

let rt = PluggableRuntimeImplementation::new(Arc::clone(&self.task_manager));
let mut builder = builder
.envs(self.env.iter())
Expand All @@ -70,11 +68,13 @@ impl Handler {
threading: Default::default(),
})
.runtime(Arc::new(rt))
.fs(Box::new(fs))
.fs(Box::new(self.fs()?))
.preopen_dir(Path::new("/"))?;

for dir in preopen_dirs {
builder.add_preopen_dir(dir)?;
for mapping in &self.mapped_dirs {
builder
.add_preopen_dir(&mapping.guest)
.with_context(|| format!("Unable to preopen \"{}\"", mapping.guest))?;
}

let module = self.module.clone();
Expand Down Expand Up @@ -136,8 +136,43 @@ impl Handler {
Ok(response)
}

fn fs(&self) -> Result<(impl FileSystem, Vec<PathBuf>), Error> {
crate::runners::wasi::unioned_filesystem(&self.mapped_dirs, &self.container)
fn fs(&self) -> Result<TmpFileSystem, Error> {
let root_fs = RootFileSystemBuilder::new().build();

if !self.mapped_dirs.is_empty() {
let fs_backing: Arc<dyn FileSystem + Send + Sync> =
Arc::new(PassthruFileSystem::new(crate::default_fs_backing()));

for MappedDirectory { host, guest } in self.mapped_dirs.iter() {
let guest = match guest.starts_with('/') {
true => PathBuf::from(guest),
false => Path::new("/").join(guest),
};
tracing::debug!(
host=%host.display(),
guest=%guest.display(),
"mounting host directory",
);

if let Some(parent) = guest.parent() {
create_dir_all(&root_fs, parent)
.with_context(|| format!("Unable to create \"{}\"", parent.display()))?;
}

root_fs
.mount(guest.clone(), &fs_backing, host.clone())
.with_context(|| {
format!(
"Unable to mount \"{}\" to \"{}\"",
host.display(),
guest.display()
)
})
.map_err(|e| dbg!(e))?;
}
}

Ok(root_fs)
}
}

Expand All @@ -149,6 +184,20 @@ impl Deref for Handler {
}
}

fn create_dir_all(fs: &dyn FileSystem, path: &Path) -> Result<(), Error> {
if fs.metadata(path).is_ok() {
return Ok(());
}

if let Some(parent) = path.parent() {
create_dir_all(fs, parent)?;
}

fs.create_dir(path)?;

Ok(())
}

/// Drive the request to completion by streaming the request body to the
/// instance and waiting for it to exit.
async fn drive_request_to_completion(
Expand Down
3 changes: 1 addition & 2 deletions tests/integration/cli/tests/run2.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ mod webc_on_disk {
}

#[test]
#[ignore = "WASI runners only give you access to the webc fs for now"]
fn wasi_runner_with_mounted_directories_and_webc_volumes() {
let temp = TempDir::new().unwrap();
std::fs::write(temp.path().join("main.py"), "print('Hello, World!')").unwrap();
Expand Down Expand Up @@ -89,8 +90,6 @@ mod webc_on_disk {
.env("RUST_LOG", RUST_LOG)
.assert();

panic!("{}", String::from_utf8_lossy(&assert.get_output().stderr));

assert.success().stdout(contains("Hello, World!"));
}

Expand Down

0 comments on commit db9753b

Please sign in to comment.