Skip to content

Commit

Permalink
Merge pull request #4826 from wasmerio/run-289-remove-old-stale-webcf…
Browse files Browse the repository at this point in the history 
…ilesystem

chore(virtual-fs)!: Remove old stale WebcFileSystem
  • Loading branch information
@syrusakbary
syrusakbary authored Jun 11, 2024
2 parents 10a55f5 + a623ee0 commit 190a6da
Show file tree
Hide file tree
Showing 3 changed files with 151 additions and 651 deletions.
2 changes: 0 additions & 2 deletions lib/virtual-fs/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ mod static_file;
pub mod static_fs;
mod trace_fs;
#[cfg(feature = "webc-fs")]
pub mod webc_fs;
#[cfg(feature = "webc-fs")]
mod webc_volume_fs;

pub mod limiter;
Expand Down
312 changes: 151 additions & 161 deletions lib/virtual-fs/src/overlay_fs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1108,17 +1108,12 @@ fn should_continue(e: FsError) -> bool {

#[cfg(test)]
mod tests {
use std::{path::PathBuf, sync::Arc};
use std::path::PathBuf;

use bytes::Bytes;
use tempfile::TempDir;
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
use webc::v1::{ParseOptions, WebCOwned};

use super::*;
use crate::{mem_fs::FileSystem as MemFS, webc_fs::WebcFileSystem, RootFileSystemBuilder};

const PYTHON: &[u8] = include_bytes!("../../c-api/examples/assets/python-0.1.0.wasmer");
use crate::mem_fs::FileSystem as MemFS;

#[tokio::test]
async fn remove_directory() {
Expand Down Expand Up @@ -1256,160 +1251,6 @@ mod tests {
);
}

#[tokio::test]
async fn wasi_runner_use_case() {
// Set up some dummy files on the host
let temp = TempDir::new().unwrap();
let first = temp.path().join("first");
let file_txt = first.join("file.txt");
let second = temp.path().join("second");
std::fs::create_dir_all(&first).unwrap();
std::fs::write(&file_txt, b"First!").unwrap();
std::fs::create_dir_all(&second).unwrap();
// configure the union FS so things are saved in memory by default
// (initialized with a set of unix-like folders), but certain folders
// are first to the host.
let primary = RootFileSystemBuilder::new().build();
let host_fs: Arc<dyn FileSystem + Send + Sync> =
Arc::new(crate::host_fs::FileSystem::default());
let first_dirs = [(&first, "/first"), (&second, "/second")];
for (host, guest) in first_dirs {
primary
.mount(PathBuf::from(guest), &host_fs, host.clone())
.unwrap();
}
// Set up the secondary file systems
let webc = WebCOwned::parse(Bytes::from_static(PYTHON), &ParseOptions::default()).unwrap();
let webc = WebcFileSystem::init_all(Arc::new(webc));

let fs = OverlayFileSystem::new(primary, [webc]);

// We should get all the normal directories from rootfs (primary)
assert!(ops::is_dir(&fs, "/lib"));
assert!(ops::is_dir(&fs, "/bin"));
assert!(ops::is_file(&fs, "/dev/stdin"));
assert!(ops::is_file(&fs, "/dev/stdout"));
// We also want to see files from the WEBC volumes (secondary)
assert!(ops::is_dir(&fs, "/lib/python3.6"));
assert!(ops::is_file(&fs, "/lib/python3.6/collections/__init__.py"));
#[cfg(never)]
{
// files on a secondary fs aren't writable
// TODO(Michael-F-Bryan): re-enable this if/when we fix
// open_readonly_file_hack()
assert_eq!(
fs.new_open_options()
.append(true)
.open("/lib/python3.6/collections/__init__.py")
.unwrap_err(),
FsError::PermissionDenied,
);
}
// you are allowed to create files that look like they are in a secondary
// folder, though
ops::touch(&fs, "/lib/python3.6/collections/something-else.py").unwrap();
// But it'll be on the primary filesystem, not the secondary one
assert!(ops::is_file(
&fs.primary,
"/lib/python3.6/collections/something-else.py"
));
assert!(!ops::is_file(
&fs.secondaries[0],
"/lib/python3.6/collections/something-else.py"
));
// You can do the same thing with folders
fs.create_dir("/lib/python3.6/something-else".as_ref())
.unwrap();
assert!(ops::is_dir(&fs.primary, "/lib/python3.6/something-else"));
assert!(!ops::is_dir(
&fs.secondaries[0],
"/lib/python3.6/something-else"
));
// It only works when you are directly inside an existing directory
// on the secondary filesystem, though
assert_eq!(
ops::touch(&fs, "/lib/python3.6/collections/this/doesnt/exist.txt").unwrap_err(),
FsError::EntryNotFound
);
// you should also be able to read files mounted from the host
assert!(ops::is_dir(&fs, "/first"));
assert!(ops::is_file(&fs, "/first/file.txt"));
assert_eq!(
ops::read_to_string(&fs, "/first/file.txt").await.unwrap(),
"First!"
);
// Overwriting them is fine and we'll see the changes on the host
ops::write(&fs, "/first/file.txt", "Updated").await.unwrap();
assert_eq!(std::fs::read_to_string(&file_txt).unwrap(), "Updated");
// The filesystem will see changes on the host that happened after it was
// set up
let another = second.join("another.txt");
std::fs::write(&another, "asdf").unwrap();
assert_eq!(
ops::read_to_string(&fs, "/second/another.txt")
.await
.unwrap(),
"asdf"
);
}

fn load_webc(bytes: &'static [u8]) -> WebcFileSystem<WebCOwned> {
let options = ParseOptions::default();
let webc = WebCOwned::parse(bytes, &options).unwrap();
WebcFileSystem::init_all(Arc::new(webc))
}

#[track_caller]
fn assert_same_directory_contents(
original: &dyn FileSystem,
path: impl AsRef<Path>,
candidate: &dyn FileSystem,
) {
let path = path.as_ref();

let original_entries: Vec<_> = original
.read_dir(path)
.unwrap()
.map(|r| r.unwrap())
.collect();
let candidate_entries: Vec<_> = candidate
.read_dir(path)
.unwrap()
.map(|r| r.unwrap())
.collect();

assert_eq!(original_entries, candidate_entries);
}

#[tokio::test]
async fn absolute_and_relative_paths_are_passed_through() {
let python = Arc::new(load_webc(PYTHON));

// The underlying filesystem doesn't care about absolute/relative paths
assert_eq!(python.read_dir("/lib".as_ref()).unwrap().count(), 4);
assert_eq!(python.read_dir("lib".as_ref()).unwrap().count(), 4);

// read_dir() should be passed through to the primary
let webc_primary =
OverlayFileSystem::new(Arc::clone(&python), [crate::EmptyFileSystem::default()]);
assert_same_directory_contents(&python, "/lib", &webc_primary);
assert_same_directory_contents(&python, "lib", &webc_primary);

// read_dir() should also be passed through to the secondary
let webc_secondary =
OverlayFileSystem::new(crate::EmptyFileSystem::default(), [Arc::clone(&python)]);
assert_same_directory_contents(&python, "/lib", &webc_secondary);
assert_same_directory_contents(&python, "lib", &webc_secondary);

// It should be fine to overlay the root fs on top of our webc file
let overlay_rootfs = OverlayFileSystem::new(
RootFileSystemBuilder::default().build(),
[Arc::clone(&python)],
);
assert_same_directory_contents(&python, "/lib", &overlay_rootfs);
assert_same_directory_contents(&python, "lib", &overlay_rootfs);
}

#[tokio::test]
async fn open_secondary_fs_files_in_write_mode() {
let primary = MemFS::default();
Expand Down Expand Up @@ -1688,4 +1529,153 @@ mod tests {
FsError::EntryNotFound
)
}

// OLD tests that used WebcFileSystem.
// Should be re-implemented with WebcVolumeFs
// #[tokio::test]
// async fn wasi_runner_use_case() {
// // Set up some dummy files on the host
// let temp = TempDir::new().unwrap();
// let first = temp.path().join("first");
// let file_txt = first.join("file.txt");
// let second = temp.path().join("second");
// std::fs::create_dir_all(&first).unwrap();
// std::fs::write(&file_txt, b"First!").unwrap();
// std::fs::create_dir_all(&second).unwrap();
// // configure the union FS so things are saved in memory by default
// // (initialized with a set of unix-like folders), but certain folders
// // are first to the host.
// let primary = RootFileSystemBuilder::new().build();
// let host_fs: Arc<dyn FileSystem + Send + Sync> =
// Arc::new(crate::host_fs::FileSystem::default());
// let first_dirs = [(&first, "/first"), (&second, "/second")];
// for (host, guest) in first_dirs {
// primary
// .mount(PathBuf::from(guest), &host_fs, host.clone())
// .unwrap();
// }
// // Set up the secondary file systems
// let webc = WebCOwned::parse(Bytes::from_static(PYTHON), &ParseOptions::default()).unwrap();
// let webc = WebcFileSystem::init_all(Arc::new(webc));
//
// let fs = OverlayFileSystem::new(primary, [webc]);
//
// // We should get all the normal directories from rootfs (primary)
// assert!(ops::is_dir(&fs, "/lib"));
// assert!(ops::is_dir(&fs, "/bin"));
// assert!(ops::is_file(&fs, "/dev/stdin"));
// assert!(ops::is_file(&fs, "/dev/stdout"));
// // We also want to see files from the WEBC volumes (secondary)
// assert!(ops::is_dir(&fs, "/lib/python3.6"));
// assert!(ops::is_file(&fs, "/lib/python3.6/collections/__init__.py"));
// #[cfg(never)]
// {
// // files on a secondary fs aren't writable
// // TODO(Michael-F-Bryan): re-enable this if/when we fix
// // open_readonly_file_hack()
// assert_eq!(
// fs.new_open_options()
// .append(true)
// .open("/lib/python3.6/collections/__init__.py")
// .unwrap_err(),
// FsError::PermissionDenied,
// );
// }
// // you are allowed to create files that look like they are in a secondary
// // folder, though
// ops::touch(&fs, "/lib/python3.6/collections/something-else.py").unwrap();
// // But it'll be on the primary filesystem, not the secondary one
// assert!(ops::is_file(
// &fs.primary,
// "/lib/python3.6/collections/something-else.py"
// ));
// assert!(!ops::is_file(
// &fs.secondaries[0],
// "/lib/python3.6/collections/something-else.py"
// ));
// // You can do the same thing with folders
// fs.create_dir("/lib/python3.6/something-else".as_ref())
// .unwrap();
// assert!(ops::is_dir(&fs.primary, "/lib/python3.6/something-else"));
// assert!(!ops::is_dir(
// &fs.secondaries[0],
// "/lib/python3.6/something-else"
// ));
// // It only works when you are directly inside an existing directory
// // on the secondary filesystem, though
// assert_eq!(
// ops::touch(&fs, "/lib/python3.6/collections/this/doesnt/exist.txt").unwrap_err(),
// FsError::EntryNotFound
// );
// // you should also be able to read files mounted from the host
// assert!(ops::is_dir(&fs, "/first"));
// assert!(ops::is_file(&fs, "/first/file.txt"));
// assert_eq!(
// ops::read_to_string(&fs, "/first/file.txt").await.unwrap(),
// "First!"
// );
// // Overwriting them is fine and we'll see the changes on the host
// ops::write(&fs, "/first/file.txt", "Updated").await.unwrap();
// assert_eq!(std::fs::read_to_string(&file_txt).unwrap(), "Updated");
// // The filesystem will see changes on the host that happened after it was
// // set up
// let another = second.join("another.txt");
// std::fs::write(&another, "asdf").unwrap();
// assert_eq!(
// ops::read_to_string(&fs, "/second/another.txt")
// .await
// .unwrap(),
// "asdf"
// );
// }
//
// #[tokio::test]
// async fn absolute_and_relative_paths_are_passed_through() {
// let python = Arc::new(load_webc(PYTHON));
//
// // The underlying filesystem doesn't care about absolute/relative paths
// assert_eq!(python.read_dir("/lib".as_ref()).unwrap().count(), 4);
// assert_eq!(python.read_dir("lib".as_ref()).unwrap().count(), 4);
//
// // read_dir() should be passed through to the primary
// let webc_primary =
// OverlayFileSystem::new(Arc::clone(&python), [crate::EmptyFileSystem::default()]);
// assert_same_directory_contents(&python, "/lib", &webc_primary);
// assert_same_directory_contents(&python, "lib", &webc_primary);
//
// // read_dir() should also be passed through to the secondary
// let webc_secondary =
// OverlayFileSystem::new(crate::EmptyFileSystem::default(), [Arc::clone(&python)]);
// assert_same_directory_contents(&python, "/lib", &webc_secondary);
// assert_same_directory_contents(&python, "lib", &webc_secondary);
//
// // It should be fine to overlay the root fs on top of our webc file
// let overlay_rootfs = OverlayFileSystem::new(
// RootFileSystemBuilder::default().build(),
// [Arc::clone(&python)],
// );
// assert_same_directory_contents(&python, "/lib", &overlay_rootfs);
// assert_same_directory_contents(&python, "lib", &overlay_rootfs);
// }
// #[track_caller]
// fn assert_same_directory_contents(
// original: &dyn FileSystem,
// path: impl AsRef<Path>,
// candidate: &dyn FileSystem,
// ) {
// let path = path.as_ref();
//
// let original_entries: Vec<_> = original
// .read_dir(path)
// .unwrap()
// .map(|r| r.unwrap())
// .collect();
// let candidate_entries: Vec<_> = candidate
// .read_dir(path)
// .unwrap()
// .map(|r| r.unwrap())
// .collect();
//
// assert_eq!(original_entries, candidate_entries);
// }
}
Loading

0 comments on commit 190a6da

Please sign in to comment.