Merge #451

451: implement map-dir for WASI; fix bug in path_open r=MarkMcCaskey a=MarkMcCaskey Colon separated mapping, implemented to be compatible with wasmtime resolves #458 example: ```Rust use std::fs; fn main() { let read_dir = fs::read_dir(".").unwrap(); for entry in read_dir { println!("{:?}", entry.unwrap().path()); } } ``` ```shell $ cargo run --release --bin wasmer -- run --mapdir=.:src list-files.wasm "./bin" "./installer" "./lib" "./update" "./utils" "./webassembly" ``` Co-authored-by: Mark McCaskey <[email protected]> Co-authored-by: Mark McCaskey <[email protected]>
wasmerio · May 21, 2019 · f3220a0 · f3220a0
2 parents 082c56b + 54a91a1
commit f3220a0
Show file tree

Hide file tree

Showing 37 changed files with 8,061 additions and 40 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,11 +6,11 @@ Blocks of changes will separated by version increments.
 
 ## **[Unreleased]**
 
-- *empty*
+- [#451](https://github.com/wasmerio/wasmer/pull/451) Add `--mapdir=src:dest` flag to rename host directories in the guest context
+- [#457](https://github.com/wasmerio/wasmer/pull/457) Implement file metadata for WASI, fix bugs in WASI clock code for Unix platforms
 
 ## 0.4.2 - 2019-05-16
 
-- [#457](https://github.com/wasmerio/wasmer/pull/457) Implement file metadata for WASI, fix bugs in WASI clock code for Unix platforms
 - [#416](https://github.com/wasmerio/wasmer/pull/416) Remote code loading framework
 - [#449](https://github.com/wasmerio/wasmer/pull/449) Fix bugs: opening host files in filestat and opening with write permissions unconditionally in path_open
 - [#442](https://github.com/wasmerio/wasmer/pull/442) Misc. WASI FS fixes and implement readdir

diff --git a/Makefile b/Makefile
@@ -70,10 +70,10 @@ test-emscripten-singlepass:
 	cargo test --manifest-path lib/emscripten/Cargo.toml --features singlepass -- --test-threads=1 $(runargs)
 
 test-wasi-clif:
-	cargo test --manifest-path lib/wasi/Cargo.toml --features "clif" -- $(runargs)
+	cargo test --manifest-path lib/wasi/Cargo.toml --features "clif" -- --test-threads=1 $(runargs)
 
 test-wasi-singlepass:
-	cargo test --manifest-path lib/wasi/Cargo.toml --features "singlepass" -- $(runargs)
+	cargo test --manifest-path lib/wasi/Cargo.toml --features "singlepass" -- --test-threads=1 $(runargs)
 
 singlepass-debug-release:
 	cargo +nightly build --features backend:singlepass,debug --release

diff --git a/examples/plugin.rs b/examples/plugin.rs
@@ -17,7 +17,7 @@ fn main() {
     ));
 
     // WASI imports
-    let mut base_imports = generate_import_object(vec![], vec![], vec![]);
+    let mut base_imports = generate_import_object(vec![], vec![], vec![], vec![]);
     // env is the default namespace for extern functions
     let custom_imports = imports! {
         "env" => {

diff --git a/lib/wasi/build/wasitests.rs b/lib/wasi/build/wasitests.rs
@@ -75,13 +75,35 @@ pub fn compile(file: &str, ignores: &HashSet<String>) -> Option<String> {
         ""
     };
 
+    let src_code = fs::read_to_string(file).expect("read src file");
+    let args = extract_args_from_source_file(&src_code);
+
+    let mapdir_args = if let Some(a) = args {
+        if !a.mapdir.is_empty() {
+            let mut out_str = String::new();
+            out_str.push_str("vec![");
+            for (alias, real_dir) in a.mapdir {
+                out_str.push_str(&format!(
+                    "(\"{}\".to_string(), \"{}\".to_string()),",
+                    alias, real_dir
+                ));
+            }
+            out_str.push_str("]");
+            out_str
+        } else {
+            "vec![]".to_string()
+        }
+    } else {
+        "vec![]".to_string()
+    };
+
     let contents = format!(
         "#[test]{ignore}
 fn test_{rs_module_name}() {{
     assert_wasi_output!(
         \"../../{module_path}\",
         \"{rs_module_name}\",
-        vec![],
+        {mapdir_args},
         \"../../{test_output_path}\"
     );
 }}
@@ -90,6 +112,7 @@ fn test_{rs_module_name}() {{
         module_path = wasm_out_name,
         rs_module_name = rs_module_name,
         test_output_path = format!("{}.out", normalized_name),
+        mapdir_args = mapdir_args,
     );
     let rust_test_filepath = format!(
         concat!(env!("CARGO_MANIFEST_DIR"), "/tests/{}.rs"),
@@ -143,3 +166,42 @@ fn read_ignore_list() -> HashSet<String> {
         .map(|v| v.to_lowercase())
         .collect()
 }
+
+struct Args {
+    pub mapdir: Vec<(String, String)>,
+}
+
+/// Pulls args to the program out of a comment at the top of the file starting with "// Args:"
+fn extract_args_from_source_file(source_code: &str) -> Option<Args> {
+    if source_code.starts_with("// Args:") {
+        let mut args = Args { mapdir: vec![] };
+        for arg_line in source_code
+            .lines()
+            .skip(1)
+            .take_while(|line| line.starts_with("// "))
+        {
+            let tokenized = arg_line
+                .split_whitespace()
+                .skip(1)
+                .map(String::from)
+                .collect::<Vec<String>>();
+            match tokenized[1].as_ref() {
+                "mapdir" => {
+                    if let [alias, real_dir] = &tokenized[2].split(':').collect::<Vec<&str>>()[..] {
+                        args.mapdir.push((alias.to_string(), real_dir.to_string()));
+                    } else {
+                        eprintln!(
+                            "Parse error in mapdir {} not parsed correctly",
+                            &tokenized[2]
+                        );
+                    }
+                }
+                e => {
+                    eprintln!("WARN: comment arg: {} is not supported", e);
+                }
+            }
+        }
+        return Some(args);
+    }
+    None
+}
diff --git a/lib/wasi/src/lib.rs b/lib/wasi/src/lib.rs
@@ -14,6 +14,7 @@ use self::state::{WasiFs, WasiState};
 use self::syscalls::*;
 
 use std::ffi::c_void;
+use std::path::PathBuf;
 
 pub use self::utils::is_wasi_module;
 
@@ -29,6 +30,7 @@ pub fn generate_import_object(
     args: Vec<Vec<u8>>,
     envs: Vec<Vec<u8>>,
     preopened_files: Vec<String>,
+    mapped_dirs: Vec<(String, PathBuf)>,
 ) -> ImportObject {
     let state_gen = move || {
         fn state_destructor(data: *mut c_void) {
@@ -38,7 +40,7 @@ pub fn generate_import_object(
         }
 
         let state = Box::new(WasiState {
-            fs: WasiFs::new(&preopened_files).unwrap(),
+            fs: WasiFs::new(&preopened_files, &mapped_dirs).unwrap(),
             args: &args[..],
             envs: &envs[..],
         });

diff --git a/lib/wasi/src/state.rs b/lib/wasi/src/state.rs
@@ -175,21 +175,20 @@ pub struct WasiFs {
 }
 
 impl WasiFs {
-    pub fn new(preopened_dirs: &[String]) -> Result<Self, String> {
-        /*let repo = RepoOpener::new()
-        .create(true)
-        .open("mem://wasmer-test-fs", "")
-        .map_err(|e| e.to_string())?;*/
+    pub fn new(
+        preopened_dirs: &[String],
+        mapped_dirs: &[(String, PathBuf)],
+    ) -> Result<Self, String> {
         debug!("wasi::fs::inodes");
         let inodes = Arena::new();
         let mut wasi_fs = Self {
-            //repo: repo,
             name_map: HashMap::new(),
             inodes: inodes,
             fd_map: HashMap::new(),
             next_fd: Cell::new(3),
             inode_counter: Cell::new(1000),
         };
+        debug!("wasi::fs::preopen_dirs");
         for dir in preopened_dirs {
             debug!("Attempting to preopen {}", &dir);
             // TODO: think about this
@@ -218,6 +217,36 @@ impl WasiFs {
                 .create_fd(default_rights, default_rights, 0, inode)
                 .expect("Could not open fd");
         }
+        debug!("wasi::fs::mapped_dirs");
+        for (alias, real_dir) in mapped_dirs {
+            debug!("Attempting to open {:?} at {}", dest_dir, alias);
+            // TODO: think about this
+            let default_rights = 0x1FFFFFFF; // all rights
+            let cur_dir_metadata = real_dir
+                .metadata()
+                .expect("mapped dir not at previously verified location");
+            let kind = if cur_dir_metadata.is_dir() {
+                Kind::Dir {
+                    parent: None,
+                    path: real_dir.clone(),
+                    entries: Default::default(),
+                }
+            } else {
+                return Err(format!(
+                    "WASI only supports pre-opened directories right now; found \"{:?}\"",
+                    &real_dir,
+                ));
+            };
+            // TODO: handle nested pats in `file`
+            let inode_val =
+                InodeVal::from_file_metadata(&cur_dir_metadata, alias.clone(), true, kind);
+
+            let inode = wasi_fs.inodes.insert(inode_val);
+            wasi_fs.inodes[inode].stat.st_ino = wasi_fs.inode_counter.get();
+            wasi_fs
+                .create_fd(default_rights, default_rights, 0, inode)
+                .expect("Could not open fd");
+        }
         debug!("wasi::fs::end");
         Ok(wasi_fs)
     }
@@ -419,29 +448,10 @@ impl WasiFs {
     }
 
     pub fn get_base_path_for_directory(&self, directory: Inode) -> Option<String> {
-        let mut path_segments = vec![];
-        let mut cur_inode = directory;
-        loop {
-            path_segments.push(self.inodes[cur_inode].name.clone());
-
-            if let Kind::Dir { parent, .. } = &self.inodes[cur_inode].kind {
-                if let Some(p_inode) = parent {
-                    cur_inode = *p_inode;
-                } else {
-                    break;
-                }
-            } else {
-                return None;
-            }
+        if let Kind::Dir { path, .. } = &self.inodes[directory].kind {
+            return Some(path.to_string_lossy().to_string());
         }
-
-        path_segments.reverse();
-        Some(
-            path_segments
-                .iter()
-                .skip(1)
-                .fold(path_segments.first()?.clone(), |a, b| a + "/" + b),
-        )
+        None
     }
 }
 

diff --git a/lib/wasi/src/syscalls/mod.rs b/lib/wasi/src/syscalls/mod.rs
@@ -795,6 +795,7 @@ pub fn fd_readdir(
         for entry in entries.iter().skip(cookie as usize) {
             cur_cookie += 1;
             let entry_path = entry.path();
+            let entry_path = wasi_try!(entry_path.file_name().ok_or(__WASI_EIO));
             let entry_path_str = entry_path.to_string_lossy();
             let namlen = entry_path_str.len();
             debug!("Returning dirent for {}", entry_path_str);
@@ -1486,10 +1487,11 @@ pub fn path_open(
                 .fs
                 .create_fd(fs_rights_base, fs_rights_inheriting, fs_flags, child))
         } else {
-            let file_metadata = wasi_try!(file_path.metadata().map_err(|_| __WASI_ENOENT));
+            debug!("Attempting to load file from host system");
+            let file_metadata = file_path.metadata();
             // if entry does not exist in parent directory, try to lazily
             // load it; possibly creating or truncating it if flags set
-            let kind = if file_metadata.is_dir() {
+            let kind = if file_metadata.is_ok() && file_metadata.unwrap().is_dir() {
                 // special dir logic
                 Kind::Dir {
                     parent: Some(cur_dir_inode),

diff --git a/lib/wasi/tests/wasitests/_common.rs b/lib/wasi/tests/wasitests/_common.rs
@@ -1,5 +1,5 @@
 macro_rules! assert_wasi_output {
-    ($file:expr, $name:expr, $args:expr, $expected:expr) => {{
+    ($file:expr, $name:expr, $mapdir_args:expr, $expected:expr) => {{
         use wasmer_dev_utils::stdio::StdioCapturer;
         use wasmer_runtime_core::{backend::Compiler, Func};
         use wasmer_wasi::generate_import_object;
@@ -32,7 +32,8 @@ macro_rules! assert_wasi_output {
         let module = wasmer_runtime_core::compile_with(&wasm_bytes[..], &get_compiler())
             .expect("WASM can't be compiled");
 
-        let import_object = generate_import_object(vec![], vec![], vec![".".to_string()]);
+        let import_object =
+            generate_import_object(vec![], vec![], vec![".".to_string()], $mapdir_args);
 
         let instance = module
             .instantiate(&import_object)

diff --git a/lib/wasi/tests/wasitests/mapdir.rs b/lib/wasi/tests/wasitests/mapdir.rs
@@ -0,0 +1,9 @@
+#[test]
+fn test_mapdir() {
+    assert_wasi_output!(
+        "../../wasitests/mapdir.wasm",
+        "mapdir",
+        vec![],
+        "../../wasitests/mapdir.out"
+    );
+}
diff --git a/lib/wasi/tests/wasitests/mod.rs b/lib/wasi/tests/wasitests/mod.rs
@@ -8,4 +8,5 @@ mod _common;
 mod create_dir;
 mod file_metadata;
 mod hello;
+mod mapdir;
 mod quine;
diff --git a/lib/wasi/wasitests/mapdir b/lib/wasi/wasitests/mapdir
diff --git a/lib/wasi/wasitests/mapdir.out b/lib/wasi/wasitests/mapdir.out
@@ -0,0 +1,6 @@
+"wasitests/test_fs/hamlet/README.md"
+"wasitests/test_fs/hamlet/act1"
+"wasitests/test_fs/hamlet/act2"
+"wasitests/test_fs/hamlet/act3"
+"wasitests/test_fs/hamlet/act4"
+"wasitests/test_fs/hamlet/act5"
diff --git a/lib/wasi/wasitests/mapdir.rs b/lib/wasi/wasitests/mapdir.rs
@@ -0,0 +1,20 @@
+// Args:
+// mapdir: .:wasitests/test_fs/hamlet
+
+use std::fs;
+
+fn main() {
+    #[cfg(not(target = "wasi"))]
+    let read_dir = fs::read_dir("wasitests/test_fs/hamlet").unwrap();
+    #[cfg(target = "wasi")]
+    let read_dir = fs::read_dir(".").unwrap();
+    let mut out = vec![];
+    for entry in read_dir {
+        out.push(format!("{:?}", entry.unwrap().path()));
+    }
+    out.sort();
+
+    for p in out {
+        println!("{}", p);
+    }
+}
diff --git a/lib/wasi/wasitests/mapdir.wasm b/lib/wasi/wasitests/mapdir.wasm
diff --git a/lib/wasi/wasitests/test_fs/README.md b/lib/wasi/wasitests/test_fs/README.md
@@ -0,0 +1,7 @@
+# Test FS
+
+This is a test "file system" used in some of the WASI integration tests.
+
+It's just a bunch of files in a tree.
+
+If you make changes here, please regenerate the tests with `make wasitests`!
diff --git a/lib/wasi/wasitests/test_fs/hamlet/README.md b/lib/wasi/wasitests/test_fs/hamlet/README.md
@@ -0,0 +1,3 @@
+# Hamlet split in to acts and scenes
+
+From: http://shakespeare.mit.edu/hamlet/full.html
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		# Hamlet split in to acts and scenes

		From: http://shakespeare.mit.edu/hamlet/full.html