Merge #555

555: wasi filesystem rewrite + implementation of many syscalls r=MarkMcCaskey a=MarkMcCaskey

This lets the go compiler print its help text...

The rest of symlink code to follow...

This PR also does a major refactor of the file system code.  So we'll have to add more tests to be sure we didn't break anything

Co-authored-by: Mark McCaskey <[email protected]>
Co-authored-by: Mark McCaskey <[email protected]>

CHANGELOG.md

@@ -5,6 +5,11 @@ All PRs to the Wasmer repository must add to this file.
 Blocks of changes will separated by version increments.
 
 ## **[Unreleased]**
+- [#555](https://github.com/wasmerio/wasmer/pull/555) WASI filesystem rewrite.  Major improvements
+  - adds virtual root showing all preopened directories
+  - improved sandboxing and code-reuse
+  - symlinks work in a lot more situations
+  - many various improvements to most syscalls touching the filesystem
 
 ## 0.5.6
 - [#565](https://github.com/wasmerio/wasmer/pull/565) Update wapm and bump version to 0.5.6

Makefile

@@ -8,7 +8,13 @@ generate-emtests:
 	WASM_EMSCRIPTEN_GENERATE_EMTESTS=1 cargo build -p wasmer-emscripten-tests --release
 
 generate-wasitests:
-	WASM_WASI_GENERATE_WASITESTS=1 cargo build -p wasmer-wasi-tests --release -vv
+	WASM_WASI_GENERATE_WASITESTS=1 cargo build -p wasmer-wasi-tests --release -vv \
+	&& echo "formatting" \
+	&& cargo fmt
+
+spectests-generate: generate-spectests
+emtests-generate: generate-emtests
+wasitests-generate: generate-wasitests
 
 generate: generate-spectests generate-emtests generate-wasitests
 

lib/runtime-core/src/memory/ptr.rs

@@ -126,6 +126,15 @@ impl<T: Copy + ValueType> WasmPtr<T, Array> {
             [index as usize..slice_full_len];
         Some(cell_ptrs)
     }
+
+    pub fn get_utf8_string<'a>(self, memory: &'a Memory, str_len: u32) -> Option<&'a str> {
+        if self.offset as usize + str_len as usize > memory.size().bytes().0 {
+            return None;
+        }
+        let ptr = unsafe { memory.view::<u8>().as_ptr().add(self.offset as usize) as *const u8 };
+        let slice: &[u8] = unsafe { std::slice::from_raw_parts(ptr, str_len as usize) };
+        std::str::from_utf8(slice).ok()
+    }
 }
 
 unsafe impl<T: Copy, Ty> WasmExternType for WasmPtr<T, Ty> {

lib/wasi-tests/build/wasitests.rs

@@ -89,7 +89,7 @@ pub fn compile(file: &str, ignores: &HashSet<String>) -> Option<String> {
     };
 
     let src_code = fs::read_to_string(file).expect("read src file");
-    let args = extract_args_from_source_file(&src_code).unwrap_or_default();
+    let args: Args = extract_args_from_source_file(&src_code).unwrap_or_default();
 
     let mapdir_args = {
         let mut out_str = String::new();
@@ -116,12 +116,25 @@ pub fn compile(file: &str, ignores: &HashSet<String>) -> Option<String> {
         out_str
     };
 
+    let dir_args = {
+        let mut out_str = String::new();
+        out_str.push_str("vec![");
+
+        for entry in args.po_dirs {
+            out_str.push_str(&format!("\"{}\".to_string(),", entry));
+        }
+
+        out_str.push_str("]");
+        out_str
+    };
+
     let contents = format!(
         "#[test]{ignore}
 fn test_{rs_module_name}() {{
     assert_wasi_output!(
         \"../../{module_path}\",
         \"{rs_module_name}\",
+        {dir_args},
         {mapdir_args},
         {envvar_args},
         \"../../{test_output_path}\"
@@ -132,6 +145,7 @@ fn test_{rs_module_name}() {{
         module_path = wasm_out_name,
         rs_module_name = rs_module_name,
         test_output_path = format!("{}.out", normalized_name),
+        dir_args = dir_args,
         mapdir_args = mapdir_args,
         envvar_args = envvar_args
     );
@@ -192,6 +206,8 @@ fn read_ignore_list() -> HashSet<String> {
 struct Args {
     pub mapdir: Vec<(String, String)>,
     pub envvars: Vec<(String, String)>,
+    /// pre-opened directories
+    pub po_dirs: Vec<String>,
 }
 
 /// Pulls args to the program out of a comment at the top of the file starting with "// Args:"
@@ -237,6 +253,9 @@ fn extract_args_from_source_file(source_code: &str) -> Option<Args> {
                         eprintln!("Parse error in env {} not parsed correctly", &tokenized[1]);
                     }
                 }
+                "dir" => {
+                    args.po_dirs.push(tokenized[1].to_string());
+                }
                 e => {
                     eprintln!("WARN: comment arg: {} is not supported", e);
                 }

lib/wasi-tests/tests/wasitests/_common.rs

@@ -1,5 +1,5 @@
 macro_rules! assert_wasi_output {
-    ($file:expr, $name:expr, $mapdir_args:expr, $envvar_args:expr, $expected:expr) => {{
+    ($file:expr, $name:expr, $po_dir_args: expr, $mapdir_args:expr, $envvar_args:expr, $expected:expr) => {{
         use wasmer_dev_utils::stdio::StdioCapturer;
         use wasmer_runtime_core::{backend::Compiler, Func};
         use wasmer_wasi::generate_import_object;
@@ -33,8 +33,7 @@ macro_rules! assert_wasi_output {
         let module = wasmer_runtime_core::compile_with(&wasm_bytes[..], &get_compiler())
             .expect("WASM can't be compiled");
 
-        let import_object =
-            generate_import_object(vec![], vec![], vec![".".to_string()], $mapdir_args);
+        let import_object = generate_import_object(vec![], vec![], $po_dir_args, $mapdir_args);
 
         let instance = module
             .instantiate(&import_object)

lib/wasi-tests/tests/wasitests/create_dir.rs

@@ -1,9 +1,9 @@
 #[test]
-#[ignore]
 fn test_create_dir() {
     assert_wasi_output!(
         "../../wasitests/create_dir.wasm",
         "create_dir",
+        vec![".".to_string(),],
         vec![],
         vec![],
         "../../wasitests/create_dir.out"

lib/wasi-tests/tests/wasitests/envvar.rs

@@ -4,6 +4,7 @@ fn test_envvar() {
         "../../wasitests/envvar.wasm",
         "envvar",
         vec![],
+        vec![],
         vec!["DOG=1".to_string(), "CAT=2".to_string(),],
         "../../wasitests/envvar.out"
     );

lib/wasi-tests/tests/wasitests/file_metadata.rs

@@ -3,6 +3,7 @@ fn test_file_metadata() {
     assert_wasi_output!(
         "../../wasitests/file_metadata.wasm",
         "file_metadata",
+        vec![".".to_string(),],
         vec![],
         vec![],
         "../../wasitests/file_metadata.out"

lib/wasi-tests/tests/wasitests/fs_sandbox_test.rs

@@ -5,6 +5,7 @@ fn test_fs_sandbox_test() {
         "fs_sandbox_test",
         vec![],
         vec![],
+        vec![],
         "../../wasitests/fs_sandbox_test.out"
     );
 }

lib/wasi-tests/tests/wasitests/fseek.rs

@@ -3,6 +3,7 @@ fn test_fseek() {
     assert_wasi_output!(
         "../../wasitests/fseek.wasm",
         "fseek",
+        vec![],
         vec![(
             ".".to_string(),
             ::std::path::PathBuf::from("wasitests/test_fs/hamlet")

lib/wasi-tests/tests/wasitests/hello.rs

@@ -5,6 +5,7 @@ fn test_hello() {
         "hello",
         vec![],
         vec![],
+        vec![],
         "../../wasitests/hello.out"
     );
 }

lib/wasi-tests/tests/wasitests/mapdir.rs

@@ -3,6 +3,7 @@ fn test_mapdir() {
     assert_wasi_output!(
         "../../wasitests/mapdir.wasm",
         "mapdir",
+        vec![],
         vec![(
             ".".to_string(),
             ::std::path::PathBuf::from("wasitests/test_fs/hamlet")

lib/wasi-tests/tests/wasitests/mod.rs

@@ -13,3 +13,6 @@ mod fseek;
 mod hello;
 mod mapdir;
 mod quine;
+mod readlink;
+mod wasi_sees_virtual_root;
+mod writing;

lib/wasi-tests/tests/wasitests/quine.rs

@@ -3,6 +3,7 @@ fn test_quine() {
     assert_wasi_output!(
         "../../wasitests/quine.wasm",
         "quine",
+        vec![".".to_string(),],
         vec![],
         vec![],
         "../../wasitests/quine.out"

lib/wasi-tests/tests/wasitests/readlink.rs

@@ -0,0 +1,14 @@
+#[test]
+fn test_readlink() {
+    assert_wasi_output!(
+        "../../wasitests/readlink.wasm",
+        "readlink",
+        vec![],
+        vec![(
+            ".".to_string(),
+            ::std::path::PathBuf::from("wasitests/test_fs/hamlet")
+        ),],
+        vec![],
+        "../../wasitests/readlink.out"
+    );
+}

lib/wasi-tests/tests/wasitests/wasi_sees_virtual_root.rs

@@ -0,0 +1,24 @@
+#[test]
+fn test_wasi_sees_virtual_root() {
+    assert_wasi_output!(
+        "../../wasitests/wasi_sees_virtual_root.wasm",
+        "wasi_sees_virtual_root",
+        vec![],
+        vec![
+            (
+                "act1".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/hamlet/act1")
+            ),
+            (
+                "act2".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/hamlet/act2")
+            ),
+            (
+                "act1-again".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/hamlet/act1")
+            ),
+        ],
+        vec![],
+        "../../wasitests/wasi_sees_virtual_root.out"
+    );
+}

lib/wasi-tests/tests/wasitests/writing.rs

@@ -0,0 +1,24 @@
+#[test]
+fn test_writing() {
+    assert_wasi_output!(
+        "../../wasitests/writing.wasm",
+        "writing",
+        vec![],
+        vec![
+            (
+                "act1".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/hamlet/act1")
+            ),
+            (
+                "act2".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/hamlet/act2")
+            ),
+            (
+                "act1-again".to_string(),
+                ::std::path::PathBuf::from("wasitests/test_fs/hamlet/act1")
+            ),
+        ],
+        vec![],
+        "../../wasitests/writing.out"
+    );
+}

lib/wasi-tests/wasitests/create_dir.rs

@@ -1,3 +1,6 @@
+// Args:
+// dir: .
+
 use std::fs;
 use std::io::{Read, Seek, SeekFrom, Write};
 use std::path::*;

lib/wasi-tests/wasitests/file_metadata.out

@@ -1,3 +1,3 @@
 is dir: false
 filetype: false true false
-file info: 456
+file info: 476

lib/wasi-tests/wasitests/file_metadata.rs

@@ -1,3 +1,6 @@
+// Args:
+// dir: .
+
 use std::fs;
 use std::io::Read;
 

lib/wasi-tests/wasitests/ignores.txt

@@ -1 +1 @@
-create_dir
+

lib/wasi-tests/wasitests/mapdir.out

@@ -4,3 +4,4 @@
 "./act3"
 "./act4"
 "./act5"
+"./bookmarks"

lib/wasi-tests/wasitests/mapdir.rs

@@ -4,8 +4,6 @@
 use std::fs;
 
 fn main() {
-    #[cfg(not(target_os = "wasi"))]
-    let cur_dir = std::env::current_dir().unwrap();
     #[cfg(not(target_os = "wasi"))]
     std::env::set_current_dir("wasitests/test_fs/hamlet").unwrap();
 
@@ -19,7 +17,4 @@ fn main() {
     for p in out {
         println!("{}", p);
     }
-    // return to the current directory
-    #[cfg(not(target_os = "wasi"))]
-    std::env::set_current_dir(cur_dir).unwrap();
 }

lib/wasi-tests/wasitests/quine.out

@@ -1,3 +1,6 @@
+// Args:
+// dir: .
+
 use std::fs;
 use std::io::Read;
 

lib/wasi-tests/wasitests/quine.rs

@@ -1,3 +1,6 @@
+// Args:
+// dir: .
+
 use std::fs;
 use std::io::Read;
 

lib/wasi-tests/wasitests/readlink.out

@@ -0,0 +1,4 @@
+../act1/scene2.txt
+SCENE II. A room of state in the castle.
+
+    Enter KING CLAUDIUS, QUEEN GERTRUDE, HAMLET, POLONIUS, LAERTES, VOLTIMAND, CORNELI

lib/wasi-tests/wasitests/readlink.rs

@@ -0,0 +1,21 @@
+// Args:
+// mapdir: .:wasitests/test_fs/hamlet
+use std::io::Read;
+
+fn main() {
+    #[cfg(not(target_os = "wasi"))]
+    std::env::set_current_dir("wasitests/test_fs/hamlet").unwrap();
+
+    let sym_link_path = "bookmarks/2019-07-16";
+
+    let link_path = std::fs::read_link(sym_link_path).unwrap();
+    println!("{}", link_path.to_string_lossy());
+
+    let mut some_contents = std::fs::File::open(sym_link_path).unwrap();
+
+    let mut buffer = [0; 128];
+
+    assert_eq!(some_contents.read(&mut buffer).unwrap(), 128);
+    let str_val = std::str::from_utf8(&buffer[..]).unwrap();
+    println!("{}", str_val);
+}

lib/wasi-tests/wasitests/test_fs/hamlet/bookmarks/2019-07-16

@@ -0,0 +1 @@
+../act1/scene2.txt

lib/wasi-tests/wasitests/wasi_sees_virtual_root.out

@@ -0,0 +1,13 @@
+"/act1"
+"/act1-again"
+"/act2"
+"/act1"
+"/act1-again"
+"/act2"
+"/act1"
+"/act1-again"
+"/act2"
+"/act1"
+"/act1-again"
+"/act2"
+ROOT IS SAFE