Auto-generate wasi-types from .wit files

Cargo.toml

@@ -44,6 +44,8 @@ members = [
     "lib/vm",
     "lib/wasi",
     "lib/wasi-types",
+    "lib/wasi-types-generated",
+    "lib/wasi-types-generated/wasi-types-generator-extra",
     "lib/wasi-experimental-io-devices",
     "lib/wasi-local-networking",
     "lib/types",

lib/api/src/js/externals/memory_view.rs

@@ -58,6 +58,35 @@ impl<'a> MemoryView<'a> {
         self.size
     }
 
+    // TODO: do we want a proper implementation here instead?
+    /// Retrieve a slice of the memory contents.
+    ///
+    /// # Safety
+    ///
+    /// Until the returned slice is dropped, it is undefined behaviour to
+    /// modify the memory contents in any way including by calling a wasm
+    /// function that writes to the memory or by resizing the memory.
+    #[doc(hidden)]
+    pub unsafe fn data_unchecked(&self) -> &[u8] {
+        unimplemented!("direct data pointer access is not possible in JavaScript");
+    }
+
+    // TODO: do we want a proper implementation here instead?
+    /// Retrieve a mutable slice of the memory contents.
+    ///
+    /// # Safety
+    ///
+    /// This method provides interior mutability without an UnsafeCell. Until
+    /// the returned value is dropped, it is undefined behaviour to read or
+    /// write to the pointed-to memory in any way except through this slice,
+    /// including by calling a wasm function that reads the memory contents or
+    /// by resizing this Memory.
+    #[allow(clippy::mut_from_ref)]
+    #[doc(hidden)]
+    pub unsafe fn data_unchecked_mut(&self) -> &mut [u8] {
+        unimplemented!("direct data pointer access is not possible in JavaScript");
+    }
+
     /// Returns the size (in [`Pages`]) of the `Memory`.
     ///
     /// # Example

lib/api/src/sys/module.rs

@@ -234,7 +234,9 @@ impl Module {
     }
 
     /// Serializes a module into a binary representation that the `Engine`
-    /// can later process via [`Module::deserialize`].
+    /// can later process via
+    #[cfg_attr(feature = "compiler", doc = "[`Module::deserialize`].")]
+    #[cfg_attr(not(feature = "compiler"), doc = "`Module::deserialize`.")]
     ///
     /// # Usage
     ///
@@ -252,7 +254,9 @@ impl Module {
     }
 
     /// Serializes a module into a file that the `Engine`
-    /// can later process via [`Module::deserialize_from_file`].
+    /// can later process via
+    #[cfg_attr(feature = "compiler", doc = "[`Module::deserialize_from_file`].")]
+    #[cfg_attr(not(feature = "compiler"), doc = "`Module::deserialize_from_file`.")]
     ///
     /// # Usage
     ///

lib/api/src/sys/store.rs

@@ -26,7 +26,9 @@ pub(crate) struct StoreInner {
 ///
 /// The `Store` holds the engine (that is —amongst many things— used to compile
 /// the Wasm bytes into a valid module artifact), in addition to the
-/// [`Tunables`] (that are used to create the memories, tables and globals).
+#[cfg_attr(feature = "compiler", doc = "[`Tunables`]")]
+#[cfg_attr(not(feature = "compiler"), doc = "`Tunables`")]
+/// (that are used to create the memories, tables and globals).
 ///
 /// Spec: <https://webassembly.github.io/spec/core/exec/runtime.html#store>
 pub struct Store {

lib/types/src/value.rs

@@ -98,6 +98,7 @@ macro_rules! primitives {
     )*)
 }
 primitives! {
+    bool
     i8 u8
     i16 u16
     i32 u32

lib/wasi-experimental-io-devices/Cargo.toml

@@ -16,8 +16,8 @@ maintenance = { status = "experimental" }
 [dependencies]
 wasmer-wasi = { version = "=3.0.0-beta", path = "../wasi", default-features=false }
 tracing = "0.1"
-minifb = { version = "0.19", optional = true }
-nix = "0.20.2"
+minifb = { version = "0.23", optional = true }
+nix = "0.25.0"
 ref_thread_local = "0.1"
 serde = "1"
 typetag = "0.1"

lib/wasi-experimental-io-devices/src/link-ext.rs

@@ -5,7 +5,10 @@ use std::collections::{BTreeSet, VecDeque};
 use std::convert::TryInto;
 use std::io::{Read, Seek, SeekFrom, Write};
 use tracing::debug;
-use wasmer_wasi::{types::*, WasiInodes};
+use wasmer_wasi::{
+    types::{wasi::Filesize, *},
+    WasiInodes,
+};
 use wasmer_wasi::{Fd, VirtualFile, WasiFs, WasiFsError, ALL_RIGHTS, VIRTUAL_ROOT_FD};
 
 use minifb::{Key, KeyRepeat, MouseButton, Scale, Window, WindowOptions};
@@ -417,7 +420,7 @@ impl VirtualFile for FrameBuffer {
     fn size(&self) -> u64 {
         0
     }
-    fn set_len(&mut self, _new_size: __wasi_filesize_t) -> Result<(), WasiFsError> {
+    fn set_len(&mut self, _new_size: Filesize) -> Result<(), WasiFsError> {
         Ok(())
     }
     fn unlink(&mut self) -> Result<(), WasiFsError> {

lib/wasi-types-generated/.gitignore

@@ -0,0 +1 @@
+wit-bindgen/

lib/wasi-types-generated/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "wasmer-wasi-types-generated"
+version = "3.0.0-beta"
+edition = "2018"
+license = "MIT"
+description = "Autogenerated wasi types, generated from .wit files"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+wasmer = { default-features = false, path = "../api", version = "3.0.0-beta" }
+wit-bindgen-rust = { package = "wasmer-wit-bindgen-rust", version = "0.1.1" }
+wit-bindgen-rust-wasm = { package = "wasmer-wit-bindgen-gen-rust-wasm", version = "0.1.1" }
+wit-bindgen-core = { package = "wasmer-wit-bindgen-gen-core", version = "0.1.1" }
+wit-parser = { package = "wasmer-wit-parser", version = "0.1.1" }
+
+[dev-dependencies.pretty_assertions]
+version = "1.3.0"
+
+[features]
+js = ["wasmer/js"]
+sys = ["wasmer/sys"]

lib/wasi-types-generated/README.md

@@ -0,0 +1,5 @@
+Run `regenerate.sh` to regenerate the wasi-types from
+the `wasi-clean/typenames.wit` into the final Rust bindings.
+
+The `wasi-types-generator-extra` generates some extra code
+that wit-bindgen currently can't provide.

lib/wasi-types-generated/regenerate.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+BASEDIR=$(dirname "$0")
+
+rm -f \
+    "$BASEDIR"/src/bindings.rs \
+    "$BASEDIR"/src/*/bindings.rs
+
+cat "$BASEDIR"/wit-clean/typenames.wit "$BASEDIR"/wit-clean/wasi_unstable.wit > "$BASEDIR"/wit-clean/output.wit
+
+git clone https://github.com/wasmerio/wit-bindgen --branch force-generate-structs --single-branch
+git pull origin force-generate-structs
+cd wit-bindgen
+cargo build
+cd ..
+
+./wit-bindgen/target/debug/wit-bindgen rust-wasm \
+    --import "$BASEDIR"/wit-clean/output.wit \
+    --force-generate-structs \
+    --out-dir "$BASEDIR"/src/wasi \
+
+awk '{sub(/mod output/,"pub mod output")}1' src/wasi/bindings.rs > src/wasi/bindings2.rs
+cargo fmt --all
+cp src/wasi/bindings2.rs src/wasi/bindings.rs
+rm src/wasi/bindings2.rs
+
+cd ./wasi-types-generator-extra
+cargo build
+pwd
+../../../target/debug/wasi-types-generator-extra
+cd ..
+
+cargo fmt --all

lib/wasi-types-generated/src/lib.rs

@@ -0,0 +1,42 @@
+pub mod wasi;
+
+// Prevent the CI from passing if the wasi/bindings.rs is not
+// up to date with the output.wit file
+#[test]
+#[cfg(feature = "sys")]
+fn fail_if_wit_files_arent_up_to_date() {
+    use wit_bindgen_core::Generator;
+
+    let output_wit = concat!(env!("CARGO_MANIFEST_DIR"), "/wit-clean/output.wit");
+    let bindings_target =
+        include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/src/wasi/bindings.rs"));
+    let mut generator = wit_bindgen_rust_wasm::Opts {
+        ..wit_bindgen_rust_wasm::Opts::default()
+    }
+    .build();
+    let output_wit_parsed = wit_parser::Interface::parse_file(output_wit).unwrap();
+    let imports = vec![output_wit_parsed];
+    let exports = vec![];
+    let mut files = Default::default();
+    generator.generate_all(
+        &imports, &exports, &mut files, /* generate_structs */ true,
+    );
+    let generated = files
+        .iter()
+        .filter_map(|(k, v)| if k == "bindings.rs" { Some(v) } else { None })
+        .next()
+        .unwrap();
+    let generated_str = String::from_utf8_lossy(generated);
+    let generated_str = generated_str
+        .lines()
+        .map(|s| s.to_string())
+        .collect::<Vec<String>>()
+        .join("\r\n");
+    let generated_str = generated_str.replace("mod output {", "pub mod output {");
+    let bindings_target = bindings_target
+        .lines()
+        .map(|s| s.to_string())
+        .collect::<Vec<String>>()
+        .join("\r\n");
+    pretty_assertions::assert_eq!(generated_str, bindings_target); // output.wit out of date? regenerate bindings.rs
+}