Merge pull request #56 from holochain/2021-04-27-singlepass

2021 04 27 singlepass

CHANGELOG-UNRELEASED.md

@@ -6,8 +6,17 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ### Added
 
+- `HostShortCircuit` variant for `WasmError`
+- moved a lot of memory handling to the `WasmerEnv` handling
+- added a simple `MODULE_CACHE` as a status
+
 ### Changed
 
+- Uses wasmer 1+
+- Uses latest holonix
+- Externs follow (ptr, len) -> ptrlen as (u32, u32) -> u64
+- all guest functions are `#[inline(always)]`
+
 ### Deprecated
 
 ### Removed

crates/common/src/lib.rs

@@ -24,3 +24,45 @@ pub type Len = WasmSize;
 /// a WasmSize integer that points to a position in wasm linear memory that the host and guest are
 /// sharing to communicate across function calls
 pub type GuestPtr = WasmSize;
+
+pub type GuestPtrLen = u64;
+
+pub fn split_u64(u: GuestPtrLen) -> (GuestPtr, Len) {
+    let bytes = u.to_le_bytes();
+    (
+        GuestPtr::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]),
+        Len::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]),
+    )
+}
+
+pub fn merge_u64(guest_ptr: GuestPtr, len: Len) -> GuestPtrLen {
+    let guest_ptr_bytes = guest_ptr.to_le_bytes();
+    let len_bytes = len.to_le_bytes();
+    GuestPtrLen::from_le_bytes([
+        guest_ptr_bytes[0],
+        guest_ptr_bytes[1],
+        guest_ptr_bytes[2],
+        guest_ptr_bytes[3],
+        len_bytes[0],
+        len_bytes[1],
+        len_bytes[2],
+        len_bytes[3],
+    ])
+}
+
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+
+    #[test]
+    fn round_trip() {
+        let guest_ptr = 9000000 as GuestPtr;
+        let len = 1000 as GuestPtr;
+
+        let (out_guest_ptr, out_len) = split_u64(merge_u64(guest_ptr, len));
+
+        assert_eq!(guest_ptr, out_guest_ptr,);
+
+        assert_eq!(len, out_len,);
+    }
+}

crates/common/src/result.rs

@@ -33,6 +33,9 @@ pub enum WasmError {
     Guest(String),
     /// Something to do with host logic that we don't know about
     Host(String),
+    /// Something to do with host logic that we don't know about
+    /// AND wasm execution MUST immediately halt.
+    HostShortCircuit(String),
     /// Somehow wasmer failed to compile machine code from wasm byte code
     Compile(String),
 

crates/guest/Cargo.toml

@@ -18,3 +18,4 @@ holochain_serialized_bytes = { version = "=0.0.50", features = [] }
 holochain_wasmer_common = { version = "=0.0.67", path = "../common" }
 serde = "=1.0.123"
 tracing = "0.1"
+parking_lot = "0.11.1"

crates/guest/src/allocation.rs

@@ -1,68 +1,34 @@
 use holochain_wasmer_common::*;
-use std::mem;
-
-/// Attempt to extract the length at the given guest_ptr.
-//. Note that the guest_ptr could point at garbage and the "length prefix" would be garbage and
-//. then some arbitrary memory would be referenced so not erroring does not imply safety.
-pub fn length_prefix_at_guest_ptr(guest_ptr: GuestPtr) -> Len {
-    let len_bytes: &[u8] =
-        unsafe { std::slice::from_raw_parts(guest_ptr as *const u8, std::mem::size_of::<Len>()) };
-    u32::from_le_bytes([len_bytes[0], len_bytes[1], len_bytes[2], len_bytes[3]])
-}
 
+/// Allocate bytes that won't be dropped by the allocator.
+/// Return the pointer to the leaked allocation so the host can write to it.
 #[no_mangle]
-/// Allocate a length __plus a length prefix__ in bytes that won't be dropped by the allocator.
-/// Return the pointer to it so a length prefix + bytes can be written to the allocation.
+#[inline(always)]
 pub extern "C" fn __allocate(len: Len) -> GuestPtr {
-    let dummy: Vec<u8> = Vec::with_capacity((len + std::mem::size_of::<Len>() as Len) as usize);
+    let dummy: Vec<u8> = Vec::with_capacity(len as usize);
     let ptr = dummy.as_ptr() as GuestPtr;
-    let _ = mem::ManuallyDrop::new(dummy);
+    let _ = core::mem::ManuallyDrop::new(dummy);
     ptr
 }
 
-/// Free a length-prefixed allocation.
+/// Free an allocation.
 /// Needed because we leak memory every time we call `__allocate` and `write_bytes`.
 #[no_mangle]
-pub extern "C" fn __deallocate(guest_ptr: GuestPtr) {
-    // Failing to deallocate when requested is unrecoverable.
-    let len = length_prefix_at_guest_ptr(guest_ptr) + std::mem::size_of::<Len>() as Len;
+#[inline(always)]
+pub extern "C" fn __deallocate(guest_ptr: GuestPtr, len: Len) {
     let _: Vec<u8> =
         unsafe { Vec::from_raw_parts(guest_ptr as *mut u8, len as usize, len as usize) };
 }
 
-/// Attempt to consume bytes out of a length-prefixed allocation at the given pointer position.
+/// Attempt to consume bytes from a known guest_ptr and len.
 ///
 /// Consume in this context means take ownership of previously forgotten data.
 ///
 /// This needs to work for bytes written into the guest from the host and for bytes written with
 /// the write_bytes() function within the guest.
-pub fn consume_bytes(guest_ptr: GuestPtr) -> Vec<u8> {
-    // The Vec safety requirements are much stricter than a simple slice:
-    //
-    // - the pointer must have been generated with Vec/String on the same allocator
-    //   - yes: the guest always creates its own GuestPtr for vector allocations, note that if we
-    //     change the GuestPtr at all (e.g. to try and offset the length prefix) this will break,
-    //     we must use the _exact_ GuestPtr previously allocated
-    // - the pointer and T need the same size and alignment
-    //   - yes: we can cast GuestPtr to *mut u8
-    // - length needs to be compatible with capacity
-    //   - yes: we can set length and capacity equal
-    // - capacity needs to match capacity at the time of the pointer creation
-    //   - yes: this is trickier, what we _want_ is a vector of just the payload bytes without the
-    //     length prefix, but if we try to change either the capacity or the pointer to offset the
-    //     prefix directly then we end up with MemoryOutOfBounds exceptions down the line
-    // - @see https://doc.rust-lang.org/std/vec/struct.Vec.html#safety
-    //
-    // For example, this did _not_ work and leads to memory related panics down the line:
-    // let v: Vec<u8> = Vec::from_raw_parts(
-    //     (guest_ptr + std::mem::size_of::<Len>() as Len) as *mut u8,
-    //     len as usize,
-    //     (len + std::mem::size_of::<Len>() as Len) as usize,
-    // );
-
-    // We need the same length used to allocate the vector originally, so it includes the prefix
-    let len = length_prefix_at_guest_ptr(guest_ptr) + std::mem::size_of::<Len>() as Len;
-    let mut v: Vec<u8> = unsafe {
+#[inline(always)]
+pub fn consume_bytes(guest_ptr: GuestPtr, len: Len) -> Vec<u8> {
+    unsafe {
         Vec::from_raw_parts(
             // must match the pointer produced by the original allocation exactly
             guest_ptr as *mut u8,
@@ -71,41 +37,29 @@ pub fn consume_bytes(guest_ptr: GuestPtr) -> Vec<u8> {
             // must match the capacity set during the original allocation exactly
             len as usize,
         )
-    };
-
-    // This leads to an additional allocation for a new vector starting after the length prefix
-    // the old vector will be dropped and cleaned up by the allocator after this call
-    // the split off bytes will take ownership moving forward.
-    //
-    // Note that we could have tried to do something with std::slice::from_raw_parts() in this
-    // function but we'd still need a new allocation at the point of slice.to_vec() and then
-    // we'd need to manually free whatever the slice was pointing at.
-    v.split_off(std::mem::size_of::<Len>())
+    }
 }
 
-/// Attempt to write a slice of bytes into a length prefixed allocation.
+/// Attempt to write a slice of bytes.
 ///
 /// This is identical to the following:
 /// - host has some slice of bytes
 /// - host calls __allocate with the slice length
 /// - guest returns GuestPtr to the host
-/// - host writes a length prefix and the slice bytes into the guest at GuestPtr location
+/// - host writes the bytes into the guest at GuestPtr location
 /// - host hands the GuestPtr back to the guest
 ///
 /// In this case everything happens within the guest and a GuestPtr is returned if successful.
 ///
 /// This also leaks the written bytes, exactly like the above process.
 ///
 /// This facilitates the guest handing a GuestPtr back to the host as the _return_ value of guest
-/// functions so that the host can read the _output_ of guest logic from a length-prefixed pointer.
+/// functions so that the host can read the _output_ of guest logic from a pointer.
 ///
-/// A good host will call __deallocate with the GuestPtr produced here once it has read the bytes
-/// out of the guest, otherwise the bytes will be permanently leaked for the lifetime of the guest.
-pub fn write_bytes(slice: &[u8]) -> GuestPtr {
-    let len_bytes = slice.len().to_le_bytes();
-
-    let v: Vec<u8> = len_bytes.iter().chain(slice.iter()).cloned().collect();
+/// The host MUST ensure either __deallocate is called or the entire wasm memory is dropped.
+#[inline(always)]
+pub fn write_bytes(v: Vec<u8>) -> GuestPtr {
     let ptr: GuestPtr = v.as_ptr() as GuestPtr;
-    let _ = mem::ManuallyDrop::new(v);
+    let _ = core::mem::ManuallyDrop::new(v);
     ptr
 }