diff --git a/lib/api/src/access.rs b/lib/api/src/access.rs new file mode 100644 index 00000000000..da34c8ec407 --- /dev/null +++ b/lib/api/src/access.rs @@ -0,0 +1,229 @@ +use std::mem::MaybeUninit; + +use crate::{WasmRef, WasmSlice}; + +pub(super) enum SliceCow<'a, T> { + #[allow(dead_code)] + Borrowed(&'a mut [T]), + #[allow(dead_code)] + Owned(Vec, bool), +} + +impl<'a, T> AsRef<[T]> for SliceCow<'a, T> { + fn as_ref(&self) -> &[T] { + match self { + Self::Borrowed(buf) => buf, + Self::Owned(buf, _) => buf, + } + } +} + +impl<'a, T> AsMut<[T]> for SliceCow<'a, T> { + fn as_mut(&mut self) -> &mut [T] { + // Note: Zero padding is not required here as its a typed copy which does + // not leak the bytes into the memory + // https://stackoverflow.com/questions/61114026/does-stdptrwrite-transfer-the-uninitialized-ness-of-the-bytes-it-writes + match self { + Self::Borrowed(buf) => buf, + Self::Owned(buf, modified) => { + *modified = true; + buf.as_mut() + } + } + } +} + +/// Provides direct memory access to a piece of memory that +/// is owned by WASM +pub struct WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + pub(super) slice: WasmSlice<'a, T>, + pub(super) buf: SliceCow<'a, T>, +} + +impl<'a, T> AsRef<[T]> for WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn as_ref(&self) -> &[T] { + self.buf.as_ref() + } +} + +impl<'a, T> AsMut<[T]> for WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn as_mut(&mut self) -> &mut [T] { + self.buf.as_mut() + } +} + +impl<'a, T> WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + /// Returns an iterator of all the elements in the slice + pub fn iter(&'a self) -> std::slice::Iter<'a, T> { + self.as_ref().iter() + } + + /// Returns an iterator of all the elements in the slice + pub fn iter_mut(&'a mut self) -> std::slice::IterMut<'a, T> { + self.buf.as_mut().iter_mut() + } + + /// Number of elements in this slice + pub fn len(&self) -> usize { + self.buf.as_ref().len() + } + + /// If the slice is empty + pub fn is_empty(&self) -> bool { + self.buf.as_ref().is_empty() + } +} + +impl<'a> WasmSliceAccess<'a, u8> { + /// Writes to the address pointed to by this `WasmPtr` in a memory. + #[inline] + pub fn copy_from_slice(&mut self, src: &[u8]) { + let dst = self.buf.as_mut(); + dst.copy_from_slice(src); + } +} + +impl<'a, T> Drop for WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn drop(&mut self) { + if let SliceCow::Owned(buf, modified) = &self.buf { + if *modified { + self.slice.write_slice(buf.as_ref()).ok(); + } + } + } +} + +pub(super) enum RefCow<'a, T> { + #[allow(dead_code)] + Borrowed(&'a mut T), + #[allow(dead_code)] + Owned(T, bool), +} + +impl<'a, T> AsRef for RefCow<'a, T> { + fn as_ref(&self) -> &T { + match self { + Self::Borrowed(val) => *val, + Self::Owned(val, _) => val, + } + } +} + +impl<'a, T> AsMut for RefCow<'a, T> { + fn as_mut(&mut self) -> &mut T { + // Note: Zero padding is not required here as its a typed copy which does + // not leak the bytes into the memory + // https://stackoverflow.com/questions/61114026/does-stdptrwrite-transfer-the-uninitialized-ness-of-the-bytes-it-writes + match self { + Self::Borrowed(val) => *val, + Self::Owned(val, modified) => { + *modified = true; + val + } + } + } +} + +/// Provides direct memory access to a piece of memory that +/// is owned by WASM +pub struct WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + pub(super) ptr: WasmRef<'a, T>, + pub(super) buf: RefCow<'a, T>, +} + +impl<'a, T> AsRef for WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn as_ref(&self) -> &T { + self.buf.as_ref() + } +} + +impl<'a, T> AsMut for WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn as_mut(&mut self) -> &mut T { + self.buf.as_mut() + } +} + +impl<'a, T> WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + /// Reads the address pointed to by this `WasmPtr` in a memory. + #[inline] + #[allow(clippy::clone_on_copy)] + pub fn read(&self) -> T + where + T: Clone, + { + self.as_ref().clone() + } + + /// Writes to the address pointed to by this `WasmPtr` in a memory. + #[inline] + pub fn write(&mut self, val: T) { + // Note: Zero padding is not required here as its a typed copy which does + // not leak the bytes into the memory + // https://stackoverflow.com/questions/61114026/does-stdptrwrite-transfer-the-uninitialized-ness-of-the-bytes-it-writes + *(self.as_mut()) = val; + } +} + +impl<'a, T> Drop for WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn drop(&mut self) { + if let RefCow::Owned(val, modified) = &self.buf { + if *modified { + self.ptr.write(*val).ok(); + } + } + } +} + +impl<'a, T> WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + /// Returns a mutable slice that is not yet initialized + pub fn as_mut_uninit(&mut self) -> &mut [MaybeUninit] { + let ret: &mut [T] = self.buf.as_mut(); + let ret: &mut [MaybeUninit] = unsafe { std::mem::transmute(ret) }; + ret + } +} + +impl<'a, T> WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + /// Returns a reference to an unitialized reference to this value + pub fn as_mut_uninit(&mut self) -> &mut MaybeUninit { + let ret: &mut T = self.buf.as_mut(); + let ret: &mut MaybeUninit = unsafe { std::mem::transmute(ret) }; + ret + } +} diff --git a/lib/api/src/js/mem_access.rs b/lib/api/src/js/mem_access.rs index de82f5c6e1c..5c7aeaa7831 100644 --- a/lib/api/src/js/mem_access.rs +++ b/lib/api/src/js/mem_access.rs @@ -1,6 +1,8 @@ +use crate::access::{RefCow, SliceCow, WasmRefAccess}; use crate::js::externals::memory::MemoryBuffer; use crate::js::RuntimeError; use crate::js::{Memory32, Memory64, MemoryView, WasmPtr}; +use crate::WasmSliceAccess; use std::{ convert::TryInto, fmt, @@ -121,6 +123,12 @@ impl<'a, T: ValueType> WasmRef<'a, T> { let data = unsafe { slice::from_raw_parts(data.as_ptr() as *const _, data.len()) }; self.buffer.write(self.offset, data) } + + /// Gains direct access to the memory of this slice + #[inline] + pub fn access(self) -> Result, MemoryAccessError> { + WasmRefAccess::new(self) + } } impl<'a, T: ValueType> fmt::Debug for WasmRef<'a, T> { @@ -250,6 +258,12 @@ impl<'a, T: ValueType> WasmSlice<'a, T> { self.index(idx).write(val) } + /// Gains direct access to the memory of this slice + #[inline] + pub fn access(self) -> Result, MemoryAccessError> { + WasmSliceAccess::new(self) + } + /// Reads the entire slice into the given buffer. /// /// The length of the buffer must match the length of the slice. @@ -405,3 +419,29 @@ impl<'a, T: ValueType> DoubleEndedIterator for WasmSliceIter<'a, T> { } impl<'a, T: ValueType> ExactSizeIterator for WasmSliceIter<'a, T> {} + +impl<'a, T> WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn new(slice: WasmSlice<'a, T>) -> Result { + let buf = slice.read_to_vec()?; + Ok(Self { + slice, + buf: SliceCow::Owned(buf, false), + }) + } +} + +impl<'a, T> WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn new(ptr: WasmRef<'a, T>) -> Result { + let val = ptr.read()?; + Ok(Self { + ptr, + buf: RefCow::Owned(val, false), + }) + } +} diff --git a/lib/api/src/lib.rs b/lib/api/src/lib.rs index 369a8213e53..73dea73bd48 100644 --- a/lib/api/src/lib.rs +++ b/lib/api/src/lib.rs @@ -440,5 +440,7 @@ mod js; #[cfg(feature = "js")] pub use js::*; +mod access; mod into_bytes; +pub use access::WasmSliceAccess; pub use into_bytes::IntoBytes; diff --git a/lib/api/src/sys/mem_access.rs b/lib/api/src/sys/mem_access.rs index 3075ba60e21..239597eb257 100644 --- a/lib/api/src/sys/mem_access.rs +++ b/lib/api/src/sys/mem_access.rs @@ -1,4 +1,7 @@ -use crate::RuntimeError; +use crate::{ + access::{RefCow, SliceCow, WasmRefAccess}, + RuntimeError, WasmSliceAccess, +}; #[allow(unused_imports)] use crate::{Memory, Memory32, Memory64, MemorySize, MemoryView, WasmPtr}; use std::{ @@ -102,26 +105,20 @@ impl<'a, T: ValueType> WasmRef<'a, T> { /// Reads the location pointed to by this `WasmRef`. #[inline] pub fn read(self) -> Result { - let mut out = MaybeUninit::uninit(); - let buf = - unsafe { slice::from_raw_parts_mut(out.as_mut_ptr() as *mut u8, mem::size_of::()) }; - self.buffer.read(self.offset, buf)?; - Ok(unsafe { out.assume_init() }) + Ok(self.access()?.read()) } /// Writes to the location pointed to by this `WasmRef`. #[inline] pub fn write(self, val: T) -> Result<(), MemoryAccessError> { - let mut data = MaybeUninit::new(val); - let data = unsafe { - slice::from_raw_parts_mut( - data.as_mut_ptr() as *mut MaybeUninit, - mem::size_of::(), - ) - }; - val.zero_padding_bytes(data); - let data = unsafe { slice::from_raw_parts(data.as_ptr() as *const _, data.len()) }; - self.buffer.write(self.offset, data) + self.access()?.write(val); + Ok(()) + } + + /// Gains direct access to the memory of this slice + #[inline] + pub fn access(self) -> Result, MemoryAccessError> { + WasmRefAccess::new(self) } } @@ -240,6 +237,12 @@ impl<'a, T: ValueType> WasmSlice<'a, T> { WasmSliceIter { slice: self } } + /// Gains direct access to the memory of this slice + #[inline] + pub fn access(self) -> Result, MemoryAccessError> { + WasmSliceAccess::new(self) + } + /// Reads an element of this slice. #[inline] pub fn read(self, idx: u64) -> Result { @@ -404,3 +407,66 @@ impl<'a, T: ValueType> DoubleEndedIterator for WasmSliceIter<'a, T> { } impl<'a, T: ValueType> ExactSizeIterator for WasmSliceIter<'a, T> {} + +impl<'a, T> WasmSliceAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn new(slice: WasmSlice<'a, T>) -> Result { + let total_len = slice + .len + .checked_mul(mem::size_of::() as u64) + .ok_or(MemoryAccessError::Overflow)?; + let end = slice + .offset + .checked_add(total_len) + .ok_or(MemoryAccessError::Overflow)?; + if end > slice.buffer.len as u64 { + #[cfg(feature = "tracing")] + warn!( + "attempted to read ({} bytes) beyond the bounds of the memory view ({} > {})", + total_len, end, slice.buffer.len + ); + return Err(MemoryAccessError::HeapOutOfBounds); + } + let buf = unsafe { + let buf_ptr: *mut u8 = slice.buffer.base.add(slice.offset as usize); + let buf_ptr: *mut T = std::mem::transmute(buf_ptr); + std::slice::from_raw_parts_mut(buf_ptr, slice.len as usize) + }; + Ok(Self { + slice, + buf: SliceCow::Borrowed(buf), + }) + } +} + +impl<'a, T> WasmRefAccess<'a, T> +where + T: wasmer_types::ValueType, +{ + fn new(ptr: WasmRef<'a, T>) -> Result { + let total_len = mem::size_of::() as u64; + let end = ptr + .offset + .checked_add(total_len) + .ok_or(MemoryAccessError::Overflow)?; + if end > ptr.buffer.len as u64 { + #[cfg(feature = "tracing")] + warn!( + "attempted to read ({} bytes) beyond the bounds of the memory view ({} > {})", + total_len, end, ptr.buffer.len + ); + return Err(MemoryAccessError::HeapOutOfBounds); + } + let val = unsafe { + let val_ptr: *mut u8 = ptr.buffer.base.add(ptr.offset as usize); + let val_ptr: *mut T = std::mem::transmute(val_ptr); + &mut *val_ptr + }; + Ok(Self { + ptr, + buf: RefCow::Borrowed(val), + }) + } +} diff --git a/lib/api/src/sys/ptr.rs b/lib/api/src/sys/ptr.rs index 285688b8fff..371a9cc7eb0 100644 --- a/lib/api/src/sys/ptr.rs +++ b/lib/api/src/sys/ptr.rs @@ -1,3 +1,4 @@ +use crate::access::WasmRefAccess; use crate::sys::{externals::MemoryView, FromToNativeWasmType}; use crate::NativeWasmTypeInto; use crate::{MemoryAccessError, WasmRef, WasmSlice}; @@ -193,6 +194,15 @@ impl WasmPtr { } Ok(vec) } + + /// Creates a `WasmAccess` + #[inline] + pub fn access<'a>( + &self, + view: &'a MemoryView, + ) -> Result, MemoryAccessError> { + self.deref(view).access() + } } impl WasmPtr { diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 786d20e3610..7decf09c072 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -165,7 +165,7 @@ llvm = [ "compiler", ] debug = ["fern", "wasmer-wasi/logging"] -disable-all-logging = ["wasmer-wasi/disable-all-logging"] +disable-all-logging = ["wasmer-wasi/disable-all-logging", "log/release_max_level_off"] headless = [] headless-minimal = ["headless", "disable-all-logging", "wasi"] diff --git a/lib/wasi/src/state/builder.rs b/lib/wasi/src/state/builder.rs index 4957fb5ea99..e06e5c791eb 100644 --- a/lib/wasi/src/state/builder.rs +++ b/lib/wasi/src/state/builder.rs @@ -755,6 +755,13 @@ impl WasiEnvBuilder { env.data(store).thread.set_status_running(); let res = crate::run_wasi_func_start(start, store); + tracing::trace!( + "wasi[{}:{}]::main exit (code = {:?})", + env.data(store).pid(), + env.data(store).tid(), + res + ); + let exit_code = match &res { Ok(_) => 0, Err(err) => err.as_exit_code().unwrap_or(1), diff --git a/lib/wasi/src/syscalls/mod.rs b/lib/wasi/src/syscalls/mod.rs index 1aa18162409..685be1653d5 100644 --- a/lib/wasi/src/syscalls/mod.rs +++ b/lib/wasi/src/syscalls/mod.rs @@ -26,7 +26,6 @@ pub use wasix::*; pub mod legacy; -use std::mem::MaybeUninit; pub(crate) use std::{ borrow::{Borrow, Cow}, cell::RefCell, @@ -47,6 +46,7 @@ pub(crate) use std::{ thread::LocalKey, time::Duration, }; +use std::{io::IoSlice, mem::MaybeUninit}; pub(crate) use bytes::{Bytes, BytesMut}; pub(crate) use cooked_waker::IntoWaker; @@ -158,32 +158,6 @@ pub(crate) fn write_bytes( result } -pub(crate) fn copy_to_slice( - memory: &MemoryView, - iovs_arr_cell: WasmSlice<__wasi_ciovec_t>, - mut write_loc: &mut [MaybeUninit], -) -> Result { - let mut bytes_written = 0usize; - for iov in iovs_arr_cell.iter() { - let iov_inner = iov.read().map_err(mem_error_to_wasi)?; - - let amt = from_offset::(iov_inner.buf_len)?; - - let (left, right) = write_loc.split_at_mut(amt); - let bytes = WasmPtr::::new(iov_inner.buf) - .slice(memory, iov_inner.buf_len) - .map_err(mem_error_to_wasi)?; - - if amt != bytes.read_to_slice(left).map_err(mem_error_to_wasi)? { - return Err(Errno::Fault); - } - - write_loc = right; - bytes_written += amt; - } - Ok(bytes_written) -} - pub(crate) fn copy_from_slice( mut read_loc: &[u8], memory: &MemoryView, @@ -191,20 +165,21 @@ pub(crate) fn copy_from_slice( ) -> Result { let mut bytes_read = 0usize; - for iov in iovs_arr.iter() { - let iov_inner = iov.read().map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let mut buf = WasmPtr::::new(iovs.buf) + .slice(memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; - let to_read = from_offset::(iov_inner.buf_len)?; + let to_read = from_offset::(iovs.buf_len)?; let to_read = to_read.min(read_loc.len()); if to_read == 0 { break; } let (left, right) = read_loc.split_at(to_read); - - let buf = WasmPtr::::new(iov_inner.buf) - .slice(memory, to_read.try_into().map_err(|_| Errno::Overflow)?) - .map_err(mem_error_to_wasi)?; - buf.write_slice(left).map_err(mem_error_to_wasi)?; + buf.copy_from_slice(left); read_loc = right; bytes_read += to_read; @@ -219,21 +194,17 @@ pub(crate) fn read_bytes( ) -> Result { let mut bytes_read = 0usize; - // We allocate the raw_bytes first once instead of - // N times in the loop. - let mut raw_bytes: Vec = vec![0; 10240]; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let mut buf = WasmPtr::::new(iovs.buf) + .slice(memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; - for iov in iovs_arr.iter() { - let iov_inner = iov.read().map_err(mem_error_to_wasi)?; - raw_bytes.clear(); - let to_read = from_offset::(iov_inner.buf_len)?; - raw_bytes.resize(to_read, 0); - let has_read = reader.read(&mut raw_bytes).map_err(map_io_err)?; + let to_read = buf.len(); + let has_read = reader.read(buf.as_mut()).map_err(map_io_err)?; - let buf = WasmPtr::::new(iov_inner.buf) - .slice(memory, iov_inner.buf_len) - .map_err(mem_error_to_wasi)?; - buf.write_slice(&raw_bytes).map_err(mem_error_to_wasi)?; bytes_read += has_read; if has_read != to_read { return Ok(bytes_read); diff --git a/lib/wasi/src/syscalls/wasi/fd_read.rs b/lib/wasi/src/syscalls/wasi/fd_read.rs index 09ab495236a..91cf7eeae4b 100644 --- a/lib/wasi/src/syscalls/wasi/fd_read.rs +++ b/lib/wasi/src/syscalls/wasi/fd_read.rs @@ -160,8 +160,9 @@ fn fd_read_internal( ) -> Result, WasiError> { wasi_try_ok_ok!(WasiEnv::process_signals_and_exit(ctx)?); - let mut env = ctx.data(); - let state = env.state.clone(); + let env = ctx.data(); + let memory = env.memory_view(&ctx); + let state = env.state(); let fd_entry = wasi_try_ok_ok!(state.fs.get_fd(fd)); let is_stdio = fd_entry.is_stdio; @@ -175,19 +176,6 @@ fn fd_read_internal( let inode = fd_entry.inode; let fd_flags = fd_entry.flags; - let max_size = { - let memory = env.memory_view(ctx); - let iovs_arr = wasi_try_mem_ok_ok!(iovs.slice(&memory, iovs_len)); - let mut max_size = 0usize; - for iovs in iovs_arr.iter() { - let iovs = wasi_try_mem_ok_ok!(iovs.read()); - let buf_len: usize = - wasi_try_ok_ok!(iovs.buf_len.try_into().map_err(|_| Errno::Overflow)); - max_size += buf_len; - } - max_size - }; - let (bytes_read, can_update_cursor) = { let mut guard = inode.write(); match guard.deref_mut() { @@ -196,8 +184,8 @@ fn fd_read_internal( let handle = handle.clone(); drop(guard); - let data = wasi_try_ok_ok!(__asyncify( - ctx, + let read = wasi_try_ok_ok!(__asyncify_light( + env, if fd_flags.contains(Fdflags::NONBLOCK) { Some(Duration::ZERO) } else { @@ -212,33 +200,40 @@ fn fd_read_internal( .map_err(map_io_err)?; } - let mut buf = Vec::with_capacity(max_size); - - let amt = handle.read_buf(&mut buf).await.map_err(|err| { - let err = From::::from(err); - match err { - Errno::Again => { - if is_stdio { - Errno::Badf - } else { - Errno::Again + let mut total_read = 0usize; + + let iovs_arr = + iovs.slice(&memory, iovs_len).map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let mut buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + + total_read += + handle.read(buf.as_mut()).await.map_err(|err| { + let err = From::::from(err); + match err { + Errno::Again => { + if is_stdio { + Errno::Badf + } else { + Errno::Again + } + } + a => a, } - } - a => a, - } - })?; - Ok(buf) + })?; + } + Ok(total_read) } )? .map_err(|err| match err { Errno::Timedout => Errno::Again, a => a, })); - env = ctx.data(); - - let memory = env.memory_view(&ctx); - let iovs_arr = wasi_try_mem_ok_ok!(iovs.slice(&memory, iovs_len)); - let read = wasi_try_ok_ok!(read_bytes(&data[..], &memory, iovs_arr)); (read, true) } else { return Ok(Err(Errno::Badf)); @@ -250,28 +245,31 @@ fn fd_read_internal( drop(guard); let tasks = env.tasks().clone(); - let res = __asyncify( - ctx, + let res = __asyncify_light( + env, if fd_flags.contains(Fdflags::NONBLOCK) { Some(Duration::ZERO) } else { None }, async { - let mut buf = Vec::with_capacity(max_size); - unsafe { - buf.set_len(max_size); + let mut total_read = 0usize; + + let iovs_arr = + iovs.slice(&memory, iovs_len).map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let mut buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + + total_read += socket + .recv(tasks.deref(), buf.as_mut_uninit(), fd_flags) + .await?; } - socket - .recv(tasks.deref(), &mut buf, fd_flags) - .await - .map(|amt| { - unsafe { - buf.set_len(amt); - } - let buf: Vec = unsafe { std::mem::transmute(buf) }; - buf - }) + Ok(total_read) }, )? .map_err(|err| match err { @@ -281,16 +279,7 @@ fn fd_read_internal( match res { Err(Errno::Connaborted) | Err(Errno::Connreset) => (0, false), res => { - let data = wasi_try_ok_ok!(res); - env = ctx.data(); - - let data_len = data.len(); - let mut reader = &data[..]; - let memory = env.memory_view(&ctx); - let iovs_arr = wasi_try_mem_ok_ok!(iovs.slice(&memory, iovs_len)); - let bytes_read = wasi_try_ok_ok!( - read_bytes(reader, &memory, iovs_arr).map(|_| data_len) - ); + let bytes_read = wasi_try_ok_ok!(res); (bytes_read, false) } } @@ -300,36 +289,37 @@ fn fd_read_internal( drop(guard); - let data = wasi_try_ok_ok!(__asyncify( - ctx, + let bytes_read = wasi_try_ok_ok!(__asyncify_light( + env, if fd_flags.contains(Fdflags::NONBLOCK) { Some(Duration::ZERO) } else { None }, async move { - // TODO: optimize with MaybeUninit - let mut data = vec![0u8; max_size]; - let amt = wasmer_vfs::AsyncReadExt::read(&mut pipe, &mut data[..]) - .await - .map_err(map_io_err)?; - data.truncate(amt); - Ok(data) + let mut total_read = 0usize; + + let iovs_arr = + iovs.slice(&memory, iovs_len).map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; + for iovs in iovs_arr.iter() { + let mut buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + + total_read += + wasmer_vfs::AsyncReadExt::read(&mut pipe, buf.as_mut()).await?; + } + Ok(total_read) } )? .map_err(|err| match err { Errno::Timedout => Errno::Again, a => a, })); - env = ctx.data(); - let data_len = data.len(); - let mut reader = &data[..]; - - let memory = env.memory_view(ctx); - let iovs_arr = wasi_try_mem_ok_ok!(iovs.slice(&memory, iovs_len)); - let bytes_read = - wasi_try_ok_ok!(read_bytes(reader, &memory, iovs_arr).map(|_| data_len)); (bytes_read, false) } Kind::Dir { .. } | Kind::Root { .. } => { @@ -364,12 +354,11 @@ fn fd_read_internal( // Yield until the notifications are triggered let tasks_inner = env.tasks().clone(); - let val = wasi_try_ok_ok!(__asyncify(ctx, None, async { poller.await })? + let val = wasi_try_ok_ok!(__asyncify_light(env, None, async { poller.await })? .map_err(|err| match err { Errno::Timedout => Errno::Again, a => a, })); - env = ctx.data(); let mut memory = env.memory_view(ctx); let reader = val.to_ne_bytes(); diff --git a/lib/wasi/src/syscalls/wasi/fd_write.rs b/lib/wasi/src/syscalls/wasi/fd_write.rs index eb500a03141..bda5bf0d18e 100644 --- a/lib/wasi/src/syscalls/wasi/fd_write.rs +++ b/lib/wasi/src/syscalls/wasi/fd_write.rs @@ -115,6 +115,8 @@ fn fd_write_internal( let fd_flags = fd_entry.flags; let (bytes_written, can_update_cursor) = { + let iovs_arr = wasi_try_mem_ok!(iovs_arr.access()); + let (mut memory, _) = env.get_memory_and_wasi_state(&ctx, 0); let mut guard = fd_entry.inode.write(); match guard.deref_mut() { @@ -123,18 +125,8 @@ fn fd_write_internal( let handle = handle.clone(); drop(guard); - let buf_len: M::Offset = iovs_arr - .iter() - .filter_map(|a| a.read().ok()) - .map(|a| a.buf_len) - .sum(); - let buf_len: usize = - wasi_try_ok!(buf_len.try_into().map_err(|_| Errno::Inval)); - let mut buf = Vec::with_capacity(buf_len); - wasi_try_ok!(write_bytes(&mut buf, &memory, iovs_arr)); - - let written = wasi_try_ok!(__asyncify( - &mut ctx, + let written = wasi_try_ok!(__asyncify_light( + env, if fd_entry.flags.contains(Fdflags::NONBLOCK) { Some(Duration::ZERO) } else { @@ -149,7 +141,17 @@ fn fd_write_internal( .map_err(map_io_err)?; } - handle.write(&buf[..]).await.map_err(map_io_err) + let mut written = 0usize; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + written += + handle.write(buf.as_ref()).await.map_err(map_io_err)?; + } + Ok(written) } )? .map_err(|err| match err { @@ -166,33 +168,32 @@ fn fd_write_internal( let socket = socket.clone(); drop(guard); - let buf_len: M::Offset = iovs_arr - .iter() - .filter_map(|a| a.read().ok()) - .map(|a| a.buf_len) - .sum(); - let buf_len: usize = wasi_try_ok!(buf_len.try_into().map_err(|_| Errno::Inval)); - let mut buf = Vec::with_capacity(buf_len); - wasi_try_ok!(write_bytes(&mut buf, &memory, iovs_arr)); - let tasks = env.tasks().clone(); - let written = wasi_try_ok!(__asyncify(&mut ctx, None, async move { - socket.send(tasks.deref(), &buf, fd_flags).await + let written = wasi_try_ok!(__asyncify_light(env, None, async move { + let mut sent = 0usize; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + sent += socket.send(tasks.deref(), buf.as_ref(), fd_flags).await?; + } + Ok(sent) })?); (written, false) } Kind::Pipe { pipe } => { - let buf_len: M::Offset = iovs_arr - .iter() - .filter_map(|a| a.read().ok()) - .map(|a| a.buf_len) - .sum(); - let buf_len: usize = wasi_try_ok!(buf_len.try_into().map_err(|_| Errno::Inval)); - let mut buf = Vec::with_capacity(buf_len); - wasi_try_ok!(write_bytes(&mut buf, &memory, iovs_arr)); - - let written = - wasi_try_ok!(std::io::Write::write(pipe, &buf[..]).map_err(map_io_err)); + let mut written = 0usize; + for iovs in iovs_arr.iter() { + let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)); + let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); + written += wasi_try_ok!( + std::io::Write::write(pipe, buf.as_ref()).map_err(map_io_err) + ); + } (written, false) } Kind::Dir { .. } | Kind::Root { .. } => { @@ -200,23 +201,41 @@ fn fd_write_internal( return Ok(Errno::Isdir); } Kind::EventNotifications(inner) => { - let mut val: [MaybeUninit; 8] = - unsafe { MaybeUninit::uninit().assume_init() }; - let written = wasi_try_ok!(copy_to_slice(&memory, iovs_arr, &mut val[..])); - if written != val.len() { - return Ok(Errno::Inval); - } - let val = u64::from_ne_bytes(unsafe { std::mem::transmute(val) }); + let mut written = 0usize; + for iovs in iovs_arr.iter() { + let buf_len: usize = + wasi_try_ok!(iovs.buf_len.try_into().map_err(|_| Errno::Inval)); + let will_be_written = buf_len; + + let val_cnt = buf_len / std::mem::size_of::(); + let val_cnt: M::Offset = + wasi_try_ok!(val_cnt.try_into().map_err(|_| Errno::Inval)); - inner.write(val); + let vals = wasi_try_ok!(WasmPtr::::new(iovs.buf) + .slice(&memory, val_cnt as M::Offset) + .map_err(mem_error_to_wasi)); + let vals = wasi_try_ok!(vals.access().map_err(mem_error_to_wasi)); + for val in vals.iter() { + inner.write(*val); + } + written += will_be_written; + } (written, false) } Kind::Symlink { .. } => return Ok(Errno::Inval), Kind::Buffer { buffer } => { - let written = - wasi_try_ok!(write_bytes(&mut buffer[offset..], &memory, iovs_arr)); - (written, true) + let mut written = 0usize; + for iovs in iovs_arr.iter() { + let buf = wasi_try_ok!(WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)); + let buf = wasi_try_ok!(buf.access().map_err(mem_error_to_wasi)); + written += wasi_try_ok!( + std::io::Write::write(buffer, buf.as_ref()).map_err(map_io_err) + ); + } + (written, false) } } }; diff --git a/lib/wasi/src/syscalls/wasix/proc_fork.rs b/lib/wasi/src/syscalls/wasix/proc_fork.rs index 26622acd889..3f8aea7b88e 100644 --- a/lib/wasi/src/syscalls/wasix/proc_fork.rs +++ b/lib/wasi/src/syscalls/wasix/proc_fork.rs @@ -275,6 +275,13 @@ pub fn proc_fork( let start = ctx.data(&store).inner().thread_spawn.clone().unwrap(); start.call(&mut store, 0, 0); } + trace!( + "wasi[{}:{}]::proc_{} - child exited (code = {})", + ctx.data(&store).pid(), + ctx.data(&store).tid(), + fork_op, + ret + ); // Clean up the environment ctx.cleanup((&mut store), Some(ret as ExitCode)); diff --git a/lib/wasi/src/syscalls/wasix/sock_recv.rs b/lib/wasi/src/syscalls/wasix/sock_recv.rs index 8ac20477027..05db3734b50 100644 --- a/lib/wasi/src/syscalls/wasix/sock_recv.rs +++ b/lib/wasi/src/syscalls/wasix/sock_recv.rs @@ -102,69 +102,31 @@ fn sock_recv_internal( let mut env = ctx.data(); let memory = env.memory_view(ctx); - let iovs_arr = wasi_try_mem_ok_ok!(ri_data.slice(&memory, ri_data_len)); - let max_size = { - let mut max_size = 0usize; - for iovs in iovs_arr.iter() { - let iovs = wasi_try_mem_ok_ok!(iovs.read()); - let buf_len: usize = - wasi_try_ok_ok!(iovs.buf_len.try_into().map_err(|_| Errno::Overflow)); - max_size += buf_len; - } - max_size - }; - - let res = { - if max_size <= 10240 { - let mut buf: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; - let writer = &mut buf[..max_size]; - let amt = wasi_try_ok_ok!(__sock_asyncify( - env, - sock, - Rights::SOCK_RECV, - |socket, fd| async move { socket.recv(env.tasks().deref(), writer, fd.flags).await }, - )); + let data = wasi_try_ok_ok!(__sock_asyncify( + env, + sock, + Rights::SOCK_RECV, + |socket, fd| async move { + let iovs_arr = ri_data + .slice(&memory, ri_data_len) + .map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - if amt > 0 { - let buf: &[MaybeUninit] = &buf[..amt]; - let buf: &[u8] = unsafe { std::mem::transmute(buf) }; - copy_from_slice(buf, &memory, iovs_arr).map(|_| amt) - } else { - Ok(0) - } - } else { - let data = wasi_try_ok_ok!(__sock_asyncify( - env, - sock, - Rights::SOCK_RECV, - |socket, fd| async move { - let mut buf = Vec::with_capacity(max_size); - unsafe { - buf.set_len(max_size); - } - socket - .recv(env.tasks().deref(), &mut buf, fd.flags) - .await - .map(|amt| { - unsafe { - buf.set_len(amt); - } - let buf: Vec = unsafe { std::mem::transmute(buf) }; - buf - }) - }, - )); + let mut total_read = 0; + for iovs in iovs_arr.iter() { + let mut buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; - let data_len = data.len(); - if data_len > 0 { - let mut reader = &data[..]; - read_bytes(reader, &memory, iovs_arr).map(|_| data_len) - } else { - Ok(0) + total_read += socket + .recv(env.tasks().deref(), buf.as_mut_uninit(), fd.flags) + .await?; } - } - }; - - Ok(res) + Ok(total_read) + }, + )); + Ok(Ok(data)) } diff --git a/lib/wasi/src/syscalls/wasix/sock_send.rs b/lib/wasi/src/syscalls/wasix/sock_send.rs index 7dca883a360..2df64c05974 100644 --- a/lib/wasi/src/syscalls/wasix/sock_send.rs +++ b/lib/wasi/src/syscalls/wasix/sock_send.rs @@ -26,39 +26,28 @@ pub fn sock_send( ) -> Result { let env = ctx.data(); let memory = env.memory_view(&ctx); - let iovs_arr = wasi_try_mem_ok!(si_data.slice(&memory, si_data_len)); let runtime = env.runtime.clone(); - let buf_len: M::Offset = { - iovs_arr - .iter() - .filter_map(|a| a.read().ok()) - .map(|a| a.buf_len) - .sum() - }; - let buf_len: usize = wasi_try_ok!(buf_len.try_into().map_err(|_| Errno::Overflow)); - let res = { - if buf_len <= 10240 { - let mut buf: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; - let writer = &mut buf[..buf_len]; - let written = wasi_try_ok!(copy_to_slice(&memory, iovs_arr, writer)); - - let reader = &buf[..written]; - let reader: &[u8] = unsafe { std::mem::transmute(reader) }; - - __sock_asyncify(env, sock, Rights::SOCK_SEND, |socket, fd| async move { - socket.send(env.tasks().deref(), reader, fd.flags).await - }) - } else { - let mut buf = Vec::with_capacity(buf_len); - wasi_try_ok!(write_bytes(&mut buf, &memory, iovs_arr)); + __sock_asyncify(env, sock, Rights::SOCK_SEND, |socket, fd| async move { + let iovs_arr = si_data + .slice(&memory, si_data_len) + .map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - let reader = &buf; - __sock_asyncify(env, sock, Rights::SOCK_SEND, |socket, fd| async move { - socket.send(env.tasks().deref(), reader, fd.flags).await - }) - } + let mut sent = 0usize; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + sent += socket + .send(env.tasks().deref(), buf.as_ref(), fd.flags) + .await?; + } + Ok(sent) + }) }; let mut ret = Errno::Success; @@ -66,11 +55,10 @@ pub fn sock_send( Ok(bytes_written) => { debug!( %bytes_written, - "wasi[{}:{}]::sock_send (fd={}, buf_len={}, flags={:?})", + "wasi[{}:{}]::sock_send (fd={}, flags={:?})", ctx.data().pid(), ctx.data().tid(), sock, - buf_len, si_flags ); bytes_written @@ -79,11 +67,10 @@ pub fn sock_send( let socket_err = err.name(); debug!( %socket_err, - "wasi[{}:{}]::sock_send (fd={}, buf_len={}, flags={:?})", + "wasi[{}:{}]::sock_send (fd={}, flags={:?})", ctx.data().pid(), ctx.data().tid(), sock, - buf_len, si_flags ); ret = err; @@ -91,9 +78,9 @@ pub fn sock_send( } }; + let memory = env.memory_view(&ctx); let bytes_written: M::Offset = wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); wasi_try_mem_ok!(ret_data_len.write(&memory, bytes_written)); - Ok(ret) } diff --git a/lib/wasi/src/syscalls/wasix/sock_send_to.rs b/lib/wasi/src/syscalls/wasix/sock_send_to.rs index 4c3220b8fe5..eb98eda42e8 100644 --- a/lib/wasi/src/syscalls/wasix/sock_send_to.rs +++ b/lib/wasi/src/syscalls/wasix/sock_send_to.rs @@ -34,14 +34,6 @@ pub fn sock_send_to( let memory = env.memory_view(&ctx); let iovs_arr = wasi_try_mem_ok!(si_data.slice(&memory, si_data_len)); - let buf_len: M::Offset = { - iovs_arr - .iter() - .filter_map(|a| a.read().ok()) - .map(|a| a.buf_len) - .sum() - }; - let buf_len: usize = wasi_try_ok!(buf_len.try_into().map_err(|_| Errno::Inval)); let (addr_ip, addr_port) = { let memory = env.memory_view(&ctx); wasi_try_ok!(read_ip_port(&memory, addr)) @@ -49,45 +41,36 @@ pub fn sock_send_to( let addr = SocketAddr::new(addr_ip, addr_port); let bytes_written = { - if buf_len <= 10240 { - let mut buf: [MaybeUninit; 10240] = unsafe { MaybeUninit::uninit().assume_init() }; - let writer = &mut buf[..buf_len]; - let written = wasi_try_ok!(copy_to_slice(&memory, iovs_arr, writer)); - - let reader = &buf[..written]; - let reader: &[u8] = unsafe { std::mem::transmute(reader) }; + wasi_try_ok!(__sock_asyncify( + env, + sock, + Rights::SOCK_SEND_TO, + |socket, fd| async move { + let iovs_arr = si_data + .slice(&memory, si_data_len) + .map_err(mem_error_to_wasi)?; + let iovs_arr = iovs_arr.access().map_err(mem_error_to_wasi)?; - wasi_try_ok!(__sock_asyncify( - env, - sock, - Rights::SOCK_SEND, - |socket, fd| async move { - socket - .send_to::(env.tasks().deref(), reader, addr, fd.flags) - .await - }, - )) - } else { - let mut buf = Vec::with_capacity(buf_len); - wasi_try_ok!(write_bytes(&mut buf, &memory, iovs_arr)); - - let reader = &buf; - wasi_try_ok!(__sock_asyncify( - env, - sock, - Rights::SOCK_SEND_TO, - |socket, fd| async move { - socket - .send_to::(env.tasks().deref(), reader, addr, fd.flags) - .await - }, - )) - } + let mut sent = 0usize; + for iovs in iovs_arr.iter() { + let buf = WasmPtr::::new(iovs.buf) + .slice(&memory, iovs.buf_len) + .map_err(mem_error_to_wasi)? + .access() + .map_err(mem_error_to_wasi)?; + sent += socket + .send_to::(env.tasks().deref(), buf.as_ref(), addr, fd.flags) + .await?; + } + Ok(sent) + }, + )) }; + let memory = env.memory_view(&ctx); let bytes_written: M::Offset = wasi_try_ok!(bytes_written.try_into().map_err(|_| Errno::Overflow)); - wasi_try_mem_ok!(ret_data_len.write(&memory, bytes_written as M::Offset)); + wasi_try_mem_ok!(ret_data_len.write(&memory, bytes_written)); Ok(Errno::Success) }