std: Use libc for filesystem ops on WASI targets

This commit is a large change to the implementation of filesystem and
other system-related operations on WASI targets. Previously the standard
library explicitly used the `wasi` crate at the 0.11.x version track
which means that it used WASIp1 APIs directly. This meant that `std` was
hard-coded to use WASIp1 syscalls and there was no separate
implementation for the WASIp{2,3} targets, for example. The high-level
goal of this commit is to decouple this interaction and avoid the use of
the `wasi` crate on the WASIp2 target.

Historically when WASIp1 was originally added to Rust the wasi-libc
library was in a much different position than it is today. Nowadays Rust
already depends on wasi-libc on WASI targets for things like memory
allocation and environment variable management. As a libc library it
also has all the functions necessary to implement all filesystem
operations Rust wants. Recently wasi-libc additionally was updated to
use WASIp2 APIs directly on the `wasm32-wasip2` target instead of using
`wasm32-wasip1` APIs. This commit is leveraging this work by enabling
Rust to completely sever the dependence on WASIp1 APIs when compiling
for `wasm32-wasip2`. This is also intended to make it easier to migrate
to `wasm32-wasip3` internally in the future where now only libc need be
updated and Rust doesn't need to explicitly change as well.

This commit fairly heavily changes the internals of the implementation
of WASI filesystem operations. The basis of implementation is now
libc-style APIs as opposed to WASIp1-style-APIs which necessitated a
number of changes throughout. Additionally the `std::os::wasi::fs`
module has had a few API-breaking changes as well, but this module is
also unstable. While this module has been "stable" since inception in
the sense that it hasn't changed, this PR is what the unstable status
was sort of trying to buy in terms of future flexibility to make
changes. For users of stable `std::fs` interfaces this is not intended
to be a breaking change but there nevertheless may be minor issues here
and there.

Concrete changes here are:

* `std` for `wasm32-wasip2` no longer depends on `wasi 0.11.x`
* The implementation of `std::os::wasi::fs`, which was previously
  unstable and still is, is now only available on `wasm32-wasip1` and is
  implemented largely directly in terms of WASIp1 APIs by directly using
  the `wasi 0.11.x` crate.
* `std::os::wasi::fs::MetadataExt`, which was previously unstable and
  still is, has removed methods that duplicate functionality in
  `std::fs::Metadata` as there's no point in having them.
* `std::os::wasi::fs::FileTypeExt`, which was previously unstable and
  still is, lost the ability to distinguish between dgram/stream
  sockets. WASIp1 sockets were never really supported, though, so this
  isn't much of a loss.
* The internal `WasiFd` type has been "gutted" to have far fewer methods
  internally. This still represents a file descriptor owned by
  `wasi-libc`, however.
* The `std::sys::fs::wasi` implementation is overhauled to largely use
  the `libc` crate and its APIs directly (which go to wasi-libc). This
  drastically changes the `ReadDir` implementation, for example.
* The `std::sys::fs::wasi::OpenOptions` API now has two ways of opening
  files. By default `libc::open` is used but of
  `std::os::wasi::fs::OpenOptionsExt` is used then a WASIp1-specific API
  is used to open a file.
* Types like `FileAttr` and `FileType` now use libc representations
  at-rest instead of WASIp1 internals.
* Various transmutations for iovec-style APIs are consolidated into a
  WASI-specific module close to the definition of the `IoSlice{,Mut}`
  types to make it easier to audit unsafe guarantees.
* The `wasip2` OS module gained a custom implementation of the `helpers`
  module as the WASIp1 version of this module is no longer applicable.

I'd like to do follow-up PRs after this one to reorganize
`std::os::wasi*` a bit to be more amenable for WASIp{2,3} targets as
well, but that'll come in a future PR. For now this is tasked with the
high-level goal of removing the dependency on WASIp1 APIs in the
standard library by relying on wasi-libc to natively use WASIp2 APIs.

Author: alexcrichton

library/std/Cargo.toml

@@ -75,7 +75,7 @@ hermit-abi = { version = "0.5.0", features = [
     'rustc-dep-of-std',
 ], public = true }
 
-[target.'cfg(target_os = "wasi")'.dependencies]
+[target.'cfg(all(target_os = "wasi", target_env = "p1"))'.dependencies]
 wasi = { version = "0.11.0", features = [
     'rustc-dep-of-std',
 ], default-features = false }

library/std/src/os/wasi/fs.rs

@@ -1,4 +1,4 @@
-//! WASI-specific extensions to primitives in the [`std::fs`] module.
+//! WASIp1-specific extensions to primitives in the [`std::fs`] module.
 //!
 //! [`std::fs`]: crate::fs
 
@@ -9,9 +9,11 @@
 use io::{Read, Write};
 
 use crate::ffi::OsStr;
-use crate::fs::{self, File, Metadata, OpenOptions};
+use crate::fs::{self, File, OpenOptions};
 use crate::io::{self, IoSlice, IoSliceMut};
+use crate::os::fd::AsRawFd;
 use crate::path::{Path, PathBuf};
+use crate::sys::err2io;
 use crate::sys_common::{AsInner, AsInnerMut, FromInner};
 
 /// WASI-specific extensions to [`File`].
@@ -200,7 +202,7 @@ pub trait FileExt {
     ///
     /// This corresponds to the `path_filestat_get` syscall.
     #[doc(alias = "path_filestat_get")]
-    fn metadata_at<P: AsRef<Path>>(&self, lookup_flags: u32, path: P) -> io::Result<Metadata>;
+    fn metadata_at<P: AsRef<Path>>(&self, lookup_flags: i32, path: P) -> io::Result<fs::Metadata>;
 
     /// Unlinks a file.
     ///
@@ -224,19 +226,24 @@ pub trait FileExt {
 
 impl FileExt for fs::File {
     fn read_vectored_at(&self, bufs: &mut [IoSliceMut<'_>], offset: u64) -> io::Result<usize> {
-        self.as_inner().as_inner().pread(bufs, offset)
+        let bufs = crate::sys::io::IoSliceMut::as_wasip1_slice(bufs);
+        unsafe { wasi::fd_pread(self.as_raw_fd() as wasi::Fd, bufs, offset).map_err(err2io) }
     }
 
     fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result<usize> {
-        self.as_inner().as_inner().pwrite(bufs, offset)
+        let bufs = crate::sys::io::IoSlice::as_wasip1_slice(bufs);
+        unsafe { wasi::fd_pwrite(self.as_raw_fd() as wasi::Fd, bufs, offset).map_err(err2io) }
     }
 
     fn fdstat_set_flags(&self, flags: u16) -> io::Result<()> {
-        self.as_inner().as_inner().set_flags(flags)
+        unsafe { wasi::fd_fdstat_set_flags(self.as_raw_fd() as wasi::Fd, flags).map_err(err2io) }
     }
 
     fn fdstat_set_rights(&self, rights: u64, inheriting: u64) -> io::Result<()> {
-        self.as_inner().as_inner().set_rights(rights, inheriting)
+        unsafe {
+            wasi::fd_fdstat_set_rights(self.as_raw_fd() as wasi::Fd, rights, inheriting)
+                .map_err(err2io)
+        }
     }
 
     fn advise(&self, offset: u64, len: u64, advice: u8) -> io::Result<()> {
@@ -255,32 +262,37 @@ impl FileExt for fs::File {
             }
         };
 
-        self.as_inner().as_inner().advise(offset, len, advice)
+        unsafe {
+            wasi::fd_advise(self.as_raw_fd() as wasi::Fd, offset, len, advice).map_err(err2io)
+        }
     }
 
     fn allocate(&self, offset: u64, len: u64) -> io::Result<()> {
-        self.as_inner().as_inner().allocate(offset, len)
+        unsafe { wasi::fd_allocate(self.as_raw_fd() as wasi::Fd, offset, len).map_err(err2io) }
     }
 
     fn create_directory<P: AsRef<Path>>(&self, dir: P) -> io::Result<()> {
-        self.as_inner().as_inner().create_directory(osstr2str(dir.as_ref().as_ref())?)
+        let path = osstr2str(dir.as_ref().as_ref())?;
+        unsafe { wasi::path_create_directory(self.as_raw_fd() as wasi::Fd, path).map_err(err2io) }
     }
 
     fn read_link<P: AsRef<Path>>(&self, path: P) -> io::Result<PathBuf> {
-        self.as_inner().read_link(path.as_ref())
+        self.as_inner().readlink_at(path.as_ref())
     }
 
-    fn metadata_at<P: AsRef<Path>>(&self, lookup_flags: u32, path: P) -> io::Result<Metadata> {
+    fn metadata_at<P: AsRef<Path>>(&self, lookup_flags: i32, path: P) -> io::Result<fs::Metadata> {
         let m = self.as_inner().metadata_at(lookup_flags, path.as_ref())?;
         Ok(FromInner::from_inner(m))
     }
 
     fn remove_file<P: AsRef<Path>>(&self, path: P) -> io::Result<()> {
-        self.as_inner().as_inner().unlink_file(osstr2str(path.as_ref().as_ref())?)
+        let path = osstr2str(path.as_ref().as_ref())?;
+        unsafe { wasi::path_unlink_file(self.as_raw_fd() as wasi::Fd, path).map_err(err2io) }
     }
 
     fn remove_directory<P: AsRef<Path>>(&self, path: P) -> io::Result<()> {
-        self.as_inner().as_inner().remove_directory(osstr2str(path.as_ref().as_ref())?)
+        let path = osstr2str(path.as_ref().as_ref())?;
+        unsafe { wasi::path_remove_directory(self.as_raw_fd() as wasi::Fd, path).map_err(err2io) }
     }
 }
 
@@ -355,42 +367,42 @@ pub trait OpenOptionsExt {
 
 impl OpenOptionsExt for OpenOptions {
     fn lookup_flags(&mut self, flags: u32) -> &mut OpenOptions {
-        self.as_inner_mut().lookup_flags(flags);
+        self.as_inner_mut().wasip1_lookup_flags(flags);
         self
     }
 
     fn directory(&mut self, dir: bool) -> &mut OpenOptions {
-        self.as_inner_mut().directory(dir);
+        self.as_inner_mut().wasip1_directory(dir);
         self
     }
 
     fn dsync(&mut self, enabled: bool) -> &mut OpenOptions {
-        self.as_inner_mut().dsync(enabled);
+        self.as_inner_mut().wasip1_dsync(enabled);
         self
     }
 
     fn nonblock(&mut self, enabled: bool) -> &mut OpenOptions {
-        self.as_inner_mut().nonblock(enabled);
+        self.as_inner_mut().wasip1_nonblock(enabled);
         self
     }
 
     fn rsync(&mut self, enabled: bool) -> &mut OpenOptions {
-        self.as_inner_mut().rsync(enabled);
+        self.as_inner_mut().wasip1_rsync(enabled);
         self
     }
 
     fn sync(&mut self, enabled: bool) -> &mut OpenOptions {
-        self.as_inner_mut().sync(enabled);
+        self.as_inner_mut().wasip1_sync(enabled);
         self
     }
 
     fn fs_rights_base(&mut self, rights: u64) -> &mut OpenOptions {
-        self.as_inner_mut().fs_rights_base(rights);
+        self.as_inner_mut().wasip1_fs_rights_base(rights);
         self
     }
 
     fn fs_rights_inheriting(&mut self, rights: u64) -> &mut OpenOptions {
-        self.as_inner_mut().fs_rights_inheriting(rights);
+        self.as_inner_mut().wasip1_fs_rights_inheriting(rights);
         self
     }
 
@@ -408,37 +420,17 @@ pub trait MetadataExt {
     fn ino(&self) -> u64;
     /// Returns the `st_nlink` field of the internal `filestat_t`
     fn nlink(&self) -> u64;
-    /// Returns the `st_size` field of the internal `filestat_t`
-    fn size(&self) -> u64;
-    /// Returns the `st_atim` field of the internal `filestat_t`
-    fn atim(&self) -> u64;
-    /// Returns the `st_mtim` field of the internal `filestat_t`
-    fn mtim(&self) -> u64;
-    /// Returns the `st_ctim` field of the internal `filestat_t`
-    fn ctim(&self) -> u64;
 }
 
 impl MetadataExt for fs::Metadata {
     fn dev(&self) -> u64 {
-        self.as_inner().as_wasi().dev
+        self.as_inner().as_libc().st_dev
     }
     fn ino(&self) -> u64 {
-        self.as_inner().as_wasi().ino
+        self.as_inner().as_libc().st_ino
     }
     fn nlink(&self) -> u64 {
-        self.as_inner().as_wasi().nlink
-    }
-    fn size(&self) -> u64 {
-        self.as_inner().as_wasi().size
-    }
-    fn atim(&self) -> u64 {
-        self.as_inner().as_wasi().atim
-    }
-    fn mtim(&self) -> u64 {
-        self.as_inner().as_wasi().mtim
-    }
-    fn ctim(&self) -> u64 {
-        self.as_inner().as_wasi().ctim
+        self.as_inner().as_libc().st_nlink
     }
 }
 
@@ -451,28 +443,19 @@ pub trait FileTypeExt {
     fn is_block_device(&self) -> bool;
     /// Returns `true` if this file type is a character device.
     fn is_char_device(&self) -> bool;
-    /// Returns `true` if this file type is a socket datagram.
-    fn is_socket_dgram(&self) -> bool;
-    /// Returns `true` if this file type is a socket stream.
-    fn is_socket_stream(&self) -> bool;
     /// Returns `true` if this file type is any type of socket.
-    fn is_socket(&self) -> bool {
-        self.is_socket_stream() || self.is_socket_dgram()
-    }
+    fn is_socket(&self) -> bool;
 }
 
 impl FileTypeExt for fs::FileType {
     fn is_block_device(&self) -> bool {
-        self.as_inner().bits() == wasi::FILETYPE_BLOCK_DEVICE
+        self.as_inner().mode() == libc::S_IFBLK
     }
     fn is_char_device(&self) -> bool {
-        self.as_inner().bits() == wasi::FILETYPE_CHARACTER_DEVICE
-    }
-    fn is_socket_dgram(&self) -> bool {
-        self.as_inner().bits() == wasi::FILETYPE_SOCKET_DGRAM
+        self.as_inner().mode() == libc::S_IFCHR
     }
-    fn is_socket_stream(&self) -> bool {
-        self.as_inner().bits() == wasi::FILETYPE_SOCKET_STREAM
+    fn is_socket(&self) -> bool {
+        self.as_inner().mode() == libc::S_IFSOCK
     }
 }
 
@@ -499,12 +482,16 @@ pub fn link<P: AsRef<Path>, U: AsRef<Path>>(
     new_fd: &File,
     new_path: U,
 ) -> io::Result<()> {
-    old_fd.as_inner().as_inner().link(
-        old_flags,
-        osstr2str(old_path.as_ref().as_ref())?,
-        new_fd.as_inner().as_inner(),
-        osstr2str(new_path.as_ref().as_ref())?,
-    )
+    unsafe {
+        wasi::path_link(
+            old_fd.as_raw_fd() as wasi::Fd,
+            old_flags,
+            osstr2str(old_path.as_ref().as_ref())?,
+            new_fd.as_raw_fd() as wasi::Fd,
+            osstr2str(new_path.as_ref().as_ref())?,
+        )
+        .map_err(err2io)
+    }
 }
 
 /// Renames a file or directory.
@@ -517,11 +504,15 @@ pub fn rename<P: AsRef<Path>, U: AsRef<Path>>(
     new_fd: &File,
     new_path: U,
 ) -> io::Result<()> {
-    old_fd.as_inner().as_inner().rename(
-        osstr2str(old_path.as_ref().as_ref())?,
-        new_fd.as_inner().as_inner(),
-        osstr2str(new_path.as_ref().as_ref())?,
-    )
+    unsafe {
+        wasi::path_rename(
+            old_fd.as_raw_fd() as wasi::Fd,
+            osstr2str(old_path.as_ref().as_ref())?,
+            new_fd.as_raw_fd() as wasi::Fd,
+            osstr2str(new_path.as_ref().as_ref())?,
+        )
+        .map_err(err2io)
+    }
 }
 
 /// Creates a symbolic link.
@@ -533,9 +524,14 @@ pub fn symlink<P: AsRef<Path>, U: AsRef<Path>>(
     fd: &File,
     new_path: U,
 ) -> io::Result<()> {
-    fd.as_inner()
-        .as_inner()
-        .symlink(osstr2str(old_path.as_ref().as_ref())?, osstr2str(new_path.as_ref().as_ref())?)
+    unsafe {
+        wasi::path_symlink(
+            osstr2str(old_path.as_ref().as_ref())?,
+            fd.as_raw_fd() as wasi::Fd,
+            osstr2str(new_path.as_ref().as_ref())?,
+        )
+        .map_err(err2io)
+    }
 }
 
 /// Creates a symbolic link.

library/std/src/os/wasi/mod.rs

@@ -34,6 +34,7 @@
 #![doc(cfg(target_os = "wasi"))]
 
 pub mod ffi;
+#[cfg(target_env = "p1")]
 pub mod fs;
 pub mod io;
 
@@ -50,9 +51,11 @@ pub mod prelude {
     pub use super::ffi::{OsStrExt, OsStringExt};
     #[doc(no_inline)]
     #[stable(feature = "rust1", since = "1.0.0")]
+    #[cfg(target_env = "p1")]
     pub use super::fs::FileTypeExt;
     #[doc(no_inline)]
     #[stable(feature = "rust1", since = "1.0.0")]
+    #[cfg(target_env = "p1")]
     pub use super::fs::{DirEntryExt, FileExt, MetadataExt, OpenOptionsExt};
     #[doc(no_inline)]
     #[stable(feature = "rust1", since = "1.0.0")]

library/std/src/os/wasi/net/mod.rs

@@ -2,7 +2,8 @@
 
 #![unstable(feature = "wasi_ext", issue = "71213")]
 
-use crate::sys_common::AsInner;
+use crate::os::fd::AsRawFd;
+use crate::sys::err2io;
 use crate::{io, net};
 
 /// WASI-specific extensions to [`std::net::TcpListener`].
@@ -17,6 +18,6 @@ pub trait TcpListenerExt {
 
 impl TcpListenerExt for net::TcpListener {
     fn sock_accept(&self, flags: u16) -> io::Result<u32> {
-        self.as_inner().as_inner().as_inner().sock_accept(flags)
+        unsafe { wasi::sock_accept(self.as_raw_fd() as wasi::Fd, flags).map_err(err2io) }
     }
 }