Initial Eio_posix backend

Co-authored-by: Christiano Haesbaert <[email protected]>

Makefile

@@ -17,6 +17,9 @@ bench:
 test_luv:
 	EIO_BACKEND=luv dune runtest
 
+test_posix:
+	EIO_BACKEND=posix dune runtest
+
 dscheck:
 	dune exec -- ./lib_eio/tests/dscheck/test_rcfd.exe
 	dune exec -- ./lib_eio/tests/dscheck/test_sync.exe

dune-project

@@ -32,7 +32,7 @@
 (package
  (name eio_linux)
  (synopsis "Eio implementation for Linux using io-uring")
- (description "An eio implementation for Linux using io-uring.")
+ (description "An Eio implementation for Linux using io-uring.")
  (depends
   (alcotest (and (>= 1.4.0) :with-test))
   (eio (= :version))
@@ -41,10 +41,19 @@
   (fmt (>= 0.8.9))
   (cmdliner (and (>= 1.1.0) :with-test))
   (uring (>= 0.5))))
+(package
+ (name eio_posix)
+ (synopsis "Eio implementation for POSIX systems")
+ (description "An Eio implementation for most Unix-like platforms")
+ (depends
+  (eio (= :version))
+  (iomux (>= 0.2))
+  (mdx (and (>= 1.10.0) :with-test))
+  (fmt (>= 0.8.9))))
 (package
  (name eio_luv)
  (synopsis "Eio implementation using luv (libuv)")
- (description "An eio implementation for most platforms, using luv.")
+ (description "An Eio implementation for most platforms, using luv.")
  (depends
   (eio (= :version))
   (luv (>= 0.5.11))

eio_linux.opam

@@ -1,7 +1,7 @@
 # This file is generated by dune, edit dune-project instead
 opam-version: "2.0"
 synopsis: "Eio implementation for Linux using io-uring"
-description: "An eio implementation for Linux using io-uring."
+description: "An Eio implementation for Linux using io-uring."
 maintainer: ["[email protected]"]
 authors: ["Anil Madhavapeddy" "Thomas Leonard"]
 license: "ISC"

eio_luv.opam

@@ -1,7 +1,7 @@
 # This file is generated by dune, edit dune-project instead
 opam-version: "2.0"
 synopsis: "Eio implementation using luv (libuv)"
-description: "An eio implementation for most platforms, using luv."
+description: "An Eio implementation for most platforms, using luv."
 maintainer: ["[email protected]"]
 authors: ["Anil Madhavapeddy" "Thomas Leonard"]
 license: "ISC"

eio_posix.opam

@@ -0,0 +1,33 @@
+# This file is generated by dune, edit dune-project instead
+opam-version: "2.0"
+synopsis: "Eio implementation for POSIX systems"
+description: "An Eio implementation for most Unix-like platforms"
+maintainer: ["[email protected]"]
+authors: ["Anil Madhavapeddy" "Thomas Leonard"]
+license: "ISC"
+homepage: "https://github.com/ocaml-multicore/eio"
+doc: "https://ocaml-multicore.github.io/eio/"
+bug-reports: "https://github.com/ocaml-multicore/eio/issues"
+depends: [
+  "dune" {>= "3.0"}
+  "eio" {= version}
+  "iomux" {>= "0.2"}
+  "mdx" {>= "1.10.0" & with-test}
+  "fmt" {>= "0.8.9"}
+  "odoc" {with-doc}
+]
+build: [
+  ["dune" "subst"] {dev}
+  [
+    "dune"
+    "build"
+    "-p"
+    name
+    "-j"
+    jobs
+    "@install"
+    "@runtest" {with-test}
+    "@doc" {with-doc}
+  ]
+]
+dev-repo: "git+https://github.com/ocaml-multicore/eio.git"

lib_eio_posix/domain_mgr.ml

@@ -0,0 +1,77 @@
+(*
+ * Copyright (C) 2023 Thomas Leonard
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *)
+
+open Eio.Std
+
+[@@@alert "-unstable"]
+
+(* Run an event loop in the current domain, using [fn x] as the root fiber. *)
+let run_event_loop fn x =
+  Sched.with_sched @@ fun sched ->
+  let open Effect.Deep in
+  let extra_effects : _ effect_handler = {
+    effc = fun (type a) (e : a Effect.t) : ((a, Sched.exit) continuation -> Sched.exit) option ->
+      match e with
+      | Eio_unix.Private.Get_monotonic_clock -> Some (fun k -> continue k (Time.mono_clock : Eio.Time.Mono.t))
+      | Eio_unix.Private.Socket_of_fd (sw, close_unix, fd) -> Some (fun k ->
+          Unix.set_nonblock fd;
+          let fd = Fd.of_unix ~sw ~blocking:false ~close_unix fd in
+          continue k (Flow.of_fd fd :> Eio_unix.socket)
+        )
+      | Eio_unix.Private.Socketpair (sw, domain, ty, protocol) -> Some (fun k ->
+          let a, b = Unix.socketpair ~cloexec:true domain ty protocol in
+          Unix.set_nonblock a;
+          Unix.set_nonblock b;
+          let a = Fd.of_unix ~sw ~blocking:false ~close_unix:true a |> Flow.of_fd in
+          let b = Fd.of_unix ~sw ~blocking:false ~close_unix:true b |> Flow.of_fd in
+          continue k ((a :> Eio_unix.socket), (b :> Eio_unix.socket))
+        )
+      | Eio_unix.Private.Pipe sw -> Some (fun k ->
+          let r, w = Unix.pipe ~cloexec:true () in
+          (* See issue #319, PR #327 *)
+          Unix.set_nonblock r;
+          Unix.set_nonblock w;
+          let make x = Flow.of_fd (Fd.of_unix ~sw ~blocking:false ~close_unix:true x) in
+          let r = (make r :> <Eio.Flow.source; Eio.Flow.close; Eio_unix.unix_fd>) in
+          let w = (make w :> <Eio.Flow.sink; Eio.Flow.close; Eio_unix.unix_fd>) in
+          continue k (r, w)
+        )
+      | _ -> None
+  }
+  in
+  Sched.run ~extra_effects sched fn x
+
+let v = object
+  inherit Eio.Domain_manager.t
+
+  method run_raw fn =
+    let domain = ref None in
+    Eio.Private.Suspend.enter (fun _ctx enqueue ->
+        domain := Some (Domain.spawn (fun () -> Fun.protect fn ~finally:(fun () -> enqueue (Ok ()))))
+      );
+    Domain.join (Option.get !domain)
+
+  method run fn =
+    let domain = ref None in
+    Eio.Private.Suspend.enter (fun ctx enqueue ->
+        let cancelled, set_cancelled = Promise.create () in
+        Eio.Private.Fiber_context.set_cancel_fn ctx (Promise.resolve set_cancelled);
+        domain := Some (Domain.spawn (fun () ->
+            Fun.protect (run_event_loop (fun () -> fn ~cancelled))
+              ~finally:(fun () -> enqueue (Ok ()))))
+      );
+    Domain.join (Option.get !domain)
+end

lib_eio_posix/dune

@@ -0,0 +1,9 @@
+(library
+ (name eio_posix)
+ (public_name eio_posix)
+ (enabled_if (= %{os_type} "Unix"))
+ (foreign_stubs
+  (language c)
+  (flags :standard -D_LARGEFILE64_SOURCE)
+  (names eio_posix_stubs))
+ (libraries eio eio.utils eio.unix fmt iomux))

lib_eio_posix/eio_posix.ml

@@ -0,0 +1,48 @@
+(*
+ * Copyright (C) 2023 Thomas Leonard
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *)
+
+module Low_level = Low_level
+
+type stdenv = <
+  stdin  : <Eio.Flow.source; Eio_unix.unix_fd>;
+  stdout : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  stderr : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  net : Eio.Net.t;
+  domain_mgr : Eio.Domain_manager.t;
+  clock : Eio.Time.clock;
+  mono_clock : Eio.Time.Mono.t;
+  fs : Eio.Fs.dir Eio.Path.t;
+  cwd : Eio.Fs.dir Eio.Path.t;
+  secure_random : Eio.Flow.source;
+  debug : Eio.Debug.t;
+>
+
+let run main =
+  (* SIGPIPE makes no sense in a modern application. *)
+  Sys.(set_signal sigpipe Signal_ignore);
+  Domain_mgr.run_event_loop main @@ object (_ : stdenv)
+    method stdin = (Flow.of_fd Low_level.Fd.stdin :> <Eio.Flow.source; Eio_unix.unix_fd>)
+    method stdout = (Flow.of_fd Low_level.Fd.stdout :> <Eio.Flow.sink; Eio_unix.unix_fd>)
+    method stderr = (Flow.of_fd Low_level.Fd.stderr :> <Eio.Flow.sink; Eio_unix.unix_fd>)
+    method debug = Eio.Private.Debug.v
+    method clock = Time.clock
+    method mono_clock = Time.mono_clock
+    method net = Net.v
+    method domain_mgr = Domain_mgr.v
+    method cwd = ((Fs.cwd, "") :> Eio.Fs.dir Eio.Path.t)
+    method fs = ((Fs.fs, "") :> Eio.Fs.dir Eio.Path.t)
+    method secure_random = Flow.secure_random
+  end

lib_eio_posix/eio_posix.mli

@@ -0,0 +1,24 @@
+(** Fallback Eio backend for POSIX systems. *)
+
+type stdenv = <
+  stdin  : <Eio.Flow.source; Eio_unix.unix_fd>;
+  stdout : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  stderr : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  net : Eio.Net.t;
+  domain_mgr : Eio.Domain_manager.t;
+  clock : Eio.Time.clock;
+  mono_clock : Eio.Time.Mono.t;
+  fs : Eio.Fs.dir Eio.Path.t;
+  cwd : Eio.Fs.dir Eio.Path.t;
+  secure_random : Eio.Flow.source;
+  debug : Eio.Debug.t;
+>
+(** An extended version of {!Eio.Stdenv.t} with some extra features available on POSIX systems. *)
+
+val run : (stdenv -> 'a) -> 'a
+(** [run main] runs an event loop and calls [main stdenv] inside it.
+
+    For portable code, you should use {!Eio_main.run} instead, which will call this for you if appropriate. *)
+
+module Low_level = Low_level
+(** Low-level API for making POSIX calls directly. *)

lib_eio_posix/eio_posix_stubs.c

@@ -0,0 +1,107 @@
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/eventfd.h>
+#include <sys/random.h>
+#include <sys/syscall.h>
+#include <limits.h>
+#include <errno.h>
+#include <dirent.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/uio.h>
+
+#include <caml/mlvalues.h>
+#include <caml/memory.h>
+#include <caml/alloc.h>
+#include <caml/signals.h>
+#include <caml/unixsupport.h>
+#include <caml/bigarray.h>
+
+#ifdef ARCH_SIXTYFOUR
+#define Int63_val(v) Long_val(v)
+#else
+#define Int63_val(v) (Int64_val(v)) >> 1
+#endif
+
+CAMLprim value caml_eio_posix_getrandom(value v_ba, value v_off, value v_len) {
+  CAMLparam1(v_ba);
+  ssize_t ret;
+  ssize_t off = (ssize_t)Long_val(v_off);
+  ssize_t len = (ssize_t)Long_val(v_len);
+  do {
+    void *buf = Caml_ba_data_val(v_ba) + off;
+    caml_enter_blocking_section();
+    ret = getrandom(buf, len, 0);
+    caml_leave_blocking_section();
+  } while (ret == -1 && errno == EINTR);
+  if (ret == -1) uerror("getrandom", Nothing);
+  CAMLreturn(Val_long(ret));
+}
+
+static void fill_iov(struct iovec *iov, value v_bufs) {
+  int n_bufs = Wosize_val(v_bufs);
+  for (int i = 0; i < n_bufs; i++) {
+    value v_cs = Field(v_bufs, i);
+    value v_ba = Field(v_cs, 0);
+    value v_off = Field(v_cs, 1);
+    value v_len = Field(v_cs, 2);
+    iov[i].iov_base = Caml_ba_data_val(v_ba) + Long_val(v_off);
+    iov[i].iov_len = Long_val(v_len);
+  }
+}
+
+CAMLprim value caml_eio_posix_readv(value v_fd, value v_bufs) {
+  CAMLparam1(v_bufs);
+  ssize_t r;
+  int n_bufs = Wosize_val(v_bufs);
+  struct iovec iov[n_bufs];
+
+  fill_iov(iov, v_bufs);
+
+  r = readv(Int_val(v_fd), iov, n_bufs);
+  if (r < 0) uerror("readv", Nothing);
+
+  CAMLreturn(Val_long(r));
+}
+
+CAMLprim value caml_eio_posix_writev(value v_fd, value v_bufs) {
+  CAMLparam1(v_bufs);
+  ssize_t r;
+  int n_bufs = Wosize_val(v_bufs);
+  struct iovec iov[n_bufs];
+
+  fill_iov(iov, v_bufs);
+
+  r = writev(Int_val(v_fd), iov, n_bufs);
+  if (r < 0) uerror("writev", Nothing);
+
+  CAMLreturn(Val_long(r));
+}
+
+CAMLprim value caml_eio_posix_preadv(value v_fd, value v_bufs, value v_offset) {
+  CAMLparam2(v_bufs, v_offset);
+  ssize_t r;
+  int n_bufs = Wosize_val(v_bufs);
+  struct iovec iov[n_bufs];
+
+  fill_iov(iov, v_bufs);
+
+  r = preadv(Int_val(v_fd), iov, n_bufs, Int63_val(v_offset));
+  if (r < 0) uerror("preadv", Nothing);
+
+  CAMLreturn(Val_long(r));
+}
+
+CAMLprim value caml_eio_posix_pwritev(value v_fd, value v_bufs, value v_offset) {
+  CAMLparam2(v_bufs, v_offset);
+  ssize_t r;
+  int n_bufs = Wosize_val(v_bufs);
+  struct iovec iov[n_bufs];
+
+  fill_iov(iov, v_bufs);
+
+  r = pwritev(Int_val(v_fd), iov, n_bufs, Int63_val(v_offset));
+  if (r < 0) uerror("pwritev", Nothing);
+
+  CAMLreturn(Val_long(r));
+}

lib_eio_posix/err.ml

@@ -0,0 +1,27 @@
+type Eio.Exn.Backend.t +=
+  | Outside_sandbox of string * string
+  | Absolute_path
+
+let unclassified_error e = Eio.Exn.create (Eio.Exn.X e)
+
+let () =
+  Eio.Exn.Backend.register_pp (fun f -> function
+      | Outside_sandbox (path, dir) -> Fmt.pf f "Outside_sandbox (%S, %S)" path dir; true
+      | Absolute_path -> Fmt.pf f "Absolute_path"; true
+      | _ -> false
+    )
+
+let wrap code name arg =
+  let e = Eio_unix.Unix_error (code, name, arg) in
+  match code with
+  | EEXIST -> Eio.Fs.err (Already_exists e)
+  | ENOENT -> Eio.Fs.err (Not_found e)
+  | EXDEV -> Eio.Fs.err (Permission_denied e)
+  | ECONNREFUSED -> Eio.Net.err (Connection_failure (Refused e))
+  | ECONNRESET | EPIPE -> Eio.Net.err (Connection_reset e)
+  | _ -> unclassified_error e
+
+let run fn x =
+  try fn x
+  with Unix.Unix_error(code, name, arg) ->
+    raise (wrap code name arg)