Merge pull request #448 from talex5/eio-posix

Initial Eio_posix backend

.github/workflows/main.yml

@@ -14,7 +14,7 @@ jobs:
         ocaml-compiler:
           - 5.0.x
         local-packages:
-          - eio eio_luv eio_main
+          - eio eio_posix eio_main
 
     runs-on: ${{ matrix.os }}
 
@@ -26,8 +26,9 @@ jobs:
         uses: ocaml/setup-ocaml@v2
         with:
           ocaml-compiler: ${{ matrix.ocaml-compiler }}
-          opam-local-packages: ${{ matrix.local-packages }}
+          opam-local-packages:
           opam-disable-sandboxing: true
 
+      - run: opam --cli=2.1 pin -yn --with-version=dev .
       - run: opam install ${{ matrix.local-packages }} --deps-only --with-test
       - run: opam install ${{ matrix.local-packages }} --with-test

Makefile

@@ -17,6 +17,9 @@ bench:
 test_luv:
 	EIO_BACKEND=luv dune runtest
 
+test_posix:
+	EIO_BACKEND=posix dune runtest
+
 dscheck:
 	dune exec -- ./lib_eio/tests/dscheck/test_rcfd.exe
 	dune exec -- ./lib_eio/tests/dscheck/test_sync.exe

dune-project

@@ -32,7 +32,7 @@
 (package
  (name eio_linux)
  (synopsis "Eio implementation for Linux using io-uring")
- (description "An eio implementation for Linux using io-uring.")
+ (description "An Eio implementation for Linux using io-uring.")
  (depends
   (alcotest (and (>= 1.4.0) :with-test))
   (eio (= :version))
@@ -41,10 +41,19 @@
   (fmt (>= 0.8.9))
   (cmdliner (and (>= 1.1.0) :with-test))
   (uring (>= 0.5))))
+(package
+ (name eio_posix)
+ (synopsis "Eio implementation for POSIX systems")
+ (description "An Eio implementation for most Unix-like platforms")
+ (depends
+  (eio (= :version))
+  (iomux (>= 0.2))
+  (mdx (and (>= 1.10.0) :with-test))
+  (fmt (>= 0.8.9))))
 (package
  (name eio_luv)
  (synopsis "Eio implementation using luv (libuv)")
- (description "An eio implementation for most platforms, using luv.")
+ (description "An Eio implementation for most platforms, using luv.")
  (depends
   (eio (= :version))
   (luv (>= 0.5.11))
@@ -55,8 +64,10 @@
  (name eio_main)
  (synopsis "Effect-based direct-style IO mainloop for OCaml")
  (description "Selects an appropriate Eio backend for the current platform.")
+ (depopts eio_luv)
  (depends
-  (eio_linux (and (= :version) (= :os "linux")))
   (mdx (and (>= 1.10.0) :with-test))
-  (eio_luv (= :version))))
+  (eio_linux (and (= :version) (= :os "linux")))
+  (eio_posix (and (= :version) (<> :os "windows")))
+  (eio_luv (and (= :version) (or (= :os "windows") :with-test)))))
 (using mdx 0.2)

eio_linux.opam

@@ -1,7 +1,7 @@
 # This file is generated by dune, edit dune-project instead
 opam-version: "2.0"
 synopsis: "Eio implementation for Linux using io-uring"
-description: "An eio implementation for Linux using io-uring."
+description: "An Eio implementation for Linux using io-uring."
 maintainer: ["[email protected]"]
 authors: ["Anil Madhavapeddy" "Thomas Leonard"]
 license: "ISC"

eio_luv.opam

@@ -1,7 +1,7 @@
 # This file is generated by dune, edit dune-project instead
 opam-version: "2.0"
 synopsis: "Eio implementation using luv (libuv)"
-description: "An eio implementation for most platforms, using luv."
+description: "An Eio implementation for most platforms, using luv."
 maintainer: ["[email protected]"]
 authors: ["Anil Madhavapeddy" "Thomas Leonard"]
 license: "ISC"

eio_main.opam

@@ -10,11 +10,13 @@ doc: "https://ocaml-multicore.github.io/eio/"
 bug-reports: "https://github.com/ocaml-multicore/eio/issues"
 depends: [
   "dune" {>= "3.0"}
-  "eio_linux" {= version & os = "linux"}
   "mdx" {>= "1.10.0" & with-test}
-  "eio_luv" {= version}
+  "eio_linux" {= version & os = "linux"}
+  "eio_posix" {= version & os != "windows"}
+  "eio_luv" {= version & os = "windows" | with-test}
   "odoc" {with-doc}
 ]
+depopts: ["eio_luv"]
 build: [
   ["dune" "subst"] {dev}
   [

eio_posix.opam

@@ -0,0 +1,33 @@
+# This file is generated by dune, edit dune-project instead
+opam-version: "2.0"
+synopsis: "Eio implementation for POSIX systems"
+description: "An Eio implementation for most Unix-like platforms"
+maintainer: ["[email protected]"]
+authors: ["Anil Madhavapeddy" "Thomas Leonard"]
+license: "ISC"
+homepage: "https://github.com/ocaml-multicore/eio"
+doc: "https://ocaml-multicore.github.io/eio/"
+bug-reports: "https://github.com/ocaml-multicore/eio/issues"
+depends: [
+  "dune" {>= "3.0"}
+  "eio" {= version}
+  "iomux" {>= "0.2"}
+  "mdx" {>= "1.10.0" & with-test}
+  "fmt" {>= "0.8.9"}
+  "odoc" {with-doc}
+]
+build: [
+  ["dune" "subst"] {dev}
+  [
+    "dune"
+    "build"
+    "-p"
+    name
+    "-j"
+    jobs
+    "@install"
+    "@runtest" {with-test}
+    "@doc" {with-doc}
+  ]
+]
+dev-repo: "git+https://github.com/ocaml-multicore/eio.git"

lib_eio_posix/domain_mgr.ml

@@ -0,0 +1,87 @@
+(*
+ * Copyright (C) 2023 Thomas Leonard
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *)
+
+open Eio.Std
+
+[@@@alert "-unstable"]
+
+(* Run an event loop in the current domain, using [fn x] as the root fiber. *)
+let run_event_loop fn x =
+  Sched.with_sched @@ fun sched ->
+  let open Effect.Deep in
+  let extra_effects : _ effect_handler = {
+    effc = fun (type a) (e : a Effect.t) : ((a, Sched.exit) continuation -> Sched.exit) option ->
+      match e with
+      | Eio_unix.Private.Get_monotonic_clock -> Some (fun k -> continue k (Time.mono_clock : Eio.Time.Mono.t))
+      | Eio_unix.Private.Socket_of_fd (sw, close_unix, unix_fd) -> Some (fun k ->
+          let fd = Fd.of_unix ~sw ~blocking:false ~close_unix unix_fd in
+          Unix.set_nonblock unix_fd;
+          continue k (Flow.of_fd fd :> Eio_unix.socket)
+        )
+      | Eio_unix.Private.Socketpair (sw, domain, ty, protocol) -> Some (fun k ->
+          match
+            let unix_a, unix_b = Unix.socketpair ~cloexec:true domain ty protocol in
+            let a = Fd.of_unix ~sw ~blocking:false ~close_unix:true unix_a in
+            let b = Fd.of_unix ~sw ~blocking:false ~close_unix:true unix_b in
+            Unix.set_nonblock unix_a;
+            Unix.set_nonblock unix_b;
+            (Flow.of_fd a :> Eio_unix.socket), (Flow.of_fd b :> Eio_unix.socket)
+          with
+          | r -> continue k r
+          | exception Unix.Unix_error (code, name, arg) ->
+              discontinue k (Err.wrap code name arg)
+        )
+      | Eio_unix.Private.Pipe sw -> Some (fun k ->
+          match
+            let unix_r, unix_w = Unix.pipe ~cloexec:true () in
+            let r = Fd.of_unix ~sw ~blocking:false ~close_unix:true unix_r in
+            let w = Fd.of_unix ~sw ~blocking:false ~close_unix:true unix_w in
+            Unix.set_nonblock unix_r;
+            Unix.set_nonblock unix_w;
+            let source = (Flow.of_fd r :> <Eio.Flow.source; Eio.Flow.close; Eio_unix.unix_fd>) in
+            let sink = (Flow.of_fd w :> <Eio.Flow.sink; Eio.Flow.close; Eio_unix.unix_fd>) in
+            (source, sink)
+          with
+          | r -> continue k r
+          | exception Unix.Unix_error (code, name, arg) ->
+            discontinue k (Err.wrap code name arg)
+        )
+      | _ -> None
+  }
+  in
+  Sched.run ~extra_effects sched fn x
+
+let v = object
+  inherit Eio.Domain_manager.t
+
+  method run_raw fn =
+    let domain = ref None in
+    Eio.Private.Suspend.enter (fun _ctx enqueue ->
+        domain := Some (Domain.spawn (fun () -> Fun.protect fn ~finally:(fun () -> enqueue (Ok ()))))
+      );
+    Domain.join (Option.get !domain)
+
+  method run fn =
+    let domain = ref None in
+    Eio.Private.Suspend.enter (fun ctx enqueue ->
+        let cancelled, set_cancelled = Promise.create () in
+        Eio.Private.Fiber_context.set_cancel_fn ctx (Promise.resolve set_cancelled);
+        domain := Some (Domain.spawn (fun () ->
+            Fun.protect (run_event_loop (fun () -> fn ~cancelled))
+              ~finally:(fun () -> enqueue (Ok ()))))
+      );
+    Domain.join (Option.get !domain)
+end

lib_eio_posix/dune

@@ -0,0 +1,13 @@
+(library
+ (name eio_posix)
+ (public_name eio_posix)
+ (enabled_if (= %{os_type} "Unix"))
+ (foreign_stubs
+  (language c)
+  (flags :standard -D_LARGEFILE64_SOURCE)
+  (names eio_posix_stubs))
+ (libraries eio eio.utils eio.unix fmt iomux))
+
+(rule
+ (targets config.ml)
+ (action (run ./include/discover.exe)))

lib_eio_posix/eio_posix.ml

@@ -0,0 +1,51 @@
+(*
+ * Copyright (C) 2023 Thomas Leonard
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *)
+
+module Low_level = Low_level
+
+type stdenv = <
+  stdin  : <Eio.Flow.source; Eio_unix.unix_fd>;
+  stdout : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  stderr : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  net : Eio.Net.t;
+  domain_mgr : Eio.Domain_manager.t;
+  clock : Eio.Time.clock;
+  mono_clock : Eio.Time.Mono.t;
+  fs : Eio.Fs.dir Eio.Path.t;
+  cwd : Eio.Fs.dir Eio.Path.t;
+  secure_random : Eio.Flow.source;
+  debug : Eio.Debug.t;
+>
+
+let run main =
+  (* SIGPIPE makes no sense in a modern application. *)
+  Sys.(set_signal sigpipe Signal_ignore);
+  let stdin = (Flow.of_fd Low_level.Fd.stdin :> <Eio.Flow.source; Eio_unix.unix_fd>) in
+  let stdout = (Flow.of_fd Low_level.Fd.stdout :> <Eio.Flow.sink; Eio_unix.unix_fd>) in
+  let stderr = (Flow.of_fd Low_level.Fd.stderr :> <Eio.Flow.sink; Eio_unix.unix_fd>) in
+  Domain_mgr.run_event_loop main @@ object (_ : stdenv)
+    method stdin = stdin
+    method stdout = stdout
+    method stderr = stderr
+    method debug = Eio.Private.Debug.v
+    method clock = Time.clock
+    method mono_clock = Time.mono_clock
+    method net = Net.v
+    method domain_mgr = Domain_mgr.v
+    method cwd = ((Fs.cwd, "") :> Eio.Fs.dir Eio.Path.t)
+    method fs = ((Fs.fs, "") :> Eio.Fs.dir Eio.Path.t)
+    method secure_random = Flow.secure_random
+  end

lib_eio_posix/eio_posix.mli

@@ -0,0 +1,24 @@
+(** Fallback Eio backend for POSIX systems. *)
+
+type stdenv = <
+  stdin  : <Eio.Flow.source; Eio_unix.unix_fd>;
+  stdout : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  stderr : <Eio.Flow.sink; Eio_unix.unix_fd>;
+  net : Eio.Net.t;
+  domain_mgr : Eio.Domain_manager.t;
+  clock : Eio.Time.clock;
+  mono_clock : Eio.Time.Mono.t;
+  fs : Eio.Fs.dir Eio.Path.t;
+  cwd : Eio.Fs.dir Eio.Path.t;
+  secure_random : Eio.Flow.source;
+  debug : Eio.Debug.t;
+>
+(** An extended version of {!Eio.Stdenv.t} with some extra features available on POSIX systems. *)
+
+val run : (stdenv -> 'a) -> 'a
+(** [run main] runs an event loop and calls [main stdenv] inside it.
+
+    For portable code, you should use {!Eio_main.run} instead, which will call this for you if appropriate. *)
+
+module Low_level = Low_level
+(** Low-level API for making POSIX calls directly. *)