diff --git a/src/irmin-cli/cli.ml b/src/irmin-cli/cli.ml index 5ef2a3cbe7..90c3cdcc06 100644 --- a/src/irmin-cli/cli.ml +++ b/src/irmin-cli/cli.ml @@ -19,6 +19,8 @@ open Cmdliner open Resolver module Graphql = Irmin_graphql_unix +type eio = Import.eio + let deprecated_info = (Term.info [@alert "-deprecated"]) let deprecated_man_format = (Term.man_format [@alert "-deprecated"]) let deprecated_eval_choice = (Term.eval_choice [@alert "-deprecated"]) @@ -53,7 +55,7 @@ let term_info title ~doc ~man = deprecated_info ~sdocs:global_option_section ~docs:global_option_section ~doc ~man title -type command = (unit Term.t * Term.info[@alert "-deprecated"]) +type command = env:eio -> (unit Term.t * Term.info[@alert "-deprecated"]) type sub = { name : string; @@ -62,7 +64,8 @@ type sub = { term : unit Term.t; } -let create_command c = +let create_command c ~env = + let c = c ~env in let man = [ `S "DESCRIPTION"; `P c.doc ] @ c.man in (c.term, term_info c.name ~doc:c.doc ~man) @@ -113,14 +116,14 @@ let run t = try t () with err -> print_exc err let mk (fn : 'a) : 'a Term.t = Term.(const (fun () -> fn) $ setup_log) (* INIT *) -let init = +let init ~env = { name = "init"; doc = "Initialize a store."; man = []; term = (let init (S (_, _store, _)) = () in - Term.(mk init $ store ())); + Term.(mk init $ store ~env)); } let print fmt = Fmt.kstr print_endline fmt @@ -136,7 +139,7 @@ let branch f x = get "branch" f x let commit f x = get "commit" f x (* GET *) -let get = +let get ~env = { name = "get"; doc = "Read the value associated with a key."; @@ -152,11 +155,11 @@ let get = exit 1 | Some v -> print "%a" (Irmin.Type.pp S.Contents.t) v in - Term.(mk get $ store () $ path)); + Term.(mk get $ store ~env $ path)); } (* LIST *) -let list = +let list ~env = { name = "list"; doc = "List subdirectories."; @@ -180,11 +183,11 @@ let list = in List.iter (print "%a" pp) paths in - Term.(mk list $ store () $ path_or_empty)); + Term.(mk list $ store ~env $ path_or_empty)); } (* TREE *) -let tree = +let tree ~env = { name = "tree"; doc = "List the store contents."; @@ -236,7 +239,7 @@ let tree = print "%s%s%s" k dots v) all in - Term.(mk tree $ store ())); + Term.(mk tree $ store ~env)); } let author = @@ -248,7 +251,7 @@ let message = Arg.(value & opt (some string) None & doc) (* SET *) -let set = +let set ~env = { name = "set"; doc = "Update the value associated with a key."; @@ -267,11 +270,11 @@ let set = let value = value S.Contents.t v in S.set_exn t ~info:(info (module S) ?author "%s" message) path value in - Term.(mk set $ store () $ author $ message $ path $ v)); + Term.(mk set $ store ~env $ author $ message $ path $ v)); } (* REMOVE *) -let remove = +let remove ~env = { name = "remove"; doc = "Delete a key."; @@ -288,7 +291,7 @@ let remove = ~info:(info (module S) ?author "%s" message) (key S.Path.t path) in - Term.(mk remove $ store () $ author $ message $ path)); + Term.(mk remove $ store ~env $ author $ message $ path)); } let apply e f = @@ -298,7 +301,7 @@ let apply e f = | r, _ -> r (* CLONE *) -let clone = +let clone ~env = { name = "clone"; doc = "Copy a remote respository to a local store"; @@ -316,11 +319,11 @@ let clone = | Ok `Empty -> () | Error (`Msg e) -> failwith e in - Term.(mk clone $ Resolver.remote () $ depth)); + Term.(mk clone $ Resolver.remote ~env $ depth)); } (* FETCH *) -let fetch = +let fetch ~env = { name = "fetch"; doc = "Download objects and refs from another repository."; @@ -338,11 +341,11 @@ let fetch = let _ = Sync.pull_exn t x `Set in () in - Term.(mk fetch $ Resolver.remote ())); + Term.(mk fetch $ Resolver.remote ~env)); } (* MERGE *) -let merge = +let merge ~env = { name = "merge"; doc = "Merge branches."; @@ -371,11 +374,11 @@ let merge = let doc = Arg.info ~docv:"BRANCH" ~doc:"Branch to merge from." [] in Arg.(required & pos 0 (some string) None & doc) in - Term.(mk merge $ store () $ author $ message $ branch_name)); + Term.(mk merge $ store ~env $ author $ message $ branch_name)); } (* PULL *) -let pull = +let pull ~env = { name = "pull"; doc = "Fetch and merge with another repository."; @@ -394,11 +397,11 @@ let pull = in () in - Term.(mk pull $ remote () $ author $ message)); + Term.(mk pull $ remote ~env $ author $ message)); } (* PUSH *) -let push = +let push ~env = { name = "push"; doc = "Update remote references along with associated objects."; @@ -414,11 +417,11 @@ let push = let _ = Sync.push_exn t x in () in - Term.(mk push $ remote ())); + Term.(mk push $ remote ~env)); } (* SNAPSHOT *) -let snapshot = +let snapshot ~env = { name = "snapshot"; doc = "Return a snapshot for the current state of the database."; @@ -432,11 +435,11 @@ let snapshot = print "%a" S.Commit.pp_hash k; () in - Term.(mk snapshot $ store ())); + Term.(mk snapshot $ store ~env)); } (* REVERT *) -let revert = +let revert ~env = { name = "revert"; doc = "Revert the contents of the store to a previous state."; @@ -457,7 +460,7 @@ let revert = | Some s -> S.Head.set t s | None -> failwith "invalid commit" in - Term.(mk revert $ store () $ snapshot)); + Term.(mk revert $ store ~env $ snapshot)); } (* WATCH *) @@ -543,7 +546,7 @@ let handle_diff (type a b) and type Schema.Metadata.t = S.metadata) diff command proc -let watch = +let watch ~env = { name = "watch"; doc = "Get notifications when values change."; @@ -573,11 +576,11 @@ let watch = let doc = Arg.info ~docv:"COMMAND" ~doc:"Command to execute" [] in Arg.(value & pos_right 0 string [] & doc) in - Term.(mk watch $ store () $ path $ command)); + Term.(mk watch $ store ~env $ path $ command)); } (* DOT *) -let dot = +let dot ~env = { name = "dot"; doc = "Dump the contents of the store as a Graphviz file."; @@ -639,7 +642,7 @@ let dot = in if i <> 0 then [%logs.err "The %s.dot is corrupted" basename]) in - Term.(mk dot $ store () $ basename $ depth $ no_dot_call $ full)); + Term.(mk dot $ store ~env $ basename $ depth $ no_dot_call $ full)); } let config_man = @@ -676,7 +679,7 @@ let config_man = @ help_sections ) (* HELP *) -let help = +let help ~env:_ = { name = "help"; doc = "Display help about Irmin and Irmin commands."; @@ -710,7 +713,7 @@ let help = } (* GRAPHQL *) -let graphql = +let graphql ~env = { name = "graphql"; doc = "Run a graphql server."; @@ -748,19 +751,19 @@ let graphql = ~mode:(`TCP (`Port port)) server in - Term.(mk graphql $ store () $ port $ addr)); + Term.(mk graphql $ store ~env $ port $ addr)); } (* SERVER *) -let server = +let server ~env = { name = "server"; doc = "Run irmin-server."; man = []; - term = Server.main_term; + term = Server.main_term ~env; } -let options = +let options ~env = { name = "options"; doc = "Get information about backend specific configuration options."; @@ -768,26 +771,22 @@ let options = term = (let options (store, hash, contents) = let module Conf = Irmin.Backend.Conf in - let store, _ = Resolver.load_config ?store ?hash ?contents () in + let store, _ = Resolver.load_config ~env ?store ?hash ?contents () in let spec = Store.spec store in Seq.iter (fun (Conf.K k) -> let name = Conf.name k in if name = "root" || name = "uri" then () else - let ty = Conf.ty k in + let ty = Conf.typename k in let doc = Conf.doc k |> Option.value ~default:"" in - let ty = - Fmt.str "%a" Irmin.Type.pp_ty ty - |> Astring.String.filter (fun c -> c <> '\n') - in Fmt.pr "%s: %s\n\t%s\n" name ty doc) (Conf.Spec.keys spec) in Term.(mk options $ Store.term ())); } -let branches = +let branches ~env = { name = "branches"; doc = "List branches"; @@ -800,7 +799,7 @@ let branches = let branches = S.Branch.list (S.repo t) in List.iter (Fmt.pr "%a\n" (Irmin.Type.pp S.branch_t)) branches in - Term.(mk branches $ store ())); + Term.(mk branches $ store ~env)); } let weekday Unix.{ tm_wday; _ } = @@ -830,7 +829,7 @@ let month Unix.{ tm_mon; _ } = | 11 -> "Dec" | _ -> assert false -let log = +let log ~env = { name = "log"; doc = "List commits"; @@ -911,10 +910,59 @@ let log = () with Sys_error s when String.equal s "Broken pipe" -> () in - Term.(mk commits $ store () $ plain $ pager $ num $ skip $ reverse)); + Term.(mk commits $ store ~env $ plain $ pager $ num $ skip $ reverse)); } -let default = +let commands : (env:eio -> sub) list = + [ + help; + init; + get; + set; + remove; + list; + tree; + clone; + fetch; + merge; + pull; + push; + snapshot; + revert; + watch; + dot; + graphql; + server; + options; + branches; + log; + ] + +let common_commands = + [ + init; + get; + set; + remove; + list; + tree; + clone; + fetch; + merge; + pull; + push; + snapshot; + revert; + watch; + dot; + graphql; + server; + options; + branches; + log; + ] + +let default ~env = let doc = "Irmin, the database that never forgets." in let man = [ @@ -933,66 +981,28 @@ let default = "usage: irmin [--version]\n\ \ [--help]\n\ \ []\n\n\ - The most commonly used subcommands are:\n\ - \ init %s\n\ - \ get %s\n\ - \ set %s\n\ - \ remove %s\n\ - \ list %s\n\ - \ tree %s\n\ - \ clone %s\n\ - \ fetch %s\n\ - \ merge %s\n\ - \ pull %s\n\ - \ push %s\n\ - \ snapshot %s\n\ - \ revert %s\n\ - \ watch %s\n\ - \ dot %s\n\ - \ graphql %s\n\ - \ server %s\n\ - \ options %s\n\ - \ branches %s\n\ - \ log %s\n\n\ - See `irmin help ` for more information on a specific command.\n\ - %!" - init.doc get.doc set.doc remove.doc list.doc tree.doc clone.doc fetch.doc - merge.doc pull.doc push.doc snapshot.doc revert.doc watch.doc dot.doc - graphql.doc server.doc options.doc branches.doc log.doc + The most commonly used subcommands are:\n"; + List.iter + (fun cmd -> + let cmd = cmd ~env in + Fmt.pr " %-11s %s\n" cmd.name cmd.doc) + common_commands; + Fmt.pr + "\n\ + See `irmin help ` for more information on a specific command.@." in ( Term.(mk usage $ const ()), deprecated_info "irmin" ~version:Irmin.version ~sdocs:global_option_section ~doc ~man ) -let commands = - List.map create_command - [ - help; - init; - get; - set; - remove; - list; - tree; - clone; - fetch; - merge; - pull; - push; - snapshot; - revert; - watch; - dot; - graphql; - server; - options; - branches; - log; - ] +let commands = List.map create_command commands let run ~default:x y = Eio_main.run @@ fun env -> - Irmin_fs.run env#fs @@ fun () -> Lwt_eio.with_event_loop ~clock:env#clock @@ fun _ -> Irmin.Backend.Watch.set_listen_dir_hook Irmin_watcher.hook; - match deprecated_eval_choice x y with `Error _ -> exit 1 | _ -> () + let env = (env :> eio) in + let run cmd = cmd ~env in + match deprecated_eval_choice (run x) (List.map run y) with + | `Error _ -> exit 1 + | _ -> () diff --git a/src/irmin-cli/cli.mli b/src/irmin-cli/cli.mli index 6141f56afa..70a4f45b95 100644 --- a/src/irmin-cli/cli.mli +++ b/src/irmin-cli/cli.mli @@ -16,7 +16,10 @@ (** CLI commands. *) -type command = (unit Cmdliner.Term.t * Cmdliner.Term.info[@alert "-deprecated"]) +type eio = Import.eio + +type command = + env:eio -> (unit Cmdliner.Term.t * Cmdliner.Term.info[@alert "-deprecated"]) (** [Cmdliner] commands. *) val default : command @@ -38,5 +41,5 @@ type sub = { } (** Subcommand. *) -val create_command : sub -> command +val create_command : (env:eio -> sub) -> command (** Build a subcommand. *) diff --git a/src/irmin-cli/dune b/src/irmin-cli/dune index 8496c072ab..53ba1c313d 100644 --- a/src/irmin-cli/dune +++ b/src/irmin-cli/dune @@ -19,6 +19,7 @@ cohttp-lwt-unix unix yaml + eio eio_main lwt_eio) (preprocess diff --git a/src/irmin-cli/import.ml b/src/irmin-cli/import.ml index fff6d09bdb..bb36eaba59 100644 --- a/src/irmin-cli/import.ml +++ b/src/irmin-cli/import.ml @@ -15,3 +15,7 @@ *) include Irmin.Export_for_backends + +type eio = + < cwd : Eio.Fs.dir_ty Eio.Path.t + ; clock : float Eio.Time.clock_ty Eio.Time.clock > diff --git a/src/irmin-cli/resolver.ml b/src/irmin-cli/resolver.ml index 4d2106ef08..5c9dea5ced 100644 --- a/src/irmin-cli/resolver.ml +++ b/src/irmin-cli/resolver.ml @@ -23,21 +23,23 @@ let global_option_section = "COMMON OPTIONS" module Conf = Irmin.Backend.Conf -let try_parse ty v = - match Irmin.Type.of_string ty v with +let try_parse of_string v = + match of_string v with | Error e -> ( let x = Format.sprintf "{\"some\": %s}" v in - match Irmin.Type.of_string ty x with + match of_string x with | Error _ -> let y = Format.sprintf "{\"some\": \"%s\"}" v in - Irmin.Type.of_string ty y |> Result.map_error (fun _ -> e) + of_string y |> Result.map_error (fun _ -> e) | v -> v) | v -> v let pconv t = let pp = Irmin.Type.pp t in let parse s = - match try_parse t s with Ok x -> `Ok x | Error (`Msg e) -> `Error e + match try_parse (Irmin.Type.of_string t) s with + | Ok x -> `Ok x + | Error (`Msg e) -> `Error e in (parse, pp) @@ -296,7 +298,14 @@ module Store = struct v spec (module S) let mem = create Irmin_mem.Conf.spec (module Irmin_mem) - let fs = create Irmin_fs.Conf.spec (module Irmin_fs_unix) + + let fs env = + let spec = + Irmin_fs_unix.spec ~path:(Eio.Stdenv.cwd env) + ~clock:(Eio.Stdenv.clock env) + in + create spec (module Irmin_fs_unix) + let git (module C : Irmin.Contents.S) = v_git (module Xgit.FS.KV (C)) let git_mem (module C : Irmin.Contents.S) = v_git (module Xgit.Mem.KV (C)) @@ -324,23 +333,24 @@ module Store = struct let all = ref [ - ("git", Fixed_hash git); - ("git-mem", Fixed_hash git_mem); - ("fs", Variable_hash fs); - ("mem", Variable_hash mem); - ("pack", Variable_hash pack); - ("tezos", Fixed tezos); + ("git", fun _ -> Fixed_hash git); + ("git-mem", fun _ -> Fixed_hash git_mem); + ("fs", fun env -> Variable_hash (fs env)); + ("mem", fun _ -> Variable_hash mem); + ("pack", fun _ -> Variable_hash pack); + ("tezos", fun _ -> Fixed tezos); ] let default = "git" |> fun n -> ref (n, List.assoc n !all) let add name ?default:(x = false) m = + let m (_ : eio) = m in all := (name, m) :: !all; if x then default := (name, m) - let find name = + let find name env = match List.assoc_opt (String.Ascii.lowercase name) !all with - | Some s -> s + | Some s -> s env | None -> let valid = String.concat ~sep:", " (List.split !all |> fst) in let msg = @@ -456,10 +466,10 @@ let parse_config ?root y spec = | Some (Irmin.Backend.Conf.K k), Some v -> let v = json_of_yaml v |> Yojson.Basic.to_string in let v = - match Irmin.Type.of_json_string (Conf.ty k) v with + match Conf.of_json_string k v with | Error _ -> let v = Format.sprintf "{\"some\": %s}" v in - Irmin.Type.of_json_string (Conf.ty k) v |> Result.get_ok + Conf.of_json_string k v |> Result.get_ok | Ok v -> v in Conf.add config k v @@ -475,7 +485,7 @@ let parse_config ?root y spec = let config = match (root, Conf.Spec.find_key spec "root") with | Some root, Some (K r) -> - let v = Irmin.Type.of_string (Conf.ty r) root |> Result.get_ok in + let v = Conf.of_string r root |> Result.get_ok in Conf.add config r v | _ -> config in @@ -489,7 +499,7 @@ let load_plugin ?plugin config = | Ok (Some v) -> Dynlink.loadfile_private (Yaml.Util.to_string_exn v) | _ -> ()) -let get_store ?plugin config (store, hash, contents) = +let get_store ~env ?plugin config (store, hash, contents) = let () = load_plugin ?plugin config in let store = match store with @@ -500,6 +510,7 @@ let get_store ?plugin config (store, hash, contents) = match store with Some s -> Store.find s | None -> Store.find s) | _ -> snd !Store.default) in + let store = store env in let contents = match contents with | Some s -> Contents.find s @@ -532,9 +543,9 @@ let get_store ?plugin config (store, hash, contents) = | _ -> Fmt.failwith "Cannot customize the hash function for the given store") -let load_config ?plugin ?root ?config_path ?store ?hash ?contents () = +let load_config ~env ?plugin ?root ?config_path ?store ?hash ?contents () = let y = read_config_file config_path in - let store = get_store ?plugin y (store, hash, contents) in + let store = get_store ~env ?plugin y (store, hash, contents) in let spec = Store.spec store in let config = parse_config ?root y spec in (store, config) @@ -564,10 +575,10 @@ let get_commit (type a b) | None -> of_string (find_key config "commit") | Some t -> of_string (Some t) -let build_irmin_config config root opts (store, hash, contents) branch commit - plugin : store = +let build_irmin_config ~env config root opts (store, hash, contents) branch + commit plugin : store = let (T { impl; spec; remote }) = - get_store ?plugin config (store, hash, contents) + get_store ~env ?plugin config (store, hash, contents) in let (module S) = Store.Impl.generic_keyed impl in let branch = get_branch (module S) config branch in @@ -586,8 +597,7 @@ let build_irmin_config config root opts (store, hash, contents) branch commit | Some x -> x | None -> invalid_arg ("opt: " ^ k) in - let ty = Conf.ty key in - let v = try_parse ty v |> Result.get_ok in + let v = try_parse (Conf.of_string key) v |> Result.get_ok in let config = Conf.add config key v in config) config (List.flatten opts) @@ -626,10 +636,10 @@ let plugin = let doc = "Register new contents, store or hash types" in Arg.(value & opt (some string) None & info ~doc [ "plugin" ]) -let store () = +let store ~env = let create plugin store (root, config_path, opts) branch commit = let y = read_config_file config_path in - build_irmin_config y root opts store branch commit plugin + build_irmin_config ~env y root opts store branch commit plugin in Term.(const create $ plugin $ Store.term () $ config_term $ branch $ commit) @@ -653,7 +663,7 @@ type Irmin.remote += R of Cohttp.Header.t option * string (* FIXME: this is a very crude heuristic to choose the remote kind. Would be better to read the config file and look for remote alias. *) -let infer_remote hash contents branch headers str = +let infer_remote ~env hash contents branch headers str = let hash = match hash with None -> snd !Hash.default | Some c -> c in let contents = match contents with @@ -664,7 +674,7 @@ let infer_remote hash contents branch headers str = let r = if Sys.file_exists (str / ".git") then Store.git contents else if Sys.file_exists (str / "store.dict") then Store.pack hash contents - else Store.fs hash contents + else Store.fs env hash contents in match r with | Store.T { impl; spec; _ } -> @@ -673,7 +683,7 @@ let infer_remote hash contents branch headers str = let config = match Conf.Spec.find_key spec "root" with | Some (K r) -> - let v = Irmin.Type.of_string (Conf.ty r) str |> Result.get_ok in + let v = Conf.of_string r str |> Result.get_ok in Conf.add config r v | _ -> config in @@ -691,7 +701,7 @@ let infer_remote hash contents branch headers str = in R (headers, str) -let remote () = +let remote ~env = let repo = let doc = Arg.info ~docv:"REMOTE" @@ -703,9 +713,10 @@ let remote () = headers str = let y = read_config_file config_path in let store = - build_irmin_config y root opts (store, hash, contents) branch commit None + build_irmin_config ~env y root opts (store, hash, contents) branch commit + None in - let remote () = infer_remote hash contents branch headers str in + let remote () = infer_remote ~env hash contents branch headers str in (store, remote) in Term.( diff --git a/src/irmin-cli/resolver.mli b/src/irmin-cli/resolver.mli index 9fbc8f637f..fc90d466f2 100644 --- a/src/irmin-cli/resolver.mli +++ b/src/irmin-cli/resolver.mli @@ -43,6 +43,8 @@ type contents = Contents.t (** {1 Global Configuration} *) +type eio := Import.eio + module Store : sig module Impl : sig (** The type of {i implementations} of an Irmin store. @@ -86,10 +88,10 @@ module Store : sig t val mem : hash -> contents -> t - val fs : hash -> contents -> t + val fs : eio -> hash -> contents -> t val git : contents -> t val pack : hash -> contents -> t - val find : string -> store_functor + val find : string -> eio -> store_functor val add : string -> ?default:bool -> store_functor -> unit val spec : t -> Irmin.Backend.Conf.Spec.t val generic_keyed : t -> (module Irmin.Generic_key.S) @@ -103,6 +105,7 @@ end (** {1 Stores} *) val load_config : + env:eio -> ?plugin:string -> ?root:string -> ?config_path:string -> @@ -126,10 +129,10 @@ val load_config : type store = | S : 'a Store.Impl.t * (unit -> 'a) * Store.remote_fn option -> store -val store : unit -> store Cmdliner.Term.t +val store : env:eio -> store Cmdliner.Term.t (** Parse the command-line arguments and then the config file. *) type Irmin.remote += R of Cohttp.Header.t option * string -val remote : unit -> (store * (unit -> Irmin.remote)) Cmdliner.Term.t +val remote : env:eio -> (store * (unit -> Irmin.remote)) Cmdliner.Term.t (** Parse a remote store location. *) diff --git a/src/irmin-cli/server.ml b/src/irmin-cli/server.ml index 380615d2a6..20471e2ca6 100644 --- a/src/irmin-cli/server.ml +++ b/src/irmin-cli/server.ml @@ -29,10 +29,11 @@ let setup_log = Cmdliner.Term.( const setup_log $ Fmt_cli.style_renderer () $ Logs_cli.level ()) -let main ~readonly ~root ~uri ~tls ~store ~contents ~hash ~dashboard +let main ~env ~readonly ~root ~uri ~tls ~store ~contents ~hash ~dashboard ~config_path (module Codec : Conn.Codec.S) fingerprint = + Lwt_eio.run_lwt @@ fun () -> let store, config = - Resolver.load_config ?root ?config_path ?store ?hash ?contents () + Resolver.load_config ~env ?root ?config_path ?store ?hash ?contents () in let config = Irmin_server.Cli.Conf.v config uri in let (module Store : Irmin.Generic_key.S) = @@ -61,16 +62,15 @@ let main ~readonly ~root ~uri ~tls ~store ~contents ~hash ~dashboard Logs.app (fun l -> l "Listening on %a, store: %s" Uri.pp_hum uri root); Server.serve server -let main readonly root uri tls (store, hash, contents) codec config_path +let main ~env readonly root uri tls (store, hash, contents) codec config_path dashboard fingerprint () = let codec = match codec with | `Bin -> (module Conn.Codec.Bin : Conn.Codec.S) | `Json -> (module Conn.Codec.Json) in - Lwt_main.run - @@ main ~readonly ~root ~uri ~tls ~store ~contents ~hash ~config_path - ~dashboard codec fingerprint + main ~env ~readonly ~root ~uri ~tls ~store ~contents ~hash ~config_path + ~dashboard codec fingerprint open Cmdliner @@ -107,9 +107,9 @@ let dashboard = in Arg.(value @@ opt (some int) None doc) -let main_term = +let main_term ~env = Term.( - const main + const (main ~env) $ readonly $ root $ Irmin_server.Cli.uri diff --git a/src/irmin-client/unix/bin/client.ml b/src/irmin-client/unix/bin/client.ml index 8ac3c90651..63bd18f250 100644 --- a/src/irmin-client/unix/bin/client.ml +++ b/src/irmin-client/unix/bin/client.ml @@ -260,7 +260,7 @@ let iterations = in Arg.(value @@ opt int 1 doc) -let config = +let config ~env = let create uri (branch : string option) tls (store, hash, contents) codec config_path () = let codec = @@ -270,7 +270,7 @@ let config = in let (module Codec) = codec in let store, config = - Irmin_cli.Resolver.load_config ?config_path ?store ?hash ?contents () + Irmin_cli.Resolver.load_config ~env ?config_path ?store ?hash ?contents () in let config = Irmin_server.Cli.Conf.v config uri in let (module Store : Irmin.Generic_key.S) = @@ -298,6 +298,8 @@ let help = (Term.info "irmin-client" [@alert "-deprecated"]) ) let[@alert "-deprecated"] () = + Eio_main.run @@ fun env -> + let config = config ~env:(env :> Irmin_cli.eio) in Term.exit @@ Term.eval_choice help [ diff --git a/src/irmin-fs/irmin_fs.ml b/src/irmin-fs/irmin_fs.ml index c08c50353e..4fdc93759f 100644 --- a/src/irmin-fs/irmin_fs.ml +++ b/src/irmin-fs/irmin_fs.ml @@ -15,43 +15,49 @@ *) open! Import -open Eio open Astring let src = Logs.Src.create "irmin.fs" ~doc:"Irmin disk persistence" module Log = (val Logs.src_log src : Logs.LOG) -let ( / ) = Path.( / ) +let ( / ) = Filename.concat module type Config = sig - val dir : Fs.dir_ty Path.t -> Fs.dir_ty Path.t + val dir : string -> string val file_of_key : string -> string val key_of_file : string -> string end module type IO = sig - type path = Fs.dir_ty Path.t + type io - val rec_files : path -> path list - val file_exists : path -> bool - val read_file : path -> string option - val mkdir : path -> unit + val io_of_config : Irmin.config -> io + + type path = string + + val rec_files : io:io -> path -> path list + val file_exists : io:io -> path -> bool + val read_file : io:io -> path -> string option + val mkdir : io:io -> path -> unit type lock - val lock_file : path -> lock - val write_file : ?temp_dir:path -> ?lock:lock -> path -> string -> unit + val lock_file : io:io -> path -> lock + + val write_file : + io:io -> temp_dir:path -> ?lock:lock -> path -> string -> unit val test_and_set_file : - ?temp_dir:path -> + io:io -> + temp_dir:path -> lock:lock -> path -> test:string option -> set:string option -> bool - val remove_file : ?lock:lock -> path -> unit + val remove_file : io:io -> ?lock:lock -> path -> unit end (* ~path *) @@ -76,15 +82,15 @@ module Read_only_ext struct type key = K.t type value = V.t - type 'a t = { path : Fs.dir_ty Path.t } + type 'a t = { path : string; io : IO.io } let get_path config = Option.value Conf.(find_root config) ~default:"." let v config = - let fs = Irmin.Backend.Conf.Env.fs () in - let path = Path.(fs / get_path config) in - IO.mkdir path; - { path } + let io = IO.io_of_config config in + let path = get_path config in + IO.mkdir ~io path; + { path; io } let close _ = () let cast t = (t :> read_write t) @@ -93,12 +99,13 @@ struct let file_of_key { path; _ } key = path / S.file_of_key (Irmin.Type.to_string K.t key) - let lock_of_key { path; _ } key = - IO.lock_file (path / "lock" / S.file_of_key (Irmin.Type.to_string K.t key)) + let lock_of_key { io; path } key = + IO.lock_file ~io + (path / "lock" / S.file_of_key (Irmin.Type.to_string K.t key)) let mem t key = let file = file_of_key t key in - IO.file_exists file + IO.file_exists ~io:t.io file let of_bin_string = Irmin.Type.(unstage (of_bin_string V.t)) @@ -113,17 +120,17 @@ struct let find t key = [%log.debug "find %a" pp_key key]; - match IO.read_file (file_of_key t key) with + match IO.read_file ~io:t.io (file_of_key t key) with | None -> None | Some x -> value x - let list t = + let list { path; io } = [%log.debug "list"]; - let files = IO.rec_files (S.dir t.path) in + let files = IO.rec_files ~io (S.dir path) in let files = - let p = String.length (snd t.path) in + let p = String.length path in List.fold_left - (fun acc (_, file) -> + (fun acc file -> let n = String.length file in if n <= p + 1 then acc else @@ -156,11 +163,11 @@ struct [%log.debug "add %a" pp_key key]; let file = file_of_key t key in let temp_dir = temp_dir t in - match IO.file_exists file with + match IO.file_exists ~io:t.io file with | true -> () | false -> let str = to_bin_string value in - IO.write_file ~temp_dir file str + IO.write_file ~io:t.io ~temp_dir file str end module Atomic_write_ext @@ -217,7 +224,7 @@ struct [%log.err "listen_dir: %s" e]; None in - W.listen_dir t.w (snd dir) ~key ~value:(RO.find t.t) + W.listen_dir t.w dir ~key ~value:(RO.find t.t) let watch_key t key ?init f = let stop = listen_dir t in @@ -240,14 +247,14 @@ struct let temp_dir = temp_dir t in let file = RO.file_of_key t.t key in let lock = RO.lock_of_key t.t key in - IO.write_file ~temp_dir file ~lock (raw_value value); + IO.write_file ~io:t.t.io ~temp_dir file ~lock (raw_value value); W.notify t.w key (Some value) let remove t key = [%log.debug "remove %a" RO.pp_key key]; let file = RO.file_of_key t.t key in let lock = RO.lock_of_key t.t key in - let () = IO.remove_file ~lock file in + let () = IO.remove_file ~io:t.t.io ~lock file in W.notify t.w key None let test_and_set t key ~test ~set = @@ -257,8 +264,8 @@ struct let lock = RO.lock_of_key t.t key in let raw_value = function None -> None | Some v -> Some (raw_value v) in let b = - IO.test_and_set_file file ~temp_dir ~lock ~test:(raw_value test) - ~set:(raw_value set) + IO.test_and_set_file ~io:t.t.io file ~temp_dir ~lock + ~test:(raw_value test) ~set:(raw_value set) in let () = if b then W.notify t.w key set in b @@ -266,7 +273,8 @@ struct let clear t = [%log.debug "clear"]; let remove_file key () = - IO.remove_file ~lock:(RO.lock_of_key t.t key) (RO.file_of_key t.t key) + IO.remove_file ~io:t.t.io ~lock:(RO.lock_of_key t.t key) + (RO.file_of_key t.t key) in list t |> fun keys -> Eio.Fiber.all (List.map remove_file keys) end @@ -330,26 +338,31 @@ module KV (IO : IO) = struct end module IO_mem = struct + type io = unit + + let io_of_config _ = () + + type path = string + type t = { watches : (string, string -> unit) Hashtbl.t; - files : (Fs.dir_ty Path.t, string) Hashtbl.t; + files : (path, string) Hashtbl.t; } let t = { watches = Hashtbl.create 3; files = Hashtbl.create 13 } - type path = Fs.dir_ty Path.t type lock = Eio.Mutex.t let locks = Hashtbl.create 10 - let lock_file (_, file) = + let lock_file ~io:() file = try Hashtbl.find locks file with Not_found -> let l = Eio.Mutex.create () in Hashtbl.add locks file l; l - let with_lock l f = + let with_lock ~io:() l f = match l with None -> f () | Some l -> Eio.Mutex.use_rw ~protect:false l f let set_listen_hook () = @@ -363,29 +376,28 @@ module IO_mem = struct Hashtbl.iter (fun dir f -> if String.is_prefix ~affix:dir file then f file) t.watches - (* |> Eio.Fiber.all *) - let mkdir _ = () + let mkdir ~io:() _ = () - let remove_file ?lock file = - with_lock lock (fun () -> Hashtbl.remove t.files file) + let remove_file ~io ?lock file = + with_lock ~io lock (fun () -> Hashtbl.remove t.files file) - let rec_files (_, dir) = + let rec_files ~io:() dir = Hashtbl.fold - (fun ((_, k) as v) _ acc -> - if String.is_prefix ~affix:dir k then v :: acc else acc) + (fun file _ acc -> + if String.is_prefix ~affix:dir file then file :: acc else acc) t.files [] - let file_exists file = Hashtbl.mem t.files file + let file_exists ~io:() file = Hashtbl.mem t.files file - let read_file file = + let read_file ~io:() file = try let buf = Hashtbl.find t.files file in Some buf with Not_found -> None - let write_file ?temp_dir:_ ?lock ((_, file) as f) v = - let () = with_lock lock (fun () -> Hashtbl.replace t.files f v) in + let write_file ~io ~temp_dir:_ ?(lock : lock option) file v = + let () = with_lock ~io lock (fun () -> Hashtbl.replace t.files file v) in notify file let equal x y = @@ -394,7 +406,7 @@ module IO_mem = struct | Some x, Some y -> String.equal x y | _ -> false - let test_and_set_file ?temp_dir:_ ~lock file ~test ~set = + let test_and_set_file ~io ~temp_dir:_ ~lock file ~test ~set = let f () = let old = try Some (Hashtbl.find t.files file) with Not_found -> None in let b = @@ -408,10 +420,10 @@ module IO_mem = struct Hashtbl.replace t.files file v; true in - let () = if b then notify (snd file) in + let () = if b then notify file in b in - with_lock (Some lock) f + with_lock ~io (Some lock) f let clear () = Hashtbl.clear t.files; @@ -423,17 +435,3 @@ module Maker_is_a_maker : Irmin.Maker = Maker (IO_mem) (* Enforce that {!KV} is a sub-type of {!Irmin.KV_maker}. *) module KV_is_a_KV : Irmin.KV_maker = KV (IO_mem) - -let run (fs : Fs.dir_ty Path.t) fn = - Switch.run @@ fun sw -> - Irmin.Backend.Watch.set_watch_switch sw; - let open Effect.Deep in - try_with fn () - { - effc = - (fun (type a) (e : a Effect.t) -> - match e with - | Irmin.Backend.Conf.Env.Fs -> - Some (fun (k : (a, _) continuation) -> continue k fs) - | _ -> None); - } diff --git a/src/irmin-fs/irmin_fs.mli b/src/irmin-fs/irmin_fs.mli index a463246c6b..da2242530a 100644 --- a/src/irmin-fs/irmin_fs.mli +++ b/src/irmin-fs/irmin_fs.mli @@ -33,37 +33,43 @@ val config : string -> Irmin.config module type IO = sig (** {1 File-system abstractions} *) - type path = Eio.Fs.dir_ty Eio.Path.t + type io + + val io_of_config : Irmin.config -> io + + type path = string (** The type for paths. *) (** {2 Read operations} *) - val rec_files : path -> path list + val rec_files : io:io -> path -> path list (** [rec_files dir] is the list of files recursively present in [dir] and all of its sub-directories. Return filenames prefixed by [dir]. *) - val file_exists : path -> bool + val file_exists : io:io -> path -> bool (** [file_exist f] is true if [f] exists. *) - val read_file : path -> string option + val read_file : io:io -> path -> string option (** Read the contents of a file using mmap. *) (** {2 Write Operations} *) - val mkdir : path -> unit + val mkdir : io:io -> path -> unit (** Create a directory. *) type lock (** The type for file locks. *) - val lock_file : path -> lock + val lock_file : io:io -> path -> lock (** [lock_file f] is the lock associated to the file [f]. *) - val write_file : ?temp_dir:path -> ?lock:lock -> path -> string -> unit + val write_file : + io:io -> temp_dir:path -> ?lock:lock -> path -> string -> unit (** Atomic writes. *) val test_and_set_file : - ?temp_dir:path -> + io:io -> + temp_dir:path -> lock:lock -> path -> test:string option -> @@ -71,7 +77,7 @@ module type IO = sig bool (** Test and set. *) - val remove_file : ?lock:lock -> path -> unit + val remove_file : io:io -> ?lock:lock -> path -> unit (** Remove a file or directory (even if non-empty). *) end @@ -83,10 +89,9 @@ module KV (IO : IO) : Irmin.KV_maker with type info = Irmin.Info.default (** {2 Advanced configuration} *) module type Config = sig - open Eio (** Same as [Config] but gives more control on the file hierarchy. *) - val dir : Fs.dir_ty Path.t -> Fs.dir_ty Path.t + val dir : string -> string (** [dir root] is the sub-directory to look for the keys. *) val file_of_key : string -> string @@ -108,5 +113,3 @@ module IO_mem : sig val clear : unit -> unit val set_listen_hook : unit -> unit end - -val run : Eio.Fs.dir_ty Eio.Path.t -> (unit -> 'a) -> 'a diff --git a/src/irmin-fs/unix/dune b/src/irmin-fs/unix/dune index b8cb6dd1d8..324d06cd44 100644 --- a/src/irmin-fs/unix/dune +++ b/src/irmin-fs/unix/dune @@ -1,7 +1,7 @@ (library (public_name irmin-fs.unix) (name irmin_fs_unix) - (libraries irmin-fs irmin.unix lwt eio eio.unix) + (libraries irmin-fs irmin.unix eio eio.unix) (preprocess (pps ppx_irmin.internal)) (instrumentation diff --git a/src/irmin-fs/unix/eio_pool.ml b/src/irmin-fs/unix/eio_pool.ml index e32c122638..1c0b75cdab 100644 --- a/src/irmin-fs/unix/eio_pool.ml +++ b/src/irmin-fs/unix/eio_pool.ml @@ -103,8 +103,7 @@ let acquire p = (* Limit reached: wait for a free one. *) let promise, resolver = Promise.create () in Stream.add p.waiters resolver; - validate_and_return p (Promise.await_exn promise) - (* (Lwt.add_task_r [@ocaml.warning "-3"]) p.waiters >>= validate_and_return p *)) + validate_and_return p (Promise.await_exn promise)) else (* Take the first free member and validate it. *) let c = Queue.take p.list in diff --git a/src/irmin-fs/unix/irmin_fs_unix.ml b/src/irmin-fs/unix/irmin_fs_unix.ml index ef7253ad7b..9e19cde381 100644 --- a/src/irmin-fs/unix/irmin_fs_unix.ml +++ b/src/irmin-fs/unix/irmin_fs_unix.ml @@ -14,6 +14,7 @@ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. *) +module Conf = Irmin.Backend.Conf include Irmin.Export_for_backends open Eio @@ -21,9 +22,51 @@ let src = Logs.Src.create "fs.unix" ~doc:"logs fs unix events" module Log = (val Logs.src_log src : Logs.LOG) +type fs = Eio.Fs.dir_ty Eio.Path.t +type clock = float Eio.Time.clock_ty Eio.Time.clock +type io = { fs : fs; clock : clock } + +let fs_typ : fs Conf.Typ.t = Conf.Typ.create () +let clock_typ : clock Conf.Typ.t = Conf.Typ.create () + +let spec ~path:fs ~clock = + let spec = Conf.Spec.v "irmin-fs.unix" in + let fs = (fs :> fs) in + let _fs_key = + let to_string fs = Eio.Path.native_exn fs in + let of_string str = Ok Eio.Path.(fs / str) in + let of_json_string str = + match Irmin.Type.(of_json_string string) str with + | Ok str -> Ok Eio.Path.(fs / str) + | Error e -> Error e + in + Conf.key' ~typ:fs_typ ~spec ~typename:"_ Eio.Path.t" ~to_string ~of_string + ~of_json_string "fs" fs + in + let clock = (clock :> clock) in + let _clock_key = + let to_string _ = "Eio.Time.clock" in + let of_string _ = Ok clock in + let of_json_string _ = Ok clock in + Conf.key' ~typ:clock_typ ~spec ~typename:"_ Eio.Time.clock" ~to_string + ~of_string ~of_json_string "clock" clock + in + spec + +let conf ~path ~clock = Conf.empty (spec ~path ~clock) + module IO = struct + type nonrec io = io + + let io_of_config conf = + { + fs = Conf.find_key conf "fs" fs_typ; + clock = Conf.find_key conf "clock" clock_typ; + } + + type path = string + let mkdir_pool = Eio_pool.create 1 (fun () -> ()) - let mmap_threshold = 4096 (* Files smaller than this are loaded using [read]. Use of mmap is necessary to handle packfiles efficiently. Since these are stored @@ -37,66 +80,41 @@ module IO = struct (* Pool of opened files *) let openfile_pool = Eio_pool.create 200 (fun () -> ()) - let protect_unix_exn = function - | Unix.Unix_error _ as e -> raise (Failure (Printexc.to_string e)) - | e -> raise e - - let ignore_enoent = function - | Unix.Unix_error (Unix.ENOENT, _, _) -> () - | e -> raise e - - let protect f x = try f x with exn -> protect_unix_exn exn - let safe f x = try f x with exn -> ignore_enoent exn - let mkdir dirname = - let rec aux ((_, path) as dir) = - if Sys.file_exists path && Sys.is_directory path then () - else ( - if Sys.file_exists path then ( - [%log.debug "%s already exists but is a file, removing." path]; - safe Path.unlink dir); - let parent = (fst dir, Filename.dirname @@ snd dir) in - aux parent; - [%log.debug "mkdir %s" path]; - protect (Path.mkdir ~perm:0o755) dir) - in - (* TODO: Pool *) - Eio_pool.use mkdir_pool (fun () -> aux dirname) + Eio_pool.use mkdir_pool (fun () -> + Eio.Path.mkdirs ~exists_ok:true ~perm:0o755 dirname) - let file_exists (_, f) = - try Sys.file_exists f with - (* See https://github.com/ocsigen/lwt/issues/316 *) - | Unix.Unix_error (Unix.ENOTDIR, _, _) -> false - | e -> raise e + let mkdir_parent file = + match Eio.Path.split file with + | None -> () + | Some (parent, _) -> mkdir parent + + let file_exists ~io:{ fs; _ } filename = Eio.Path.(is_file (fs / filename)) module Lock = struct - let is_stale max_age file = + let is_stale ~io:{ clock; _ } max_age file = try - let s = Eio_unix.run_in_systhread (fun () -> Unix.stat file) in - if s.Unix.st_mtime < 1.0 (* ??? *) then false - else Unix.gettimeofday () -. s.Unix.st_mtime > max_age - with - | Unix.Unix_error (Unix.ENOENT, _, _) -> false - | e -> raise e + let { Eio.File.Stat.mtime; _ } = Eio.Path.stat ~follow:false file in + if mtime < 1.0 (* ??? *) then false + else Eio.Time.now clock -. mtime > max_age + with Eio.Io (Eio.Fs.E (Not_found _), _) -> false let unlock file = Path.unlink file - let lock ?(max_age = 10. *. 60. (* 10 minutes *)) ?(sleep = 0.001) - ((_, file) as fcap) = + let lock ~io ?(max_age = 10. *. 60. (* 10 minutes *)) ?(sleep = 0.001) file + = let rec aux i = - [%log.debug "lock %s %d" file i]; - let is_stale = is_stale max_age file in - if is_stale then ( - [%log.err "%s is stale, removing it." file]; - unlock fcap; + [%log.debug "lock %a %d" Eio.Path.pp file i]; + if is_stale ~io max_age file then ( + [%log.err "%a is stale, removing it." Eio.Path.pp file]; + unlock file; aux 1) else let create () = let pid = Unix.getpid () in - let parent = (fst fcap, Filename.dirname file) in - mkdir parent; + mkdir_parent file; Switch.run @@ fun sw -> - let flow = Path.open_out ~sw fcap ~create:(`Exclusive 0o600) in + let flow = Path.open_out ~sw file ~create:(`Exclusive 0o600) in Flow.copy_string (string_of_int pid) flow in try create () with @@ -107,180 +125,131 @@ module IO = struct (let i = float i in i *. i) in - Eio_unix.sleep (sleep *. backoff); + Eio.Time.sleep io.clock (sleep *. backoff); aux (i + 1) | e -> raise e in aux 1 - let with_lock file fn = + let with_lock ~io file fn = match file with | None -> fn () | Some f -> - lock f; + lock ~io f; Fun.protect fn ~finally:(fun () -> unlock f) end - type path = Eio.Fs.dir_ty Eio.Path.t - (* we use file locking *) - type lock = path + type lock = Eio.Fs.dir_ty Eio.Path.t - let lock_file x = x - let file_exists = file_exists + let lock_file ~io:{ fs; _ } x = Path.(fs / x) - let list_files kind ((_, dir) as v) = - if Sys.file_exists dir && Sys.is_directory dir then - let d = Path.read_dir v in - let d = List.sort String.compare d in - let d = List.map (Path.( / ) v) d in + let list_files kind dir = + if Eio.Path.is_directory dir then + let d = Path.read_dir dir in + let d = List.map (Path.( / ) dir) d in let d = List.filter kind d in d else [] - let directories dir = - list_files - (fun (_, f) -> try Sys.is_directory f with Sys_error _ -> false) - dir - - let files dir = - list_files - (fun (_, f) -> try not (Sys.is_directory f) with Sys_error _ -> false) - dir + let directories dir = list_files Eio.Path.is_directory dir + let files dir = list_files Eio.Path.is_file dir let write_string fd b = match String.length b with 0 -> () | _len -> Flow.copy_string b fd let _delays = Array.init 20 (fun i -> 0.1 *. (float i ** 2.)) + let remove_dir dir = Eio.Path.rmtree dir - let command fmt = - Printf.ksprintf - (fun str -> - [%log.debug "[exec] %s" str]; - let i = Sys.command str in - if i <> 0 then [%log.debug "[exec] error %d" i]) - fmt - - let remove_dir dir = - if Sys.os_type = "Win32" then command "cmd /d /v:off /c rd /s /q %S" dir - else command "rm -rf %S" dir - - let remove_file ?lock ((_, file) as f) = - Lock.with_lock lock (fun () -> - try Path.unlink f with - (* On Windows, [EACCES] can also occur in an attempt to - rename a file or directory or to remove an existing - directory. *) - | Unix.Unix_error (Unix.EACCES, _, _) - | Unix.Unix_error (Unix.EISDIR, _, _) - | Eio.Io (Eio.Exn.X (Eio_unix.Unix_error (Unix.EACCES, _, _)), _) - | Eio.Io (Eio.Exn.X (Eio_unix.Unix_error (Unix.EISDIR, _, _)), _) -> - remove_dir file - | Unix.Unix_error (Unix.ENOENT, _, _) - | Eio.Io (Eio.Fs.E (Fs.Not_found _), _) -> - () - | e -> raise e) - - let rename tmp file = Path.rename tmp file - - let with_write_file ?temp_dir file fn = - let () = match temp_dir with None -> () | Some d -> mkdir d in - let dir = (fst file, Filename.dirname @@ snd file) in - mkdir dir; - let temp_dir_path = Option.get temp_dir in - let temp_dir = snd temp_dir_path in - let file_f = snd file in - let tmp_f = - Filename.temp_file ~temp_dir (Filename.basename file_f) "write" + let remove_file ~io ?lock file = + Lock.with_lock ~io lock (fun () -> + let file = Path.(io.fs / file) in + if Path.is_directory file then remove_dir file + else + try Path.unlink file + with Eio.Io (Eio.Fs.E (Fs.Not_found _), _) -> ()) + + let temp_file ~temp_dir file suffix = + let basename = + match Eio.Path.split file with + | None -> "tmp" + | Some (_, basename) -> basename + in + let rec go i = + let tmp = Eio.Path.(temp_dir / (basename ^ string_of_int i ^ suffix)) in + if Eio.Path.kind ~follow:false tmp = `Not_found then tmp else go (i + 1) in - let tmp_name = Filename.basename tmp_f in + go 0 + + let with_write_file ~temp_dir file fn = + mkdir temp_dir; + mkdir_parent file; + let tmp_file = temp_file ~temp_dir file "write" in Eio_pool.use openfile_pool (fun () -> - [%log.debug - "Writing %s (%s) %s %s" file_f tmp_f (snd temp_dir_path) (snd file)]; - Path.( - with_open_out ~create:(`Or_truncate 0o644) (temp_dir_path / tmp_name) - fn); - rename Path.(temp_dir_path / tmp_name) file) + [%log.debug "Writing %a (%a)" Eio.Path.pp file Eio.Path.pp tmp_file]; + Path.(with_open_out ~create:(`Or_truncate 0o644) tmp_file fn); + Path.rename tmp_file file) let read_file_with_read file size = - (* let chunk_size = max 4096 (min size 0x100000) in *) let buf = Cstruct.create size in - (* let flags = [ Unix.O_RDONLY ] in - let perm = 0o0 in *) - (* let* fd = Lwt_unix.openfile file flags perm in *) Path.with_open_in file @@ fun flow -> - try - Flow.read_exact flow buf; - Cstruct.to_string buf - with End_of_file -> Cstruct.to_string buf - - let read_file_with_mmap file = - let open Bigarray in - let fd = Unix.(openfile file [ O_RDONLY; O_NONBLOCK ] 0o644) in - let ba = - Unix.map_file fd char c_layout false [| -1 |] - |> Bigarray.array1_of_genarray - in - Unix.close fd; + Flow.read_exact flow buf; + Cstruct.to_string buf - (* XXX(samoht): ideally we should not do a copy here. *) - Bigstringaf.to_string ba - - let read_file file = - let file_f = snd file in + let read_file ~io:{ fs; _ } file = try + let file = Path.(fs / file) in Eio_pool.use openfile_pool (fun () -> - [%log.debug "Reading %s" file_f]; - let stats = Unix.stat file_f in - let size = stats.Unix.st_size in - let buf = - if size >= mmap_threshold then read_file_with_mmap file_f - else read_file_with_read file size - in + [%log.debug "Reading %a" Eio.Path.pp file]; + let { Eio.File.Stat.size; _ } = Eio.Path.stat ~follow:false file in + let size = Optint.Int63.to_int size in + let buf = read_file_with_read file size in Some buf) - with - | Unix.Unix_error _ | Sys_error _ -> None - | e -> raise e + with Eio.Io _ -> None - let write_file ?temp_dir ?lock file b = + let write_file ~io ~temp_dir ?(lock : lock option) file b = + let file = Path.(io.fs / file) in + let temp_dir = Path.(io.fs / temp_dir) in let write () = - with_write_file file ?temp_dir (fun fd -> write_string fd b) + with_write_file file ~temp_dir (fun fd -> write_string fd b) in - Lock.with_lock lock (fun () -> - try write () with - | Unix.Unix_error (Unix.EISDIR, _, _) -> - remove_dir (snd file); - write () - | e -> raise e) - - let test_and_set_file ?temp_dir ~lock file ~test ~set = - Lock.with_lock (Some lock) (fun () -> - let v = read_file file in + Lock.with_lock ~io lock (fun () -> + if Path.is_directory file then remove_dir file; + write ()) + + let test_and_set_file ~io ~temp_dir ~lock file ~test ~set = + Lock.with_lock ~io (Some lock) (fun () -> + let v = read_file ~io file in let equal = match (test, v) with | None, None -> true - | Some x, Some y -> x = y (* TODO *) + | Some x, Some y -> String.equal x y | _ -> false in if not equal then false else let () = match set with - | None -> remove_file file - | Some v -> write_file ?temp_dir file v + | None -> remove_file ~io file + | Some v -> write_file ~io ~temp_dir file v in true) - let rec_files dir : Fs.dir_ty Path.t list = + let rec_files ~io:{ fs; _ } dir : path list = + let dir = Path.(fs / dir) in let rec aux accu dir = let ds = directories dir in let fs = files dir in List.fold_left aux (fs @ accu) ds in - aux [] dir + aux [] dir |> List.map snd + + let mkdir ~io:{ fs; _ } dirname = mkdir Path.(fs / dirname) end -module Append_only = Irmin_fs.Append_only (IO) +module Append_only (K : Irmin.Type.S) (V : Irmin.Type.S) = + Irmin_fs.Append_only (IO) (K) (V) + module Atomic_write = Irmin_fs.Atomic_write (IO) include Irmin_fs.Maker (IO) module KV = Irmin_fs.KV (IO) diff --git a/src/irmin-fs/unix/irmin_fs_unix.mli b/src/irmin-fs/unix/irmin_fs_unix.mli index bcfe077790..b75a48b0bb 100644 --- a/src/irmin-fs/unix/irmin_fs_unix.mli +++ b/src/irmin-fs/unix/irmin_fs_unix.mli @@ -28,3 +28,10 @@ module Maker_ext (Obj : Irmin_fs.Config) (Ref : Irmin_fs.Config) : Irmin.Maker (** {1 Common Unix utilities} *) include module type of Irmin_unix + +(** {1 Backend-specific config} *) + +val spec : + path:_ Eio.Path.t -> clock:_ Eio.Time.clock -> Irmin.Backend.Conf.Spec.t + +val conf : path:_ Eio.Path.t -> clock:_ Eio.Time.clock -> Irmin.Backend.Conf.t diff --git a/src/irmin/conf.ml b/src/irmin/conf.ml index d66e810a12..c694a1241d 100644 --- a/src/irmin/conf.ml +++ b/src/irmin/conf.ml @@ -16,6 +16,25 @@ *) open! Import +type (_, _) eq = Refl : ('a, 'a) eq + +module Typ = struct + type 'a s = .. + type 'a t = { s : 'a s; eq : 'b. 'b s -> ('a, 'b) eq option } + + let create (type a) () : a t = + let open struct + type _ s += S : a s + + let eq : type b. b s -> (a, b) eq option = function + | S -> Some Refl + | _ -> None + end in + { s = S; eq } + + let equal a b = a.eq b.s +end + module Univ = struct type t = exn @@ -31,8 +50,12 @@ type 'a key = { doc : string option; docv : string option; docs : string option; - ty : 'a Type.t; + typename : string; + to_string : 'a -> string; + of_string : string -> ('a, [ `Msg of string ]) result; + of_json_string : string -> ('a, [ `Msg of string ]) result; default : 'a; + typ : 'a Typ.t; to_univ : 'a -> Univ.t; of_univ : Univ.t -> 'a option; } @@ -50,20 +73,9 @@ module Spec = struct type t = { name : string; mutable keys : k M.t } - let all = Hashtbl.create 8 - - let v name = - let keys = M.empty in - if Hashtbl.mem all name then - Fmt.failwith "Config spec already exists: %s" name; - let x = { name; keys } in - Hashtbl.replace all name x; - x - + let v name = { name; keys = M.empty } let name { name; _ } = name let update spec name k = spec.keys <- M.add name k spec.keys - let list () = Hashtbl.to_seq_values all - let find name = Hashtbl.find_opt all name let find_key spec name = M.find_opt name spec.keys let keys spec = M.to_seq spec.keys |> Seq.map snd let clone { name; keys } = { name; keys } @@ -87,7 +99,8 @@ type t = Spec.t * Univ.t M.t let spec = fst -let key ?docs ?docv ?doc ?(allow_duplicate = false) ~spec name ty default = +let key' ?docs ?docv ?doc ?(allow_duplicate = false) ?typ ~spec ~typename + ~to_string ~of_string ~of_json_string name default = let () = String.iter (function @@ -99,16 +112,44 @@ let key ?docs ?docv ?doc ?(allow_duplicate = false) ~spec name ty default = | Some _ when allow_duplicate = false -> Fmt.invalid_arg "duplicate key: %s" name | _ -> + let typ = match typ with Some typ -> typ | None -> Typ.create () in let to_univ, of_univ = Univ.create () in - let k = { name; ty; default; to_univ; of_univ; doc; docv; docs } in + let k = + { + name; + to_string; + of_json_string; + of_string; + default; + typename; + typ; + to_univ; + of_univ; + doc; + docv; + docs; + } + in Spec.update spec name (K k); k +let key ?docs ?docv ?doc ?allow_duplicate ?typ ~spec name ty default = + let to_string = Type.to_string ty in + let typename = + Fmt.str "%a" Type.pp_ty ty |> Astring.String.filter (fun c -> c <> '\n') + in + let of_string = Type.of_string ty in + let of_json_string = Type.of_json_string ty in + key' ?docs ?docv ?doc ?allow_duplicate ?typ ~spec ~typename ~to_string + ~of_json_string ~of_string name default + let name t = t.name let doc t = t.doc let docv t = t.docv let docs t = t.docs -let ty t = t.ty +let typename t = t.typename +let of_string t = t.of_string +let of_json_string t = t.of_json_string let default t = t.default let empty spec = (spec, M.empty) let singleton spec k v = (spec, M.singleton (K k) (k.to_univ v)) @@ -143,6 +184,15 @@ let get (_, d) k = | None -> raise Not_found with Not_found -> k.default +let find_key : type a. t -> string -> a Typ.t -> a = + fun ((spec, _) as t) name typ -> + match Spec.find_key spec name with + | Some (K k) -> ( + match Typ.equal k.typ typ with + | Some Refl -> get t k + | None -> raise Not_found) + | None -> raise Not_found + let keys (_, conf) = M.to_seq conf |> Seq.map (fun (k, _) -> k) let with_spec (_, conf) spec = (spec, conf) @@ -152,7 +202,7 @@ let to_strings (_, conf) = |> Seq.map (fun (K k, v) -> ( k.name, match k.of_univ v with - | Some v -> Type.to_string k.ty v + | Some v -> k.to_string v | None -> assert false )) let pp ppf t = @@ -177,13 +227,4 @@ let find_root (spec, d) : string option = | None -> None | Some (K k) -> ( let v = find (spec, d) k in - match v with None -> None | Some v -> Some (Type.to_string k.ty v)) - -module Env = struct - type _ Effect.t += - | Fs : Eio.Fs.dir_ty Eio.Path.t Effect.t - | Net : _ Eio.Net.t Effect.t - - let fs () = Effect.perform Fs - let net () = Effect.perform Net -end + match v with None -> None | Some v -> Some (k.to_string v)) diff --git a/src/irmin/conf.mli b/src/irmin/conf.mli index c312c0b6d1..533d277bb8 100644 --- a/src/irmin/conf.mli +++ b/src/irmin/conf.mli @@ -36,12 +36,6 @@ module Spec : sig val name : t -> string (** [name spec] is the name associated with a config spec *) - val list : unit -> t Seq.t - (** [list ()] is a sequence containing all available config specs *) - - val find : string -> t option - (** [find name] is the config spec associated with [name] if available *) - val find_key : t -> string -> k option (** [find_key spec k] is the key associated with the name [k] in [spec] *) @@ -55,11 +49,18 @@ module Spec : sig the specs in [b] joined by hyphens. *) end +module Typ : sig + type 'a t + + val create : unit -> 'a t +end + val key : ?docs:string -> ?docv:string -> ?doc:string -> ?allow_duplicate:bool -> + ?typ:'a Typ.t -> spec:Spec.t -> string -> 'a Type.t -> @@ -83,11 +84,36 @@ val key : if [allow_duplicate] is [false] (the default) and [name] has already been used to create a key *) +val key' : + ?docs:string -> + ?docv:string -> + ?doc:string -> + ?allow_duplicate:bool -> + ?typ:'a Typ.t -> + spec:Spec.t -> + typename:string -> + to_string:('a -> string) -> + of_string:(string -> ('a, [ `Msg of string ]) result) -> + of_json_string:(string -> ('a, [ `Msg of string ]) result) -> + string -> + 'a -> + 'a key +(** Same as {!key} for types that don't implement [Type.t] but can be serialized + with [to_string], and deserialized with either [of_string] or + [of_json_string]. The [typename] is the user-readable description of the + type, in case of dynamic type errors. *) + val name : 'a key -> string (** The key name. *) -val ty : 'a key -> 'a Type.t -(** [tc k] is [k]'s converter. *) +val typename : 'a key -> string +(** [typename k] is the type name of [k]'s values. *) + +val of_string : 'a key -> string -> ('a, [ `Msg of string ]) result +(** [of_string k] is the parser of [k]'s values. *) + +val of_json_string : 'a key -> string -> ('a, [ `Msg of string ]) result +(** [of_json_string k] is the json parser of [k]'s values. *) val default : 'a key -> 'a (** [default k] is [k]'s default value. *) @@ -154,6 +180,10 @@ val keys : t -> k Seq.t val with_spec : t -> Spec.t -> t (** [with_spec t s] is the config [t] with spec [s] *) +val find_key : t -> string -> 'a Typ.t -> 'a +(** [find_key t name typ] returns the value associated with [name] in the config + [t]. *) + val verify : t -> t (** [verify t] is an identity function that ensures all keys match the spec @@ -166,12 +196,3 @@ val uri : Uri.t Type.t val find_root : t -> string option (** [find_root c] is [root]'s mapping in [c], if any. *) - -module Env : sig - type _ Effect.t += - | Fs : Eio.Fs.dir_ty Eio.Path.t Effect.t - | Net : _ Eio.Net.t Effect.t - - val fs : unit -> Eio.Fs.dir_ty Eio.Path.t - val net : unit -> _ Eio.Net.t -end diff --git a/src/libirmin/config.ml b/src/libirmin/config.ml index 29fab1e0d6..50a87aeec5 100644 --- a/src/libirmin/config.ml +++ b/src/libirmin/config.ml @@ -40,10 +40,11 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct fn "config_pack" (string_opt @-> string_opt @-> returning config) (fun hash contents -> + run_env @@ fun env -> try let hash = Option.map Irmin_cli.Resolver.Hash.find hash in let c : config = - Irmin_cli.Resolver.load_config ~store:"pack" ?hash ?contents () + Irmin_cli.Resolver.load_config ~env ~store:"pack" ?hash ?contents () in Root.create_config c with _ -> null config) @@ -52,8 +53,11 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct fn "config_tezos" (void @-> returning config) (fun () -> + run_env @@ fun env -> try - let c : config = Irmin_cli.Resolver.load_config ~store:"tezos" () in + let c : config = + Irmin_cli.Resolver.load_config ~env ~store:"tezos" () + in Root.create_config c with _ -> null config) @@ -61,8 +65,11 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct fn "config_git" (string_opt @-> returning config) (fun contents -> + run_env @@ fun env -> try - let c = Irmin_cli.Resolver.load_config ~store:"git" ?contents () in + let c = + Irmin_cli.Resolver.load_config ~env ~store:"git" ?contents () + in Root.create_config c with _ -> null config) @@ -70,9 +77,10 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct fn "config_git_mem" (string_opt @-> returning config) (fun contents -> + run_env @@ fun env -> try let c = - Irmin_cli.Resolver.load_config ~store:"git-mem" ?contents () + Irmin_cli.Resolver.load_config ~env ~store:"git-mem" ?contents () in Root.create_config c with _ -> null config) @@ -81,10 +89,11 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct fn "config_fs" (string_opt @-> string_opt @-> returning config) (fun hash contents -> + run_env @@ fun env -> try let hash = Option.map Irmin_cli.Resolver.Hash.find hash in let c = - Irmin_cli.Resolver.load_config ~store:"irf" ?hash ?contents () + Irmin_cli.Resolver.load_config ~env ~store:"irf" ?hash ?contents () in Root.create_config c with _ -> null config) @@ -93,10 +102,11 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct fn "config_mem" (string_opt @-> string_opt @-> returning config) (fun hash contents -> + run_env @@ fun env -> try let hash = Option.map Irmin_cli.Resolver.Hash.find hash in let c = - Irmin_cli.Resolver.load_config ~store:"mem" ?hash ?contents () + Irmin_cli.Resolver.load_config ~env ~store:"mem" ?hash ?contents () in Root.create_config c with _ -> null config) @@ -116,7 +126,7 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct | None -> (false, config) | Some (Irmin.Backend.Conf.K k) -> let t : a Irmin.Type.t = Root.get_ty ty in - if type_name t <> type_name (Irmin.Backend.Conf.ty k) then + if type_name t <> Irmin.Backend.Conf.typename k then (false, config) else let value = Root.get_value value in @@ -139,8 +149,7 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct | None -> (false, config) | Some (Irmin.Backend.Conf.K k) -> let path = - Irmin.Type.of_string (Irmin.Backend.Conf.ty k) path - |> Result.get_ok + Irmin.Backend.Conf.of_string k path |> Result.get_ok in (true, Irmin.Backend.Conf.add config k path) in diff --git a/src/libirmin/util.ml b/src/libirmin/util.ml index 3b0ad52b31..e2ab02c80c 100644 --- a/src/libirmin/util.ml +++ b/src/libirmin/util.ml @@ -43,9 +43,12 @@ module Make (I : Cstubs_inverted.INTERNAL) = struct let fn name t f = I.internal ~runtime_lock:false ("irmin_" ^ name) t f - let run fn = + let run_env fn = Eio_main.run @@ fun env -> - Lwt_eio.with_event_loop ~clock:env#clock @@ fun _ -> fn () + Lwt_eio.with_event_loop ~clock:env#clock @@ fun () -> + fn (env :> Irmin_cli.eio) + + let run fn = run_env (fun _ -> fn ()) module Root = struct let to_voidp t x = Ctypes.coerce t (ptr void) x diff --git a/test/irmin-cli/test.ml b/test/irmin-cli/test.ml index 01123ad540..d04fbe4802 100644 --- a/test/irmin-cli/test.ml +++ b/test/irmin-cli/test.ml @@ -17,10 +17,10 @@ include Irmin.Export_for_backends module Conf = struct - let test_config () = + let test_config ~env = let hash = Irmin_cli.Resolver.Hash.find "blake2b" in let _, cfg = - Irmin_cli.Resolver.load_config ~config_path:"test/irmin-cli/test.yml" + Irmin_cli.Resolver.load_config ~env ~config_path:"test/irmin-cli/test.yml" ~store:"pack" ~contents:"string" ~hash () in let spec = Irmin.Backend.Conf.spec cfg in @@ -34,8 +34,11 @@ module Conf = struct Alcotest.(check int) "index-log-size" 1234 index_log_size; Alcotest.(check bool) "fresh" true fresh - let misc : unit Alcotest.test_case list = - [ ("config", `Quick, fun () -> test_config ()) ] + let misc ~env : unit Alcotest.test_case list = + [ ("config", `Quick, fun () -> test_config ~env) ] end -let () = Alcotest.run "irmin-cli" [ ("conf", Conf.misc) ] +let () = + Eio_main.run @@ fun env -> + let env = (env :> Irmin_cli.eio) in + Alcotest.run "irmin-cli" [ ("conf", Conf.misc ~env) ] diff --git a/test/irmin-cli/test_command_line.t b/test/irmin-cli/test_command_line.t index cd99931ee0..3e11108b7b 100644 --- a/test/irmin-cli/test_command_line.t +++ b/test/irmin-cli/test_command_line.t @@ -94,3 +94,33 @@ Clone a local repo irmin: [WARNING] Updating the control file to [Used_non_minimal_indexing_strategy]. It won't be possible to GC this irmin-pack store anymore. $ irmin get --root ./cloned a/b/c 123 + +Show documentation + $ irmin + usage: irmin [--version] + [--help] + [] + + The most commonly used subcommands are: + init Initialize a store. + get Read the value associated with a key. + set Update the value associated with a key. + remove Delete a key. + list List subdirectories. + tree List the store contents. + clone Copy a remote respository to a local store + fetch Download objects and refs from another repository. + merge Merge branches. + pull Fetch and merge with another repository. + push Update remote references along with associated objects. + snapshot Return a snapshot for the current state of the database. + revert Revert the contents of the store to a previous state. + watch Get notifications when values change. + dot Dump the contents of the store as a Graphviz file. + graphql Run a graphql server. + server Run irmin-server. + options Get information about backend specific configuration options. + branches List branches + log List commits + + See `irmin help ` for more information on a specific command. diff --git a/test/irmin-fs/test.ml b/test/irmin-fs/test.ml index 17fb69d99d..7d7a322d71 100644 --- a/test/irmin-fs/test.ml +++ b/test/irmin-fs/test.ml @@ -15,7 +15,6 @@ *) let () = - Eio_main.run @@ fun env -> - Irmin_fs.run env#fs @@ fun () -> + Eio_main.run @@ fun _env -> Irmin_test.Store.run "irmin-fs" ~slow:true ~misc:[] ~sleep:Eio_unix.sleep [ (`Quick, Test_fs.suite) ] diff --git a/test/irmin-fs/test_fs_unix.ml b/test/irmin-fs/test_fs_unix.ml index 2975f784f3..91d9d8a81d 100644 --- a/test/irmin-fs/test_fs_unix.ml +++ b/test/irmin-fs/test_fs_unix.ml @@ -19,7 +19,6 @@ let stats () = (stats.Irmin_watcher.watchdogs, Irmin.Backend.Watch.workers ()) let test_db = Test_fs.test_db -let config = Test_fs.config let store = Irmin_test.store (module Irmin_fs_unix) (module Irmin.Metadata.None) let clean_dirs config = @@ -39,5 +38,6 @@ let clean ~config = clean_dirs config; Irmin.Backend.Watch.(set_listen_dir_hook none) -let suite = +let suite ~path ~clock = + let config = Irmin_fs_unix.conf ~path ~clock in Irmin_test.Suite.create ~name:"FS.UNIX" ~init ~store ~config ~clean ~stats () diff --git a/test/irmin-fs/test_unix.ml b/test/irmin-fs/test_unix.ml index f312b9f3f4..adffd629d8 100644 --- a/test/irmin-fs/test_unix.ml +++ b/test/irmin-fs/test_unix.ml @@ -16,8 +16,9 @@ let () = Eio_main.run @@ fun env -> - Irmin_fs.run env#fs @@ fun () -> Irmin_watcher.run @@ fun () -> + let path = Eio.Stdenv.cwd env in + let clock = Eio.Stdenv.clock env in Irmin_test.Store.run "irmin-fs.unix" ~slow:false ~sleep:Eio_unix.sleep ~misc:[] - [ (`Quick, Test_fs_unix.suite) ] + [ (`Quick, Test_fs_unix.suite ~path ~clock) ] diff --git a/test/irmin/test_conf.ml b/test/irmin/test_conf.ml index afcd08b7c9..e6aa40acd3 100644 --- a/test/irmin/test_conf.ml +++ b/test/irmin/test_conf.ml @@ -28,12 +28,6 @@ let test_conf () = (Invalid_argument "invalid config key: x") (fun () -> ignore (add (empty spec_b) x 1)) in - let specs = - Spec.list () |> Seq.map Spec.name |> List.of_seq |> List.sort String.compare - in - let () = - Alcotest.(check (list string)) "Spec list" [ "a"; "b"; "mem" ] specs - in let keys = Spec.keys spec_a |> Seq.map (fun (K k) -> name k)