Merge pull request #2114 from metanivek/v4_layout

src/irmin-pack/layout.ml

@@ -50,6 +50,30 @@ module V3 = struct
     [ suffix ~generation ~root; branch ~root; dict ~root; control ~root ]
 end
 
+module V4 = struct
+  let branch = toplevel "store.branches"
+  let dict = toplevel "store.dict"
+  let control = toplevel "store.control"
+
+  let suffix_chunk ~chunk_idx =
+    toplevel ("store." ^ string_of_int chunk_idx ^ ".suffix")
+
+  let gc_result ~generation =
+    toplevel ("store." ^ string_of_int generation ^ ".out")
+
+  let reachable ~generation =
+    toplevel ("store." ^ string_of_int generation ^ ".reachable")
+
+  let sorted ~generation =
+    toplevel ("store." ^ string_of_int generation ^ ".sorted")
+
+  let mapping ~generation =
+    toplevel ("store." ^ string_of_int generation ^ ".mapping")
+
+  let prefix ~generation =
+    toplevel ("store." ^ string_of_int generation ^ ".prefix")
+end
+
 (** [is_number] is a less generic than [Stdlib.int_of_string_opt]. It matches
     this equivalent regex: {v "([1-9][0-9]*|0)" v}. *)
 let is_number s =

src/irmin-pack/unix/ext.ml

@@ -195,7 +195,7 @@ module Maker (Config : Conf.S) = struct
             let root = Conf.root config in
             let fresh = Conf.fresh config in
             let readonly = Conf.readonly config in
-            let path = Irmin_pack.Layout.V3.branch ~root in
+            let path = Irmin_pack.Layout.V4.branch ~root in
             Branch.v ~fresh ~readonly path
           in
           let during_batch = false in

src/irmin-pack/unix/file_manager.ml

@@ -208,7 +208,7 @@ struct
   let reopen_prefix t ~generation =
     let open Result_syntax in
     let* prefix1 =
-      let path = Irmin_pack.Layout.V3.prefix ~root:t.root ~generation in
+      let path = Irmin_pack.Layout.V4.prefix ~root:t.root ~generation in
       [%log.debug "reload: opening %s" path];
       Prefix.open_ ~readonly:true ~path
     in
@@ -240,7 +240,9 @@ struct
       "reopen_suffix gen:%d end_poff:%d" generation (Int63.to_int end_poff)];
     let readonly = Suffix.readonly t.suffix in
     let* suffix1 =
-      let path = Irmin_pack.Layout.V3.suffix ~root:t.root ~generation in
+      let path =
+        Irmin_pack.Layout.V4.suffix_chunk ~root:t.root ~chunk_idx:generation
+      in
       [%log.debug "reload: generation changed, opening %s" path];
       if readonly then Suffix.open_ro ~path ~end_poff ~dead_header_size
       else
@@ -323,7 +325,9 @@ struct
     in
     (* 2. Open the other files *)
     let* suffix =
-      let path = Irmin_pack.Layout.V3.suffix ~root ~generation in
+      let path =
+        Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:generation
+      in
       let auto_flush_threshold =
         Irmin_pack.Conf.suffix_auto_flush_threshold config
       in
@@ -332,12 +336,12 @@ struct
         ~auto_flush_procedure:(`External cb)
     in
     let* prefix =
-      let path = Irmin_pack.Layout.V3.prefix ~root ~generation in
+      let path = Irmin_pack.Layout.V4.prefix ~root ~generation in
       only_open_after_gc ~generation ~path
     in
     let* mapping = open_mapping ~root ~generation in
     let* dict =
-      let path = Irmin_pack.Layout.V3.dict ~root in
+      let path = Irmin_pack.Layout.V4.dict ~root in
       let auto_flush_threshold =
         Irmin_pack.Conf.dict_auto_flush_threshold config
       in
@@ -380,7 +384,7 @@ struct
 
   let create_control_file ~overwrite config pl =
     let root = Irmin_pack.Conf.root config in
-    let path = Irmin_pack.Layout.V3.control ~root in
+    let path = Irmin_pack.Layout.V4.control ~root in
     Control.create_rw ~path ~overwrite pl
 
   (* Reload ***************************************************************** *)
@@ -466,7 +470,7 @@ struct
     let open Result_syntax in
     let root = Irmin_pack.Conf.root config in
     let* control =
-      let path = Irmin_pack.Layout.V3.control ~root in
+      let path = Irmin_pack.Layout.V4.control ~root in
       Control.open_ ~readonly:false ~path
     in
     let pl : Payload.t = Control.payload control in
@@ -519,10 +523,10 @@ struct
     let open Result_syntax in
     let root = Irmin_pack.Conf.root config in
     let src = Irmin_pack.Layout.V1_and_v2.pack ~root in
-    let dst = Irmin_pack.Layout.V3.suffix ~root ~generation:0 in
+    let dst = Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:0 in
     let* suffix_end_poff = read_offset_from_legacy_file src in
     let* dict_end_poff =
-      let path = Irmin_pack.Layout.V3.dict ~root in
+      let path = Irmin_pack.Layout.V4.dict ~root in
       read_offset_from_legacy_file path
     in
     let* () = Io.move_file ~src ~dst in
@@ -559,7 +563,7 @@ struct
     | `File | `Other -> Error (`Not_a_directory root)
     | `No_such_file_or_directory -> Error `No_such_file_or_directory
     | `Directory -> (
-        let path = Irmin_pack.Layout.V3.control ~root in
+        let path = Irmin_pack.Layout.V4.control ~root in
         match Io.classify_path path with
         | `File -> open_rw_with_control_file config
         | `No_such_file_or_directory ->
@@ -576,7 +580,7 @@ struct
     let use_fsync = Irmin_pack.Conf.use_fsync config in
     (* 1. Open the control file *)
     let* control =
-      let path = Irmin_pack.Layout.V3.control ~root in
+      let path = Irmin_pack.Layout.V4.control ~root in
       Control.open_ ~readonly:true ~path
       (* If no control file, then check whether the store is in v1 or v2. *)
       |> Result.map_error (function
@@ -600,16 +604,18 @@ struct
     let generation = generation pl.status in
     (* 2. Open the other files *)
     let* suffix =
-      let path = Irmin_pack.Layout.V3.suffix ~root ~generation in
+      let path =
+        Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:generation
+      in
       Suffix.open_ro ~path ~end_poff:pl.suffix_end_poff ~dead_header_size
     in
     let* prefix =
-      let path = Irmin_pack.Layout.V3.prefix ~root ~generation in
+      let path = Irmin_pack.Layout.V4.prefix ~root ~generation in
       only_open_after_gc ~path ~generation
     in
     let* mapping = open_mapping ~root ~generation in
     let* dict =
-      let path = Irmin_pack.Layout.V3.dict ~root in
+      let path = Irmin_pack.Layout.V4.dict ~root in
       Dict.open_ro ~path ~end_poff:pl.dict_end_poff ~dead_header_size
     in
     let* index =
@@ -652,7 +658,7 @@ struct
     | `No_such_file_or_directory -> Error `No_such_file_or_directory
     | `File | `Other -> Error (`Not_a_directory root)
     | `Directory -> (
-        let path = Irmin_pack.Layout.V3.control ~root in
+        let path = Irmin_pack.Layout.V4.control ~root in
         match Control.open_ ~path ~readonly:true with
         | Ok _ -> Ok `V3
         | Error `No_such_file_or_directory -> v2_or_v1 ()

src/irmin-pack/unix/gc.ml

@@ -70,7 +70,7 @@ module Make (Args : Gc_args.S) = struct
         ~after_suffix_start_offset
     in
     let unlink_result_file () =
-      let result_file = Irmin_pack.Layout.V3.gc_result ~root ~generation in
+      let result_file = Irmin_pack.Layout.V4.gc_result ~root ~generation in
       match Io.unlink result_file with
       | Ok () -> ()
       | Error (`Sys_error msg as err) ->
@@ -115,7 +115,7 @@ module Make (Args : Gc_args.S) = struct
 
   let open_new_suffix ~end_poff { root; generation; _ } =
     let open Result_syntax in
-    let path = Irmin_pack.Layout.V3.suffix ~root ~generation in
+    let path = Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:generation in
     (* As the new suffix is necessarily in V3, the dead_header_size is
        0. *)
     let dead_header_size = 0 in
@@ -172,26 +172,26 @@ module Make (Args : Gc_args.S) = struct
       let open Result_syntax in
       (* Unlink previous suffix. *)
       let suffix =
-        Irmin_pack.Layout.V3.suffix ~root ~generation:(generation - 1)
+        Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:(generation - 1)
       in
       let* () = Io.unlink suffix in
       let* () =
         if generation >= 2 then
           (* Unlink previous prefix. *)
           let prefix =
-            Irmin_pack.Layout.V3.prefix ~root ~generation:(generation - 1)
+            Irmin_pack.Layout.V4.prefix ~root ~generation:(generation - 1)
           in
           let* () = Io.unlink prefix in
           (* Unlink previous mapping. *)
           let mapping =
-            Irmin_pack.Layout.V3.mapping ~root ~generation:(generation - 1)
+            Irmin_pack.Layout.V4.mapping ~root ~generation:(generation - 1)
           in
           let* () = Io.unlink mapping in
           Ok ()
         else Ok ()
       in
       (* Unlink current gc's result.*)
-      let result = Irmin_pack.Layout.V3.gc_result ~root ~generation in
+      let result = Irmin_pack.Layout.V4.gc_result ~root ~generation in
       Io.unlink result
     in
     match result with
@@ -218,7 +218,7 @@ module Make (Args : Gc_args.S) = struct
   let read_gc_output ~root ~generation =
     let open Result_syntax in
     let read_file () =
-      let path = Irmin_pack.Layout.V3.gc_result ~root ~generation in
+      let path = Irmin_pack.Layout.V4.gc_result ~root ~generation in
       let* io = Io.open_ ~path ~readonly:true in
       let* len = Io.read_size io in
       let len = Int63.to_int len in

src/irmin-pack/unix/gc_worker.ml

@@ -106,7 +106,7 @@ module Make (Args : Gc_args.S) = struct
     write_exn ~off:accessor.poff ~len (Bytes.unsafe_to_string buffer)
 
   let create_new_suffix ~root ~generation =
-    let path = Irmin_pack.Layout.V3.suffix ~root ~generation in
+    let path = Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:generation in
     Ao.create_rw_exn ~path
 
   let run ~generation root commit_key new_prefix_end_offset =
@@ -201,7 +201,7 @@ module Make (Args : Gc_args.S) = struct
       (* Step 4. Create the new prefix. *)
       stats := Gc_stats.Worker.finish_current_step !stats "prefix: start";
       let prefix =
-        let path = Irmin_pack.Layout.V3.prefix ~root ~generation in
+        let path = Irmin_pack.Layout.V4.prefix ~root ~generation in
         Ao.create_rw_exn ~path
       in
       let () =
@@ -235,7 +235,7 @@ module Make (Args : Gc_args.S) = struct
         Dispatcher.read_exn dispatcher accessor buf
       in
       let prefix =
-        let path = Irmin_pack.Layout.V3.prefix ~root ~generation in
+        let path = Irmin_pack.Layout.V4.prefix ~root ~generation in
         Io.open_ ~path ~readonly:false |> Errs.raise_if_error
       in
       Errors.finalise_exn (fun _outcome ->
@@ -304,7 +304,7 @@ module Make (Args : Gc_args.S) = struct
 
   let write_gc_output ~root ~generation output =
     let open Result_syntax in
-    let path = Irmin_pack.Layout.V3.gc_result ~root ~generation in
+    let path = Irmin_pack.Layout.V4.gc_result ~root ~generation in
     let* io = Io.create ~path ~overwrite:true in
     let out = Irmin.Type.to_json_string gc_output_t output in
     let* () = Io.write_string io ~off:Int63.zero out in

src/irmin-pack/unix/mapping_file.ml

@@ -336,7 +336,7 @@ module Make (Io : Io.S) = struct
   type t = { arr : int64_bigarray; root : string; generation : int }
 
   let open_map ~root ~generation =
-    let path = Irmin_pack.Layout.V3.mapping ~generation ~root in
+    let path = Irmin_pack.Layout.V4.mapping ~generation ~root in
     match Io.classify_path path with
     | `File -> (
         let mmap = Int64_mmap.open_ ~fn:path ~sz:(-1) in
@@ -356,9 +356,9 @@ module Make (Io : Io.S) = struct
   let create ?report_file_sizes ~root ~generation ~register_entries () =
     assert (generation > 0);
     let open Result_syntax in
-    let path0 = Irmin_pack.Layout.V3.reachable ~generation ~root in
-    let path1 = Irmin_pack.Layout.V3.sorted ~generation ~root in
-    let path2 = Irmin_pack.Layout.V3.mapping ~generation ~root in
+    let path0 = Irmin_pack.Layout.V4.reachable ~generation ~root in
+    let path1 = Irmin_pack.Layout.V4.sorted ~generation ~root in
+    let path2 = Irmin_pack.Layout.V4.mapping ~generation ~root in
 
     let* () =
       if Sys.word_size <> 64 then Error `Gc_forbidden_on_32bit_platforms

test/irmin-pack/test_pack.ml

@@ -498,23 +498,23 @@ end
 module Layout = struct
   let test_classify_filename () =
     let module V1_and_v2 = Irmin_pack.Layout.V1_and_v2 in
-    let module V3 = Irmin_pack.Layout.V3 in
+    let module V4 = Irmin_pack.Layout.V4 in
     let c =
       Alcotest.(
         check (option (testable_repr Irmin_pack.Layout.classification_t)))
         ""
     in
     let classif = Irmin_pack.Layout.classify_filename in
     c (Some `V1_or_v2_pack) (V1_and_v2.pack ~root:"" |> classif);
-    c (Some `Branch) (V3.branch ~root:"" |> classif);
-    c (Some `Dict) (V3.dict ~root:"" |> classif);
-    c (Some (`Gc_result 0)) (V3.gc_result ~generation:0 ~root:"" |> classif);
-    c (Some (`Reachable 1)) (V3.reachable ~generation:1 ~root:"" |> classif);
-    c (Some (`Sorted 10)) (V3.sorted ~generation:10 ~root:"" |> classif);
-    c (Some (`Mapping 100)) (V3.mapping ~generation:100 ~root:"" |> classif);
-    c (Some (`Prefix 1000)) (V3.prefix ~generation:1000 ~root:"" |> classif);
-    c (Some (`Suffix 42)) (V3.suffix ~generation:42 ~root:"" |> classif);
-    c None (V3.prefix ~generation:(-1) ~root:"" |> classif);
+    c (Some `Branch) (V4.branch ~root:"" |> classif);
+    c (Some `Dict) (V4.dict ~root:"" |> classif);
+    c (Some (`Gc_result 0)) (V4.gc_result ~generation:0 ~root:"" |> classif);
+    c (Some (`Reachable 1)) (V4.reachable ~generation:1 ~root:"" |> classif);
+    c (Some (`Sorted 10)) (V4.sorted ~generation:10 ~root:"" |> classif);
+    c (Some (`Mapping 100)) (V4.mapping ~generation:100 ~root:"" |> classif);
+    c (Some (`Prefix 1000)) (V4.prefix ~generation:1000 ~root:"" |> classif);
+    c (Some (`Suffix 42)) (V4.suffix_chunk ~chunk_idx:42 ~root:"" |> classif);
+    c None (V4.prefix ~generation:(-1) ~root:"" |> classif);
     c None (classif "store.toto");
     c None (classif "store.");
     c None (classif "store");