feature: switch dune to use blake3

<!-- ps-id: 4f9dee83-85dd-4b47-ba4f-920087ea5851 -->

Signed-off-by: Rudi Grinberg <[email protected]>

Author: rgrinberg

.github/workflows/workflow.yml

@@ -81,6 +81,9 @@ jobs:
       # Install ocamlfind-secondary and ocaml-secondary-compiler, if needed
       - run: opam install ./dune.opam --deps-only --with-test
 
+      - name: Print config
+        run: opam exec -- ocamlopt -config
+
       - name: Install system deps on macOS
         run: brew install coreutils pkg-config file
         if: ${{ matrix.os == 'macos-latest' }}

doc/changes/11735.md

@@ -0,0 +1,3 @@
+- Switch from MD5 to BLAKE3 for digesting targets and rules. BLAKE3 is both
+  more performant and difficult to break than MD5 (#11735, @rgrinberg,
+  @Alizter)

src/dune_digest/cached_digest.ml

@@ -57,7 +57,7 @@ module P = Persistent.Make (struct
     type nonrec t = t
 
     let name = "DIGEST-DB"
-    let version = 6
+    let version = 7
     let to_dyn = to_dyn
 
     let test_example () =

src/dune_digest/digest.ml

@@ -1,12 +1,17 @@
 open Stdune
 
-type t = string
+module T = struct
+  type t = Blake3_mini.Digest.t
 
-external md5_fd : Unix.file_descr -> string = "dune_md5_fd"
+  let to_string = Blake3_mini.Digest.to_hex
+  let to_dyn s = Dyn.variant "digest" [ String (to_string s) ]
+  let compare x y = Ordering.of_int (Blake3_mini.Digest.compare x y)
+end
 
-module D = Stdlib.Digest
-module Set = String.Set
-module Map = String.Map
+include T
+module C = Comparable.Make (T)
+module Set = C.Set
+module Map = C.Map
 module Metrics = Dune_metrics
 
 let file file =
@@ -21,24 +26,24 @@ let file file =
       raise (Sys_error (sprintf "%s: Permission denied" file))
     | exception exn -> reraise exn
   in
-  Exn.protectx fd ~f:md5_fd ~finally:Unix.close
+  Exn.protectx fd ~f:Blake3_mini.fd ~finally:Unix.close
 ;;
 
+let equal = Blake3_mini.Digest.equal
 let hash = Poly.hash
-let equal = String.equal
 let file p = file (Path.to_string p)
-let compare x y = Ordering.of_int (D.compare x y)
-let to_string = D.to_hex
-let to_dyn s = Dyn.variant "digest" [ String (to_string s) ]
-
-let from_hex s =
-  match D.from_hex s with
-  | s -> Some s
-  | exception Invalid_argument _ -> None
+let from_hex s = Blake3_mini.Digest.of_hex s
+let hasher = lazy (Blake3_mini.create ())
+
+let string s =
+  let hasher = Lazy.force hasher in
+  Blake3_mini.feed_string hasher s ~pos:0 ~len:(String.length s);
+  let res = Blake3_mini.digest hasher in
+  Blake3_mini.reset hasher;
+  res
 ;;
 
-let string = D.string
-let to_string_raw s = s
+let to_string_raw s = Blake3_mini.Digest.to_binary s
 
 (* We use [No_sharing] to avoid generating different digests for inputs that
    differ only in how they share internal values. Without [No_sharing], if a
@@ -54,7 +59,7 @@ let generic a =
 let path_with_executable_bit =
   (* We follow the digest scheme used by Jenga. *)
   let string_and_bool ~digest_hex ~bool =
-    string (digest_hex ^ if bool then "\001" else "\000")
+    string (Blake3_mini.Digest.to_hex digest_hex ^ if bool then "\001" else "\000")
   in
   fun ~executable ~content_digest ->
     string_and_bool ~digest_hex:content_digest ~bool:executable
@@ -86,15 +91,15 @@ end
 
 exception E of Path_digest_error.t
 
-let directory_digest_version = 2
+let directory_digest_version = 3
 
 let path_with_stats ~allow_dirs path (stats : Stats_for_digest.t) =
   let rec loop path (stats : Stats_for_digest.t) =
     match stats.st_kind with
     | S_LNK ->
       Dune_filesystem_stubs.Unix_error.Detailed.catch
         (fun path ->
-           let contents = Unix.readlink (Path.to_string path) in
+           let contents = Path.to_string path |> Unix.readlink |> string in
            path_with_executable_bit ~executable:stats.executable ~content_digest:contents)
         path
       |> Result.map_error ~f:(fun x -> Path_digest_error.Unix_error x)

src/dune_digest/digest.mli

@@ -1,6 +1,6 @@
 open Stdune
 
-(** Digests (MD5) *)
+(** Digests (BLAKE3) *)
 
 type t
 

src/dune_digest/dune

@@ -7,7 +7,4 @@
   dune_console
   dune_util
   stdune
-  unix)
- (foreign_stubs
-  (names dune_digest_stubs)
-  (language c)))
+  unix))

src/dune_digest/dune_digest_stubs.c

@@ -267,7 +267,7 @@ end = struct
 
   (* The current version of the rule digest scheme. We should increment it when
      making any changes to the scheme, to avoid collisions. *)
-  let rule_digest_version = 22
+  let rule_digest_version = 23
 
   let compute_rule_digest
         (rule : Rule.t)
@@ -744,7 +744,7 @@ end = struct
 
   (* The current version of the action digest scheme. We should increment it when
      making any changes to the scheme, to avoid collisions. *)
-  let action_digest_version = 2
+  let action_digest_version = 3
 
   let execute_action_generic
         ~observing_facts

src/dune_engine/build_system.ml

@@ -31,7 +31,7 @@ module Workspace_local = struct
         type nonrec t = t
 
         let name = "INCREMENTAL-DB"
-        let version = 5
+        let version = 6
         let to_dyn = to_dyn
 
         let test_example () =

src/dune_engine/rule_cache.ml

@@ -12,7 +12,7 @@ module To_delete = struct
       type t = Path.Source.Set.t
 
       let name = "PROMOTED-TO-DELETE"
-      let version = 2
+      let version = 3
       let to_dyn = Path.Source.Set.to_dyn
       let test_example () = Path.Source.Set.empty
     end)