Reuse dependencies between project and tools

This allows the dependencies of the project to be shared with dev tools,
preventing unnecessarily rebuilding dependencies while building a tool
if an identical version of the package has already been built or vice
versa.

This requires storing build artifacts of dependencies of dev tools and
the project (when built in the "default" context) in a single directory,
since their relative path within the _build directory is part of the key
used to cache build artifacts. Previously the build artifacts of a
package were placed in a directory named after that package, however
since dev tools and the project may depend on different versions of the
same package, directories needed to be renamed to avoid collisions. Even
if the project and dev tools rely on the same version of some package,
it may still need to be rebuilt on occasion due to the presence/absence
of depopts, or due to manual modification to lockfiles. For this reason,
the directories containing package build artifacts contain in their
name, a hash of the contents of the lockfile of the metadata in the
package's transitive dependency closure.

This change introduces the concept of a "package digest" which is a
unique directory name for a package comprising its name, version, and
dependency closure hash. The DB type used to store metadata while
building packages has been updated to be keyed based on package digests,
and to store package metadata for dev tools and the project's default
build context in the same table to allow dependencies to be shared.

Signed-off-by: Stephen Sherratt <[email protected]>

Author: gridbugs

bin/pkg/group.ml

@@ -13,6 +13,7 @@ let subcommands =
   ; Outdated.command
   ; Validate_lock_dir.command
   ; Pkg_enabled.command
+  ; Print_digest.command
   ]
 ;;
 

bin/pkg/print_digest.ml

@@ -0,0 +1,36 @@
+open Import
+
+let term =
+  let+ builder = Common.Builder.term
+  and+ package_name =
+    Arg.(
+      required
+      & pos 0 (some string) None
+      & info [] ~doc:"The name of the package" ~docv:"PACKAGE")
+  and+ context_name = Common.context_arg ~doc:"Context used to determine lockdir" in
+  let builder = Common.Builder.forbid_builds builder in
+  let common, config = Common.init builder in
+  let package_name = Package_name.of_string package_name in
+  Scheduler.go_with_rpc_server ~common ~config (fun () ->
+    let open Fiber.O in
+    let+ pkg_digest_opt =
+      build_exn (fun () ->
+        Dune_rules.Pkg_rules.pkg_digest_of_project_dependency context_name package_name)
+    in
+    match pkg_digest_opt with
+    | Some pkg_digest ->
+      print_endline (Dune_rules.Pkg_rules.Pkg_digest.to_string pkg_digest)
+    | None ->
+      User_error.raise
+        [ Pp.textf
+            "The project does not depend on the package %S."
+            (Package_name.to_string package_name)
+        ])
+;;
+
+let info =
+  let doc = "Print the digest of a package in the project's lockdir." in
+  Cmd.info "print-digest" ~doc
+;;
+
+let command = Cmd.v info term

bin/pkg/print_digest.mli

@@ -0,0 +1,4 @@
+open Import
+
+(** Command to print the slug of a given package within the current project. *)
+val command : unit Cmd.t

src/dune_digest/digest.ml

@@ -91,14 +91,15 @@ module Feed = struct
     feed_c hasher c
   ;;
 
-  let digest t x = Hasher.with_singleton (fun hasher -> t hasher x)
+  let digest hasher digest = contramap string ~f:to_string hasher digest
+  let compute_digest t x = Hasher.with_singleton (fun hasher -> t hasher x)
 end
 
-let string s = Feed.digest Feed.string s
+let string s = Feed.compute_digest Feed.string s
 let to_string_raw s = Blake3_mini.Digest.to_binary s
 
 let generic a =
-  Metrics.Timer.record "generic_digest" ~f:(fun () -> Feed.digest Feed.generic a)
+  Metrics.Timer.record "generic_digest" ~f:(fun () -> Feed.compute_digest Feed.generic a)
 ;;
 
 let path_with_executable_bit =

src/dune_digest/digest.mli

@@ -27,8 +27,11 @@ module Feed : sig
   val tuple2 : 'a t -> 'b t -> ('a * 'b) t
   val tuple3 : 'a t -> 'b t -> 'c t -> ('a * 'b * 'c) t
 
+  (** Feed a digest into a hasher. *)
+  val digest : digest t
+
   (** Compute the digest of a value given a feed for the type of that value. *)
-  val digest : 'a t -> 'a -> digest
+  val compute_digest : 'a t -> 'a -> digest
 end
 
 include Comparable_intf.S with type key := t

src/dune_lang/package_name.mli

@@ -31,3 +31,5 @@ val file : t -> dir:Path.Source.t -> Path.Source.t
 val decode_opam_compatible : t Decoder.t
 val opam_fn : t -> Filename.t
 val of_opam_file_basename : Filename.t -> t option
+
+module Table : Hashtbl.S with type key = t

src/dune_lang/package_version.mli

@@ -7,6 +7,7 @@ val of_string_opt : string -> t option
 val of_string_user_error : Loc.t * string -> (t, User_message.t) result
 val to_string : t -> string
 val equal : t -> t -> bool
+val compare : t -> t -> ordering
 val hash : t -> int
 val digest_feed : t Dune_digest.Feed.t
 val to_dyn : t -> Dyn.t

src/dune_pkg/dune_pkg.ml

@@ -26,3 +26,4 @@ module Package_name = Package_name
 module Ocamlformat = Ocamlformat
 module Dev_tool = Dev_tool
 module Outdated = Outdated
+module Dune_dep = Dune_dep

src/dune_pkg/package_name.mli

@@ -6,3 +6,5 @@ include module type of Dune_lang.Package_name with type t := t
 
 val of_opam_package_name : OpamTypes.name -> t
 val to_opam_package_name : t -> OpamTypes.name
+
+module Table : Hashtbl.S with type key = t

src/dune_pkg/package_version.mli

@@ -6,6 +6,7 @@ val of_string : string -> t
 val of_string_user_error : Loc.t * string -> (t, User_message.t) result
 val to_string : t -> string
 val equal : t -> t -> bool
+val compare : t -> t -> ordering
 val hash : t -> int
 val digest_feed : t Dune_digest.Feed.t
 val to_dyn : t -> Dyn.t