[Backport release-23.11] checkpointBuildTools: init

doc/build-helpers/special.md

@@ -7,4 +7,5 @@ special/fhs-environments.section.md
 special/makesetuphook.section.md
 special/mkshell.section.md
 special/vm-tools.section.md
+special/checkpoint-build.section.md
 ```

doc/build-helpers/special/checkpoint-build.section.md

@@ -0,0 +1,39 @@
+# pkgs.checkpointBuildTools  {#sec-checkpoint-build}
+
+`pkgs.checkpointBuildTools` provides a way to build derivations incrementally. It consists of two functions to make checkpoint builds using Nix possible.
+
+For hermeticity, Nix derivations do not allow any state to be carried over between builds, making a transparent incremental build within a derivation impossible.
+
+However, we can tell Nix explicitly what the previous build state was, by representing that previous state as a derivation output. This allows the passed build state to be used for an incremental build.
+
+To change a normal derivation to a checkpoint based build, these steps must be taken:
+  - apply `prepareCheckpointBuild` on the desired derivation, e.g.
+```nix
+checkpointArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox);
+```
+  - change something you want in the sources of the package, e.g. use a source override:
+```nix
+changedVBox = pkgs.virtualbox.overrideAttrs (old: {
+  src = path/to/vbox/sources;
+});
+```
+  - use `mkCheckpointBuild changedVBox checkpointArtifacts`
+  - enjoy shorter build times
+
+## Example {#sec-checkpoint-build-example}
+```nix
+{ pkgs ? import <nixpkgs> {} }:
+let
+  inherit (pkgs.checkpointBuildTools)
+    prepareCheckpointBuild
+    mkCheckpointBuild
+    ;
+  helloCheckpoint = prepareCheckpointBuild pkgs.hello;
+  changedHello = pkgs.hello.overrideAttrs (_: {
+    doCheck = false;
+    patchPhase = ''
+      sed -i 's/Hello, world!/Hello, Nix!/g' src/hello.c
+    '';
+  });
+in mkCheckpointBuild changedHello helloCheckpoint
+```

pkgs/build-support/checkpoint-build.nix

@@ -0,0 +1,90 @@
+{ lib
+, buildPackages
+}:
+
+let
+  # rudimentary support for cross-compiling
+  # see: https://github.com/NixOS/nixpkgs/pull/279487#discussion_r1444449726
+  inherit (buildPackages)
+    mktemp
+    rsync
+    ;
+in
+
+rec {
+  /* Prepare a derivation for local builds.
+    *
+    * This function prepares checkpoint builds by storing
+    * the build output and the sources for cross checking.
+    * The build output can be used later to allow checkpoint builds
+    * by passing the derivation output to the `mkCheckpointBuild` function.
+    *
+    * To build a project with checkpoints, follow these steps:
+    * - run `prepareCheckpointBuild` on the desired derivation, e.g.
+    *     checkpointArtifacts = prepareCheckpointBuild virtualbox;
+    * - change something you want in the sources of the package,
+    *   e.g. using source override:
+    *     changedVBox = pkgs.virtuabox.overrideAttrs (old: {
+    *       src = path/to/vbox/sources;
+    *     };
+    * - use `mkCheckpointBuild changedVBox checkpointArtifacts`
+    * - enjoy shorter build times
+  */
+  prepareCheckpointBuild = drv: drv.overrideAttrs (old: {
+    outputs = [ "out" ];
+    name = drv.name + "-checkpointArtifacts";
+    # To determine differences between the state of the build directory
+    # from an earlier build and a later one we store the state of the build
+    # directory before build, but after patch phases.
+    # This way, the same derivation can be used multiple times and only changes are detected.
+    # Additionally, removed files are handled correctly in later builds.
+    preBuild = (old.preBuild or "") + ''
+      mkdir -p $out/sources
+      cp -r ./* $out/sources/
+    '';
+
+    # After the build, the build directory is copied again
+    # to get the output files.
+    # We copy the complete build folder, to take care of
+    # build tools that build in the source directory, instead of
+    # having a separate build directory such as the Linux kernel.
+    installPhase = ''
+      runHook preCheckpointInstall
+      mkdir -p $out/outputs
+      cp -r ./* $out/outputs/
+      runHook postCheckpointInstall
+    '';
+  });
+
+  /* Build a derivation based on the checkpoint output generated by
+    * the `prepareCheckpointBuild` function.
+    *
+    * Usage:
+    * let
+    *   checkpointArtifacts = prepareCheckpointBuild drv;
+    * in mkCheckpointBuild drv checkpointArtifacts
+  */
+  mkCheckpointBuild = drv: checkpointArtifacts: drv.overrideAttrs (old: {
+    # The actual checkpoint build phase.
+    # We compare the changed sources from a previous build with the current and create a patch.
+    # Afterwards we clean the build directory and copy the previous output files (including the sources).
+    # The source difference patch is then applied to get the latest changes again to allow short build times.
+    preBuild = (old.preBuild or "") + ''
+      set +e
+      sourceDifferencePatchFile=$(${mktemp}/bin/mktemp)
+      diff -ur ${checkpointArtifacts}/sources ./ > "$sourceDifferencePatchFile"
+      set -e
+      shopt -s dotglob
+      rm -r *
+      ${rsync}/bin/rsync \
+        --checksum --times --atimes --chown=$USER:$USER --chmod=+w \
+        -r ${checkpointArtifacts}/outputs/ .
+      patch -p 1 -i "$sourceDifferencePatchFile"
+      rm "$sourceDifferencePatchFile"
+    '';
+  });
+
+  mkCheckpointedBuild = lib.warn
+    "`mkCheckpointedBuild` is deprecated, use `mkCheckpointBuild` instead!"
+    mkCheckpointBuild;
+}

pkgs/test/checkpointBuild/default.nix

@@ -0,0 +1,57 @@
+{ hello, checkpointBuildTools, runCommand, texinfo, stdenv, rsync }:
+let
+  baseHelloArtifacts = checkpointBuildTools.prepareCheckpointBuild hello;
+  patchedHello = hello.overrideAttrs (old: {
+    buildInputs = [ texinfo ];
+    src = runCommand "patch-hello-src" { } ''
+      mkdir -p $out
+      cd $out
+      tar xf ${hello.src} --strip-components=1
+      patch -p1 < ${./hello.patch}
+    '';
+  });
+  checkpointBuiltHello = checkpointBuildTools.mkCheckpointBuild patchedHello baseHelloArtifacts;
+
+  checkpointBuiltHelloWithCheck = checkpointBuiltHello.overrideAttrs (old: {
+    doCheck = true;
+    checkPhase = ''
+      echo "checking if unchanged source file is not recompiled"
+        [ "$(stat --format="%Y" lib/exitfail.o)" = "$(stat --format="%Y" ${baseHelloArtifacts}/outputs/lib/exitfail.o)" ]
+    '';
+  });
+
+  baseHelloRemoveFileArtifacts = checkpointBuildTools.prepareCheckpointBuild (hello.overrideAttrs (old: {
+    patches = [ ./hello-additionalFile.patch ];
+  }));
+
+  preparedHelloRemoveFileSrc = runCommand "patch-hello-src" { } ''
+    mkdir -p $out
+    cd $out
+    tar xf ${hello.src} --strip-components=1
+    patch -p1 < ${./hello-additionalFile.patch}
+  '';
+
+  patchedHelloRemoveFile = hello.overrideAttrs (old: {
+    buildInputs = [ texinfo ];
+    src = runCommand "patch-hello-src" { } ''
+      mkdir -p $out
+      cd $out
+      ${rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${preparedHelloRemoveFileSrc}/* .
+      patch -p1 < ${./hello-removeFile.patch}
+    '';
+  });
+
+  checkpointBuiltHelloWithRemovedFile = checkpointBuildTools.mkCheckpointBuild patchedHelloRemoveFile baseHelloRemoveFileArtifacts;
+in
+stdenv.mkDerivation {
+  name = "patched-hello-returns-correct-output";
+  buildCommand = ''
+    touch $out
+
+    echo "testing output of hello binary"
+    [ "$(${checkpointBuiltHelloWithCheck}/bin/hello)" = "Hello, incremental world!" ]
+    echo "testing output of hello with removed file"
+    [ "$(${checkpointBuiltHelloWithRemovedFile}/bin/hello)" = "Hello, incremental world!" ]
+  '';
+}
+

pkgs/test/checkpointBuild/hello-additionalFile.patch

@@ -0,0 +1,67 @@
+:100644 100644 0000000 0000000 M	Makefile.in
+:000000 100644 0000000 0000000 A	src/additionalFile.c
+:100644 100644 0000000 0000000 M	src/hello.c
+:100644 100644 0000000 0000000 M	src/system.h
+
+diff --git a/Makefile.in b/Makefile.in
+index 1597d39..f63f830 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \
+ 	lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \
+ 	lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \
+ 	lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \
+-	lib/xstrndup.$(OBJEXT)
++	lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT)
+ lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS)
+ am_hello_OBJECTS = src/hello.$(OBJEXT)
+ hello_OBJECTS = $(am_hello_OBJECTS)
+@@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \
+ 	$(am__append_4) $(am__append_5) lib/version-etc.h \
+ 	lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \
+ 	lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \
+-	lib/xstrndup.h lib/xstrndup.c
++	lib/xstrndup.h lib/xstrndup.c src/additionalFile.c
+ lib_libhello_a_LIBADD = $(gl_LIBOBJS)
+ lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS)
+ EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \
+diff --git a/src/additionalFile.c b/src/additionalFile.c
+new file mode 100644
+index 0000000..34d683d
+--- /dev/null
++++ b/src/additionalFile.c
+@@ -0,0 +1,6 @@
++#include "config.h"
++#include "system.h"
++
++int somefunc() {
++    return 0;
++}
+diff --git a/src/hello.c b/src/hello.c
+index 2e7d38e..a8e36dc 100644
+--- a/src/hello.c
++++ b/src/hello.c
+@@ -146,7 +146,11 @@ main (int argc, char *argv[])
+ #endif
+
+   /* Having initialized gettext, get the default message. */
+-  greeting_msg = _("Hello, world!");
++  if (somefunc() == 0) {
++    greeting_msg = _("Hello, world!");
++  } else {
++    greeting_msg = _("Hello, incremental world!");
++  }
+
+   /* Even exiting has subtleties.  On exit, if any writes failed, change
+      the exit status.  The /dev/full device on GNU/Linux can be used for
+diff --git a/src/system.h b/src/system.h
+index d39cdb9..dc425d2 100644
+--- a/src/system.h
++++ b/src/system.h
+@@ -59,4 +59,6 @@
+     } \
+   while (0)
+
++int somefunc();
++
+ #endif /* HELLO_SYSTEM_H */

pkgs/test/checkpointBuild/hello-removeFile.patch

@@ -0,0 +1,67 @@
+:100644 100644 0000000 0000000 M	Makefile.in
+:100644 000000 0000000 0000000 D	src/additionalFile.c
+:100644 100644 0000000 0000000 M	src/hello.c
+:100755 100755 0000000 0000000 M	tests/hello-1
+
+diff --git a/Makefile.in b/Makefile.in
+index f63f830..1597d39 100644
+--- a/Makefile.in
++++ b/Makefile.in
+@@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \
+ 	lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \
+ 	lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \
+ 	lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \
+-	lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT)
++	lib/xstrndup.$(OBJEXT)
+ lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS)
+ am_hello_OBJECTS = src/hello.$(OBJEXT)
+ hello_OBJECTS = $(am_hello_OBJECTS)
+@@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \
+ 	$(am__append_4) $(am__append_5) lib/version-etc.h \
+ 	lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \
+ 	lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \
+-	lib/xstrndup.h lib/xstrndup.c src/additionalFile.c
++	lib/xstrndup.h lib/xstrndup.c
+ lib_libhello_a_LIBADD = $(gl_LIBOBJS)
+ lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS)
+ EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \
+diff --git a/src/additionalFile.c b/src/additionalFile.c
+deleted file mode 100644
+index 34d683d..0000000
+--- a/src/additionalFile.c
++++ /dev/null
+@@ -1,6 +0,0 @@
+-#include "config.h"
+-#include "system.h"
+-
+-int somefunc() {
+-    return 0;
+-}
+diff --git a/src/hello.c b/src/hello.c
+index a8e36dc..53722d9 100644
+--- a/src/hello.c
++++ b/src/hello.c
+@@ -126,6 +126,10 @@ parse_options (int argc, char *argv[], const char **greeting_msg)
+     }
+ }
+
++int somefunc() {
++    return 1;
++}
++
+ int
+ main (int argc, char *argv[])
+ {
+diff --git a/tests/hello-1 b/tests/hello-1
+index 96ffef8..f0b9f8d 100755
+--- a/tests/hello-1
++++ b/tests/hello-1
+@@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG
+
+ tmpfiles="hello-test1.ok"
+ cat <<EOF > hello-test1.ok
+-Hello, world!
++Hello, incremental world!
+ EOF
+
+ tmpfiles="$tmpfiles hello-test1.out"

pkgs/test/checkpointBuild/hello.patch

@@ -0,0 +1,26 @@
+diff --git a/src/hello.c b/src/hello.c
+index 182303c..453962f 100644
+--- a/src/hello.c
++++ b/src/hello.c
+@@ -57,7 +57,7 @@ main (int argc, char *argv[])
+ #endif
+
+   /* Having initialized gettext, get the default message. */
+-  greeting_msg = _("Hello, world!");
++  greeting_msg = _("Hello, incremental world!");
+
+   /* Even exiting has subtleties.  On exit, if any writes failed, change
+      the exit status.  The /dev/full device on GNU/Linux can be used for
+diff --git a/tests/hello-1 b/tests/hello-1
+index 3b7a815..e15fa95 100755
+--- a/tests/hello-1
++++ b/tests/hello-1
+@@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG
+
+ tmpfiles="hello-test1.ok"
+ cat <<EOF > hello-test1.ok
+-Hello, world!
++Hello, incremental world!
+ EOF
+
+ tmpfiles="$tmpfiles hello-test1.out"

pkgs/test/default.nix

@@ -117,6 +117,8 @@ with pkgs;
 
   install-shell-files = callPackage ./install-shell-files {};
 
+  checkpointBuildTools = callPackage ./checkpointBuild {};
+
   kernel-config = callPackage ./kernel.nix {};
 
   ld-library-path = callPackage ./ld-library-path {};

pkgs/top-level/all-packages.nix

@@ -424,6 +424,8 @@ with pkgs;
 
   camunda-modeler = callPackage ../applications/misc/camunda-modeler { };
 
+  checkpointBuildTools = callPackage ../build-support/checkpoint-build.nix {};
+
   caroline = callPackage ../development/libraries/caroline { };
 
   cartridges = callPackage ../applications/misc/cartridges { };