From 5554b672c1c143ff02b08516583153834e3a881e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Tue, 19 Dec 2023 11:37:15 +0100 Subject: [PATCH 1/6] Merge pull request #167670 from messemar/incremental-builds incremental builds: add derivation override functions (cherry picked from commit 5eed5416ffa70d1a6c1ff00548c746014df7e2c6) --- doc/build-helpers/special.md | 1 + .../special/checkpoint-build.section.md | 36 ++++++++++ pkgs/build-support/checkpoint-build.nix | 69 +++++++++++++++++++ pkgs/test/checkpointBuild/default.nix | 57 +++++++++++++++ .../hello-additionalFile.patch | 67 ++++++++++++++++++ .../checkpointBuild/hello-removeFile.patch | 67 ++++++++++++++++++ pkgs/test/checkpointBuild/hello.patch | 26 +++++++ pkgs/test/default.nix | 2 + pkgs/top-level/all-packages.nix | 2 + 9 files changed, 327 insertions(+) create mode 100644 doc/build-helpers/special/checkpoint-build.section.md create mode 100644 pkgs/build-support/checkpoint-build.nix create mode 100644 pkgs/test/checkpointBuild/default.nix create mode 100644 pkgs/test/checkpointBuild/hello-additionalFile.patch create mode 100644 pkgs/test/checkpointBuild/hello-removeFile.patch create mode 100644 pkgs/test/checkpointBuild/hello.patch diff --git a/doc/build-helpers/special.md b/doc/build-helpers/special.md index f88648207fdcd..265c2da92bf18 100644 --- a/doc/build-helpers/special.md +++ b/doc/build-helpers/special.md @@ -7,4 +7,5 @@ special/fhs-environments.section.md special/makesetuphook.section.md special/mkshell.section.md special/vm-tools.section.md +special/checkpoint-build.section.md ``` diff --git a/doc/build-helpers/special/checkpoint-build.section.md b/doc/build-helpers/special/checkpoint-build.section.md new file mode 100644 index 0000000000000..5f01e699b9477 --- /dev/null +++ b/doc/build-helpers/special/checkpoint-build.section.md @@ -0,0 +1,36 @@ +# pkgs.checkpointBuildTools {#sec-checkpoint-build} + +`pkgs.checkpointBuildTools` provides a way to build derivations incrementally. It consists of two functions to make checkpoint builds using Nix possible. + +For hermeticity, Nix derivations do not allow any state to carry over between builds, making a transparent incremental build within a derivation impossible. + +However, we can tell Nix explicitly what the previous build state was, by representing that previous state as a derivation output. This allows the passed build state to be used for an incremental build. + +To change a normal derivation to a checkpoint based build, these steps must be taken: + - apply `prepareCheckpointBuild` on the desired derivation + e.g.: +```nix +checkpointArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox); +``` + - change something you want in the sources of the package. (e.g. using a source override) +```nix +changedVBox = pkgs.virtualbox.overrideAttrs (old: { + src = path/to/vbox/sources; +} +``` + - use `mkCheckpointedBuild changedVBox buildOutput` + - enjoy shorter build times + +## Example {#sec-checkpoint-build-example} +```nix +{ pkgs ? import {} }: with (pkgs) checkpointBuildTools; +let + helloCheckpoint = checkpointBuildTools.prepareCheckpointBuild pkgs.hello; + changedHello = pkgs.hello.overrideAttrs (_: { + doCheck = false; + patchPhase = '' + sed -i 's/Hello, world!/Hello, Nix!/g' src/hello.c + ''; + }); +in checkpointBuildTools.mkCheckpointBuild changedHello helloCheckpoint +``` diff --git a/pkgs/build-support/checkpoint-build.nix b/pkgs/build-support/checkpoint-build.nix new file mode 100644 index 0000000000000..e08dde353e891 --- /dev/null +++ b/pkgs/build-support/checkpoint-build.nix @@ -0,0 +1,69 @@ +{ pkgs }: +rec { + /* Prepare a derivation for local builds. + * + * This function prepares checkpoint builds by provinding, + * containing the build output and the sources for cross checking. + * The build output can be used later to allow checkpoint builds + * by passing the derivation output to the `mkCheckpointBuild` function. + * + * To build a project with checkpoints follow these steps: + * - run prepareIncrementalBuild on the desired derivation + * e.G `incrementalBuildArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox);` + * - change something you want in the sources of the package( e.G using source override) + * changedVBox = pkgs.virtuabox.overrideAttrs (old: { + * src = path/to/vbox/sources; + * } + * - use `mkCheckpointedBuild changedVBox buildOutput` + * - enjoy shorter build times + */ + prepareCheckpointBuild = drv: drv.overrideAttrs (old: { + outputs = [ "out" ]; + name = drv.name + "-checkpointArtifacts"; + # To determine differences between the state of the build directory + # from an earlier build and a later one we store the state of the build + # directory before build, but after patch phases. + # This way, the same derivation can be used multiple times and only changes are detected. + # Additionally Removed files are handled correctly in later builds. + preBuild = (old.preBuild or "") + '' + mkdir -p $out/sources + cp -r ./* $out/sources/ + ''; + + # After the build the build directory is copied again + # to get the output files. + # We copy the complete build folder, to take care for + # Build tools, building in the source directory, instead of + # having a build root directory, e.G the Linux kernel. + installPhase = '' + runHook preCheckpointInstall + mkdir -p $out/outputs + cp -r ./* $out/outputs/ + runHook postCheckpointInstall + ''; + }); + + /* Build a derivation based on the checkpoint output generated by + * the `prepareCheckpointBuild function. + * + * Usage: + * let + * checkpointArtifacts = prepareCheckpointBuild drv + * in mkCheckpointedBuild drv checkpointArtifacts + */ + mkCheckpointedBuild = drv: previousBuildArtifacts: drv.overrideAttrs (old: { + # The actual checkpoint build phase. + # We compare the changed sources from a previous build with the current and create a patch + # Afterwards we clean the build directory to copy the previous output files (Including the sources) + # The source difference patch is applied to get the latest changes again to allow short build times. + preBuild = (old.preBuild or "") + '' + set +e + diff -ur ${previousBuildArtifacts}/sources ./ > sourceDifference.patch + set -e + shopt -s extglob dotglob + rm -r !("sourceDifference.patch") + ${pkgs.rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${previousBuildArtifacts}/outputs/* . + patch -p 1 -i sourceDifference.patch + ''; + }); +} diff --git a/pkgs/test/checkpointBuild/default.nix b/pkgs/test/checkpointBuild/default.nix new file mode 100644 index 0000000000000..4a59760230a6e --- /dev/null +++ b/pkgs/test/checkpointBuild/default.nix @@ -0,0 +1,57 @@ +{ hello, checkpointBuildTools, runCommandNoCC, texinfo, stdenv, rsync }: +let + baseHelloArtifacts = checkpointBuildTools.prepareCheckpointBuild hello; + patchedHello = hello.overrideAttrs (old: { + buildInputs = [ texinfo ]; + src = runCommandNoCC "patch-hello-src" { } '' + mkdir -p $out + cd $out + tar xf ${hello.src} --strip-components=1 + patch -p1 < ${./hello.patch} + ''; + }); + checkpointBuiltHello = checkpointBuildTools.mkCheckpointedBuild patchedHello baseHelloArtifacts; + + checkpointBuiltHelloWithCheck = checkpointBuiltHello.overrideAttrs (old: { + doCheck = true; + checkPhase = '' + echo "checking if unchanged source file is not recompiled" + [ "$(stat --format="%Y" lib/exitfail.o)" = "$(stat --format="%Y" ${baseHelloArtifacts}/outputs/lib/exitfail.o)" ] + ''; + }); + + baseHelloRemoveFileArtifacts = checkpointBuildTools.prepareCheckpointBuild (hello.overrideAttrs (old: { + patches = [ ./hello-additionalFile.patch ]; + })); + + preparedHelloRemoveFileSrc = runCommandNoCC "patch-hello-src" { } '' + mkdir -p $out + cd $out + tar xf ${hello.src} --strip-components=1 + patch -p1 < ${./hello-additionalFile.patch} + ''; + + patchedHelloRemoveFile = hello.overrideAttrs (old: { + buildInputs = [ texinfo ]; + src = runCommandNoCC "patch-hello-src" { } '' + mkdir -p $out + cd $out + ${rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${preparedHelloRemoveFileSrc}/* . + patch -p1 < ${./hello-removeFile.patch} + ''; + }); + + checkpointBuiltHelloWithRemovedFile = checkpointBuildTools.mkCheckpointedBuild patchedHelloRemoveFile baseHelloRemoveFileArtifacts; +in +stdenv.mkDerivation { + name = "patched-hello-returns-correct-output"; + buildCommand = '' + touch $out + + echo "testing output of hello binary" + [ "$(${checkpointBuiltHelloWithCheck}/bin/hello)" = "Hello, incremental world!" ] + echo "testing output of hello with removed file" + [ "$(${checkpointBuiltHelloWithRemovedFile}/bin/hello)" = "Hello, incremental world!" ] + ''; +} + diff --git a/pkgs/test/checkpointBuild/hello-additionalFile.patch b/pkgs/test/checkpointBuild/hello-additionalFile.patch new file mode 100644 index 0000000000000..345bc10ee49e5 --- /dev/null +++ b/pkgs/test/checkpointBuild/hello-additionalFile.patch @@ -0,0 +1,67 @@ +:100644 100644 0000000 0000000 M Makefile.in +:000000 100644 0000000 0000000 A src/additionalFile.c +:100644 100644 0000000 0000000 M src/hello.c +:100644 100644 0000000 0000000 M src/system.h + +diff --git a/Makefile.in b/Makefile.in +index 1597d39..f63f830 100644 +--- a/Makefile.in ++++ b/Makefile.in +@@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \ + lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \ + lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \ + lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \ +- lib/xstrndup.$(OBJEXT) ++ lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT) + lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS) + am_hello_OBJECTS = src/hello.$(OBJEXT) + hello_OBJECTS = $(am_hello_OBJECTS) +@@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \ + $(am__append_4) $(am__append_5) lib/version-etc.h \ + lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \ + lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \ +- lib/xstrndup.h lib/xstrndup.c ++ lib/xstrndup.h lib/xstrndup.c src/additionalFile.c + lib_libhello_a_LIBADD = $(gl_LIBOBJS) + lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS) + EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \ +diff --git a/src/additionalFile.c b/src/additionalFile.c +new file mode 100644 +index 0000000..34d683d +--- /dev/null ++++ b/src/additionalFile.c +@@ -0,0 +1,6 @@ ++#include "config.h" ++#include "system.h" ++ ++int somefunc() { ++ return 0; ++} +diff --git a/src/hello.c b/src/hello.c +index 2e7d38e..a8e36dc 100644 +--- a/src/hello.c ++++ b/src/hello.c +@@ -146,7 +146,11 @@ main (int argc, char *argv[]) + #endif + + /* Having initialized gettext, get the default message. */ +- greeting_msg = _("Hello, world!"); ++ if (somefunc() == 0) { ++ greeting_msg = _("Hello, world!"); ++ } else { ++ greeting_msg = _("Hello, incremental world!"); ++ } + + /* Even exiting has subtleties. On exit, if any writes failed, change + the exit status. The /dev/full device on GNU/Linux can be used for +diff --git a/src/system.h b/src/system.h +index d39cdb9..dc425d2 100644 +--- a/src/system.h ++++ b/src/system.h +@@ -59,4 +59,6 @@ + } \ + while (0) + ++int somefunc(); ++ + #endif /* HELLO_SYSTEM_H */ diff --git a/pkgs/test/checkpointBuild/hello-removeFile.patch b/pkgs/test/checkpointBuild/hello-removeFile.patch new file mode 100644 index 0000000000000..2939790dabce1 --- /dev/null +++ b/pkgs/test/checkpointBuild/hello-removeFile.patch @@ -0,0 +1,67 @@ +:100644 100644 0000000 0000000 M Makefile.in +:100644 000000 0000000 0000000 D src/additionalFile.c +:100644 100644 0000000 0000000 M src/hello.c +:100755 100755 0000000 0000000 M tests/hello-1 + +diff --git a/Makefile.in b/Makefile.in +index f63f830..1597d39 100644 +--- a/Makefile.in ++++ b/Makefile.in +@@ -312,7 +312,7 @@ am_lib_libhello_a_OBJECTS = lib/basename-lgpl.$(OBJEXT) \ + lib/version-etc.$(OBJEXT) lib/version-etc-fsf.$(OBJEXT) \ + lib/wctype-h.$(OBJEXT) lib/xmalloc.$(OBJEXT) \ + lib/xalloc-die.$(OBJEXT) lib/xstriconv.$(OBJEXT) \ +- lib/xstrndup.$(OBJEXT) src/additionalFile.$(OBJEXT) ++ lib/xstrndup.$(OBJEXT) + lib_libhello_a_OBJECTS = $(am_lib_libhello_a_OBJECTS) + am_hello_OBJECTS = src/hello.$(OBJEXT) + hello_OBJECTS = $(am_hello_OBJECTS) +@@ -1842,7 +1842,7 @@ lib_libhello_a_SOURCES = lib/basename-lgpl.c lib/c-ctype.h \ + $(am__append_4) $(am__append_5) lib/version-etc.h \ + lib/version-etc.c lib/version-etc-fsf.c lib/wctype-h.c \ + lib/xmalloc.c lib/xalloc-die.c lib/xstriconv.h lib/xstriconv.c \ +- lib/xstrndup.h lib/xstrndup.c src/additionalFile.c ++ lib/xstrndup.h lib/xstrndup.c + lib_libhello_a_LIBADD = $(gl_LIBOBJS) + lib_libhello_a_DEPENDENCIES = $(gl_LIBOBJS) + EXTRA_lib_libhello_a_SOURCES = lib/close.c lib/stripslash.c lib/dup2.c \ +diff --git a/src/additionalFile.c b/src/additionalFile.c +deleted file mode 100644 +index 34d683d..0000000 +--- a/src/additionalFile.c ++++ /dev/null +@@ -1,6 +0,0 @@ +-#include "config.h" +-#include "system.h" +- +-int somefunc() { +- return 0; +-} +diff --git a/src/hello.c b/src/hello.c +index a8e36dc..53722d9 100644 +--- a/src/hello.c ++++ b/src/hello.c +@@ -126,6 +126,10 @@ parse_options (int argc, char *argv[], const char **greeting_msg) + } + } + ++int somefunc() { ++ return 1; ++} ++ + int + main (int argc, char *argv[]) + { +diff --git a/tests/hello-1 b/tests/hello-1 +index 96ffef8..f0b9f8d 100755 +--- a/tests/hello-1 ++++ b/tests/hello-1 +@@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG + + tmpfiles="hello-test1.ok" + cat < hello-test1.ok +-Hello, world! ++Hello, incremental world! + EOF + + tmpfiles="$tmpfiles hello-test1.out" diff --git a/pkgs/test/checkpointBuild/hello.patch b/pkgs/test/checkpointBuild/hello.patch new file mode 100644 index 0000000000000..3d0d50c2f20e4 --- /dev/null +++ b/pkgs/test/checkpointBuild/hello.patch @@ -0,0 +1,26 @@ +diff --git a/src/hello.c b/src/hello.c +index 182303c..453962f 100644 +--- a/src/hello.c ++++ b/src/hello.c +@@ -57,7 +57,7 @@ main (int argc, char *argv[]) + #endif + + /* Having initialized gettext, get the default message. */ +- greeting_msg = _("Hello, world!"); ++ greeting_msg = _("Hello, incremental world!"); + + /* Even exiting has subtleties. On exit, if any writes failed, change + the exit status. The /dev/full device on GNU/Linux can be used for +diff --git a/tests/hello-1 b/tests/hello-1 +index 3b7a815..e15fa95 100755 +--- a/tests/hello-1 ++++ b/tests/hello-1 +@@ -21,7 +21,7 @@ export LANGUAGE LC_ALL LC_MESSAGES LANG + + tmpfiles="hello-test1.ok" + cat < hello-test1.ok +-Hello, world! ++Hello, incremental world! + EOF + + tmpfiles="$tmpfiles hello-test1.out" diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index 9543e165a8053..d63a0b4ff4403 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -117,6 +117,8 @@ with pkgs; install-shell-files = callPackage ./install-shell-files {}; + checkpoint-build = callPackage ./checkpointBuild {}; + kernel-config = callPackage ./kernel.nix {}; ld-library-path = callPackage ./ld-library-path {}; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f4c4a5b03e9e4..fbd4097caa175 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -424,6 +424,8 @@ with pkgs; camunda-modeler = callPackage ../applications/misc/camunda-modeler { }; + checkpointBuildTools = callPackage ../build-support/checkpoint-build.nix {}; + caroline = callPackage ../development/libraries/caroline { }; cartridges = callPackage ../applications/misc/cartridges { }; From 1583f4e1e02f0b2f92bccf7999a0938fd61a4a7e Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Sun, 7 Jan 2024 18:47:47 +0800 Subject: [PATCH 2/6] checkpointBuildTools: mkCheckpointedBuild -> mkCheckpointBuild All other functions are in the form of `*{c,C}heckpointBuild*`, so we deprecate the `mkCheckpointedBuild` function in favor of `mkCheckpointBuild`. Also address some inconsistencies in the docs: some `buildOutput` should actually be `incrementalBuildArtifacts`. (cherry picked from commit df62c3c87f35cd50d6ee20ea92900bb03ad827c9) --- doc/build-helpers/special/checkpoint-build.section.md | 2 +- pkgs/build-support/checkpoint-build.nix | 10 +++++++--- pkgs/test/checkpointBuild/default.nix | 4 ++-- pkgs/test/default.nix | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/build-helpers/special/checkpoint-build.section.md b/doc/build-helpers/special/checkpoint-build.section.md index 5f01e699b9477..676f04aa7a4eb 100644 --- a/doc/build-helpers/special/checkpoint-build.section.md +++ b/doc/build-helpers/special/checkpoint-build.section.md @@ -18,7 +18,7 @@ changedVBox = pkgs.virtualbox.overrideAttrs (old: { src = path/to/vbox/sources; } ``` - - use `mkCheckpointedBuild changedVBox buildOutput` + - use `mkCheckpointBuild changedVBox incrementalBuildArtifacts` - enjoy shorter build times ## Example {#sec-checkpoint-build-example} diff --git a/pkgs/build-support/checkpoint-build.nix b/pkgs/build-support/checkpoint-build.nix index e08dde353e891..5876202b0bd04 100644 --- a/pkgs/build-support/checkpoint-build.nix +++ b/pkgs/build-support/checkpoint-build.nix @@ -14,7 +14,7 @@ rec { * changedVBox = pkgs.virtuabox.overrideAttrs (old: { * src = path/to/vbox/sources; * } - * - use `mkCheckpointedBuild changedVBox buildOutput` + * - use `mkCheckpointBuild changedVBox incrementalBuildArtifacts` * - enjoy shorter build times */ prepareCheckpointBuild = drv: drv.overrideAttrs (old: { @@ -49,9 +49,9 @@ rec { * Usage: * let * checkpointArtifacts = prepareCheckpointBuild drv - * in mkCheckpointedBuild drv checkpointArtifacts + * in mkCheckpointBuild drv checkpointArtifacts */ - mkCheckpointedBuild = drv: previousBuildArtifacts: drv.overrideAttrs (old: { + mkCheckpointBuild = drv: previousBuildArtifacts: drv.overrideAttrs (old: { # The actual checkpoint build phase. # We compare the changed sources from a previous build with the current and create a patch # Afterwards we clean the build directory to copy the previous output files (Including the sources) @@ -66,4 +66,8 @@ rec { patch -p 1 -i sourceDifference.patch ''; }); + + mkCheckpointedBuild = pkgs.lib.warn + "`mkCheckpointedBuild` is deprecated, use `mkCheckpointBuild` instead!" + mkCheckpointBuild; } diff --git a/pkgs/test/checkpointBuild/default.nix b/pkgs/test/checkpointBuild/default.nix index 4a59760230a6e..9237262640999 100644 --- a/pkgs/test/checkpointBuild/default.nix +++ b/pkgs/test/checkpointBuild/default.nix @@ -10,7 +10,7 @@ let patch -p1 < ${./hello.patch} ''; }); - checkpointBuiltHello = checkpointBuildTools.mkCheckpointedBuild patchedHello baseHelloArtifacts; + checkpointBuiltHello = checkpointBuildTools.mkCheckpointBuild patchedHello baseHelloArtifacts; checkpointBuiltHelloWithCheck = checkpointBuiltHello.overrideAttrs (old: { doCheck = true; @@ -41,7 +41,7 @@ let ''; }); - checkpointBuiltHelloWithRemovedFile = checkpointBuildTools.mkCheckpointedBuild patchedHelloRemoveFile baseHelloRemoveFileArtifacts; + checkpointBuiltHelloWithRemovedFile = checkpointBuildTools.mkCheckpointBuild patchedHelloRemoveFile baseHelloRemoveFileArtifacts; in stdenv.mkDerivation { name = "patched-hello-returns-correct-output"; diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index d63a0b4ff4403..097749e218b4e 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -117,7 +117,7 @@ with pkgs; install-shell-files = callPackage ./install-shell-files {}; - checkpoint-build = callPackage ./checkpointBuild {}; + checkpointBuildTools = callPackage ./checkpointBuild {}; kernel-config = callPackage ./kernel.nix {}; From d9f56abcc7c9dc40a83762155775394817faeca6 Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Sun, 7 Jan 2024 21:27:30 +0800 Subject: [PATCH 3/6] checkpointBuildTools: minor refactor Add some minor improvements of the package, and use temp files for the source difference patch, such that it is more reliable. This follows from the suggestions of @infinisil. Co-authored-by: Robert Hensing (cherry picked from commit 5f3aad00ffc0a769a1ccc5b3b521221aa5e5d809) --- pkgs/build-support/checkpoint-build.nix | 32 ++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/pkgs/build-support/checkpoint-build.nix b/pkgs/build-support/checkpoint-build.nix index 5876202b0bd04..3537eddfbcbc5 100644 --- a/pkgs/build-support/checkpoint-build.nix +++ b/pkgs/build-support/checkpoint-build.nix @@ -1,4 +1,16 @@ -{ pkgs }: +{ lib +, buildPackages +}: + +let + # rudimentary support for cross-compiling + # see: https://github.com/NixOS/nixpkgs/pull/279487#discussion_r1444449726 + inherit (buildPackages) + mktemp + rsync + ; +in + rec { /* Prepare a derivation for local builds. * @@ -51,23 +63,27 @@ rec { * checkpointArtifacts = prepareCheckpointBuild drv * in mkCheckpointBuild drv checkpointArtifacts */ - mkCheckpointBuild = drv: previousBuildArtifacts: drv.overrideAttrs (old: { + mkCheckpointBuild = drv: checkpointArtifacts: drv.overrideAttrs (old: { # The actual checkpoint build phase. # We compare the changed sources from a previous build with the current and create a patch # Afterwards we clean the build directory to copy the previous output files (Including the sources) # The source difference patch is applied to get the latest changes again to allow short build times. preBuild = (old.preBuild or "") + '' set +e - diff -ur ${previousBuildArtifacts}/sources ./ > sourceDifference.patch + sourceDifferencePatchFile=$(${mktemp}/bin/mktemp) + diff -ur ${checkpointArtifacts}/sources ./ > "$sourceDifferencePatchFile" set -e - shopt -s extglob dotglob - rm -r !("sourceDifference.patch") - ${pkgs.rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${previousBuildArtifacts}/outputs/* . - patch -p 1 -i sourceDifference.patch + shopt -s dotglob + rm -r * + ${rsync}/bin/rsync \ + --checksum --times --atimes --chown=$USER:$USER --chmod=+w \ + -r ${checkpointArtifacts}/outputs/ . + patch -p 1 -i "$sourceDifferencePatchFile" + rm "$sourceDifferencePatchFile" ''; }); - mkCheckpointedBuild = pkgs.lib.warn + mkCheckpointedBuild = lib.warn "`mkCheckpointedBuild` is deprecated, use `mkCheckpointBuild` instead!" mkCheckpointBuild; } From d4a73a248849c6ac586ace3294e65dc4c1251a1e Mon Sep 17 00:00:00 2001 From: Bryan Lai Date: Sun, 7 Jan 2024 18:57:04 +0800 Subject: [PATCH 4/6] doc: polish the docs of `checkpointBuildTools` ... following suggestions from @phip1611 and @infinisil. (cherry picked from commit d7253bea6d7366987acce31c2c2355ffbdf389b4) --- .../special/checkpoint-build.section.md | 21 +++++---- pkgs/build-support/checkpoint-build.nix | 43 ++++++++++--------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/doc/build-helpers/special/checkpoint-build.section.md b/doc/build-helpers/special/checkpoint-build.section.md index 676f04aa7a4eb..f60afe801ed4c 100644 --- a/doc/build-helpers/special/checkpoint-build.section.md +++ b/doc/build-helpers/special/checkpoint-build.section.md @@ -2,35 +2,38 @@ `pkgs.checkpointBuildTools` provides a way to build derivations incrementally. It consists of two functions to make checkpoint builds using Nix possible. -For hermeticity, Nix derivations do not allow any state to carry over between builds, making a transparent incremental build within a derivation impossible. +For hermeticity, Nix derivations do not allow any state to be carried over between builds, making a transparent incremental build within a derivation impossible. However, we can tell Nix explicitly what the previous build state was, by representing that previous state as a derivation output. This allows the passed build state to be used for an incremental build. To change a normal derivation to a checkpoint based build, these steps must be taken: - - apply `prepareCheckpointBuild` on the desired derivation - e.g.: + - apply `prepareCheckpointBuild` on the desired derivation, e.g. ```nix checkpointArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox); ``` - - change something you want in the sources of the package. (e.g. using a source override) + - change something you want in the sources of the package, e.g. use a source override: ```nix changedVBox = pkgs.virtualbox.overrideAttrs (old: { src = path/to/vbox/sources; -} +}); ``` - - use `mkCheckpointBuild changedVBox incrementalBuildArtifacts` + - use `mkCheckpointBuild changedVBox checkpointArtifacts` - enjoy shorter build times ## Example {#sec-checkpoint-build-example} ```nix -{ pkgs ? import {} }: with (pkgs) checkpointBuildTools; +{ pkgs ? import {} }: let - helloCheckpoint = checkpointBuildTools.prepareCheckpointBuild pkgs.hello; + inherit (pkgs.checkpointBuildTools) + prepareCheckpointBuild + mkCheckpointBuild + ; + helloCheckpoint = prepareCheckpointBuild pkgs.hello; changedHello = pkgs.hello.overrideAttrs (_: { doCheck = false; patchPhase = '' sed -i 's/Hello, world!/Hello, Nix!/g' src/hello.c ''; }); -in checkpointBuildTools.mkCheckpointBuild changedHello helloCheckpoint +in mkCheckpointBuild changedHello helloCheckpoint ``` diff --git a/pkgs/build-support/checkpoint-build.nix b/pkgs/build-support/checkpoint-build.nix index 3537eddfbcbc5..c9bee45005a13 100644 --- a/pkgs/build-support/checkpoint-build.nix +++ b/pkgs/build-support/checkpoint-build.nix @@ -14,39 +14,40 @@ in rec { /* Prepare a derivation for local builds. * - * This function prepares checkpoint builds by provinding, - * containing the build output and the sources for cross checking. + * This function prepares checkpoint builds by storing + * the build output and the sources for cross checking. * The build output can be used later to allow checkpoint builds * by passing the derivation output to the `mkCheckpointBuild` function. * - * To build a project with checkpoints follow these steps: - * - run prepareIncrementalBuild on the desired derivation - * e.G `incrementalBuildArtifacts = (pkgs.checkpointBuildTools.prepareCheckpointBuild pkgs.virtualbox);` - * - change something you want in the sources of the package( e.G using source override) - * changedVBox = pkgs.virtuabox.overrideAttrs (old: { - * src = path/to/vbox/sources; - * } - * - use `mkCheckpointBuild changedVBox incrementalBuildArtifacts` + * To build a project with checkpoints, follow these steps: + * - run `prepareCheckpointBuild` on the desired derivation, e.g. + * checkpointArtifacts = prepareCheckpointBuild virtualbox; + * - change something you want in the sources of the package, + * e.g. using source override: + * changedVBox = pkgs.virtuabox.overrideAttrs (old: { + * src = path/to/vbox/sources; + * }; + * - use `mkCheckpointBuild changedVBox checkpointArtifacts` * - enjoy shorter build times */ prepareCheckpointBuild = drv: drv.overrideAttrs (old: { outputs = [ "out" ]; name = drv.name + "-checkpointArtifacts"; # To determine differences between the state of the build directory - # from an earlier build and a later one we store the state of the build + # from an earlier build and a later one we store the state of the build # directory before build, but after patch phases. # This way, the same derivation can be used multiple times and only changes are detected. - # Additionally Removed files are handled correctly in later builds. + # Additionally, removed files are handled correctly in later builds. preBuild = (old.preBuild or "") + '' mkdir -p $out/sources cp -r ./* $out/sources/ ''; - # After the build the build directory is copied again + # After the build, the build directory is copied again # to get the output files. - # We copy the complete build folder, to take care for - # Build tools, building in the source directory, instead of - # having a build root directory, e.G the Linux kernel. + # We copy the complete build folder, to take care of + # build tools that build in the source directory, instead of + # having a separate build directory such as the Linux kernel. installPhase = '' runHook preCheckpointInstall mkdir -p $out/outputs @@ -56,18 +57,18 @@ rec { }); /* Build a derivation based on the checkpoint output generated by - * the `prepareCheckpointBuild function. + * the `prepareCheckpointBuild` function. * * Usage: * let - * checkpointArtifacts = prepareCheckpointBuild drv + * checkpointArtifacts = prepareCheckpointBuild drv; * in mkCheckpointBuild drv checkpointArtifacts */ mkCheckpointBuild = drv: checkpointArtifacts: drv.overrideAttrs (old: { # The actual checkpoint build phase. - # We compare the changed sources from a previous build with the current and create a patch - # Afterwards we clean the build directory to copy the previous output files (Including the sources) - # The source difference patch is applied to get the latest changes again to allow short build times. + # We compare the changed sources from a previous build with the current and create a patch. + # Afterwards we clean the build directory and copy the previous output files (including the sources). + # The source difference patch is then applied to get the latest changes again to allow short build times. preBuild = (old.preBuild or "") + '' set +e sourceDifferencePatchFile=$(${mktemp}/bin/mktemp) From 03cc589f5ed010d33906248dcca3edfbe2b75e34 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Thu, 4 Jan 2024 21:39:07 +0100 Subject: [PATCH 5/6] tests.checkpoint-build: Fix evaluation with allowAliases false Was broken since introduced, but not caught due to https://github.com/NixOS/nixpkgs/pull/271123#discussion_r1442134594 (cherry picked from commit 69460711750d4bb4718516fdb155145e922ff610) --- pkgs/test/checkpointBuild/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/test/checkpointBuild/default.nix b/pkgs/test/checkpointBuild/default.nix index 9237262640999..0843dcd3d6eba 100644 --- a/pkgs/test/checkpointBuild/default.nix +++ b/pkgs/test/checkpointBuild/default.nix @@ -1,9 +1,9 @@ -{ hello, checkpointBuildTools, runCommandNoCC, texinfo, stdenv, rsync }: +{ hello, checkpointBuildTools, runCommand, texinfo, stdenv, rsync }: let baseHelloArtifacts = checkpointBuildTools.prepareCheckpointBuild hello; patchedHello = hello.overrideAttrs (old: { buildInputs = [ texinfo ]; - src = runCommandNoCC "patch-hello-src" { } '' + src = runCommand "patch-hello-src" { } '' mkdir -p $out cd $out tar xf ${hello.src} --strip-components=1 @@ -24,7 +24,7 @@ let patches = [ ./hello-additionalFile.patch ]; })); - preparedHelloRemoveFileSrc = runCommandNoCC "patch-hello-src" { } '' + preparedHelloRemoveFileSrc = runCommand "patch-hello-src" { } '' mkdir -p $out cd $out tar xf ${hello.src} --strip-components=1 @@ -33,7 +33,7 @@ let patchedHelloRemoveFile = hello.overrideAttrs (old: { buildInputs = [ texinfo ]; - src = runCommandNoCC "patch-hello-src" { } '' + src = runCommand "patch-hello-src" { } '' mkdir -p $out cd $out ${rsync}/bin/rsync -cutU --chown=$USER:$USER --chmod=+w -r ${preparedHelloRemoveFileSrc}/* . From a3b90e768b8d43e0efb9cc227153d55609c65999 Mon Sep 17 00:00:00 2001 From: bryango Date: Thu, 11 Jan 2024 22:52:52 +0800 Subject: [PATCH 6/6] checkpointBuildTools.mkCheckpointedBuild: update deprecation warning Co-authored-by: Robert Hensing --- pkgs/build-support/checkpoint-build.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/checkpoint-build.nix b/pkgs/build-support/checkpoint-build.nix index c9bee45005a13..cc70eddd7ba30 100644 --- a/pkgs/build-support/checkpoint-build.nix +++ b/pkgs/build-support/checkpoint-build.nix @@ -85,6 +85,6 @@ rec { }); mkCheckpointedBuild = lib.warn - "`mkCheckpointedBuild` is deprecated, use `mkCheckpointBuild` instead!" + "`mkCheckpointedBuild` is deprecated; use `mkCheckpointBuild` instead." mkCheckpointBuild; }