From ad11ffd4372cc994c3bb50c70ba9b5ac8c89e4ec Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 15 Apr 2024 12:45:45 -0700 Subject: [PATCH 1/3] haskell.lib.compose.justStaticExecutables: Forbid references to GHC This makes `justStaticExecutables` error if the produced store path contains references to GHC. This is almost always erroneous and due to the generated `Paths_*` module being imported. This helps prevent `justStaticExecutables` from producing binaries with closure sizes in the gigabytes. See: https://github.com/NixOS/nixpkgs/issues/164630 Co-authored-by: sternenseemann (cherry picked from commit d2618822ab1a3f7bcae64fb49c36a51b09a03f1f) (minus release note) --- doc/languages-frameworks/haskell.section.md | 61 ++++++++++++++++--- .../haskell-modules/generic-builder.nix | 32 +++++++++- .../haskell-modules/lib/compose.nix | 3 +- 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/doc/languages-frameworks/haskell.section.md b/doc/languages-frameworks/haskell.section.md index dde55c329a4ac..e363e90854ad3 100644 --- a/doc/languages-frameworks/haskell.section.md +++ b/doc/languages-frameworks/haskell.section.md @@ -923,14 +923,59 @@ for this to work. `justStaticExecutables drv` : Only build and install the executables produced by `drv`, removing everything -that may refer to other Haskell packages' store paths (like libraries and -documentation). This dramatically reduces the closure size of the resulting -derivation. Note that the executables are only statically linked against their -Haskell dependencies, but will still link dynamically against libc, GMP and -other system library dependencies. If dependencies use their Cabal-generated -`Paths_*` module, this may not work as well if GHC's dead code elimination -is unable to remove the references to the dependency's store path that module -contains. + that may refer to other Haskell packages' store paths (like libraries and + documentation). This dramatically reduces the closure size of the resulting + derivation. Note that the executables are only statically linked against their + Haskell dependencies, but will still link dynamically against libc, GMP and + other system library dependencies. + + If the library being built or its dependencies use their Cabal-generated + `Paths_*` module, this may not work as well if GHC's dead code elimination is + unable to remove the references to the dependency's store path that module + contains. (See [nixpkgs#164630][164630] for more information.) + + Importing the `Paths_*` module may cause builds to fail with this message: + + ``` + error: output '/nix/store/64k8iw0ryz76qpijsnl9v87fb26v28z8-my-haskell-package-1.0.0.0' is not allowed to refer to the following paths: + /nix/store/5q5s4a07gaz50h04zpfbda8xjs8wrnhg-ghc-9.6.3 + ``` + + If that happens, first disable the check for GHC references and rebuild the + derivation: + + ```nix + pkgs.haskell.lib.overrideCabal + (pkgs.haskell.lib.justStaticExecutables my-haskell-package) + (drv: { + disallowGhcReference = false; + }) + ``` + + Then use `strings` to determine which libraries are responsible: + + ``` + $ nix-build ... + $ strings result/bin/my-haskell-binary | grep /nix/store/ + ... + /nix/store/n7ciwdlg8yyxdhbrgd6yc2d8ypnwpmgq-hs-opentelemetry-sdk-0.0.3.6/bin + ... + ``` + + Finally, use `remove-references-to` to delete those store paths from the produced output: + + ```nix + pkgs.haskell.lib.overrideCabal + (pkgs.haskell.lib.justStaticExecutables my-haskell-package) + (drv: { + postInstall = '' + ${drv.postInstall or ""} + remove-references-to -t ${pkgs.haskellPackages.hs-opentelemetry-sdk} + ''; + }) + ``` + +[164630]: https://github.com/NixOS/nixpkgs/issues/164630 `enableSeparateBinOutput drv` : Install executables produced by `drv` to a separate `bin` output. This diff --git a/pkgs/development/haskell-modules/generic-builder.nix b/pkgs/development/haskell-modules/generic-builder.nix index f9acdd0f4a27b..280d42b812892 100644 --- a/pkgs/development/haskell-modules/generic-builder.nix +++ b/pkgs/development/haskell-modules/generic-builder.nix @@ -110,6 +110,28 @@ in # build products from that prior build as a starting point for accelerating # this build , previousIntermediates ? null + # References to these store paths are forbidden in the produced output. +, disallowedRequisites ? [] + # Whether to allow the produced output to refer to `ghc`. + # + # This is used by `haskell.lib.justStaticExecutables` to help prevent static + # Haskell binaries from having erroneous dependencies on GHC. + # + # Generated `Paths_*` modules include paths for the runtime library + # directory (and similar) of the package being built. If the `Paths_*` + # module is imported, this creates a dependency from the static binary + # being built to the _library_ being built (which is dynamically linked + # and depends on the GHC used to build it). + # + # To avoid this: + # 1. Build the impacted derivation. + # 2. Run `strings` on the built binary of the impacted derivation to + # locate the store paths it depends on. + # 3. Add `remove-references-to -t ${bad-store-path-in-binary}` to the + # impacted derivation's `postInstall`. + # + # See: https://github.com/NixOS/nixpkgs/issues/164630 +, disallowGhcReference ? false , # Cabal 3.8 which is shipped by default for GHC >= 9.3 always calls # `pkg-config --libs --static` as part of the configure step. This requires # Requires.private dependencies of pkg-config dependencies to be present in @@ -645,9 +667,17 @@ stdenv.mkDerivation ({ runHook postInstallIntermediates ''; + disallowedRequisites = + disallowedRequisites + ++ ( + if disallowGhcReference + then [ghc] + else [] + ); + passthru = passthru // rec { - inherit pname version; + inherit pname version disallowGhcReference; compiler = ghc; diff --git a/pkgs/development/haskell-modules/lib/compose.nix b/pkgs/development/haskell-modules/lib/compose.nix index 09cee08b91c1f..492091ef35fc2 100644 --- a/pkgs/development/haskell-modules/lib/compose.nix +++ b/pkgs/development/haskell-modules/lib/compose.nix @@ -290,7 +290,7 @@ rec { /* link executables statically against haskell libs to reduce closure size */ - justStaticExecutables = overrideCabal (drv: { + justStaticExecutables = overrideCabal (drv: { enableSharedExecutables = false; enableLibraryProfiling = false; isLibrary = false; @@ -300,6 +300,7 @@ rec { # Remove every directory which could have links to other store paths. rm -rf $out/lib $out/nix-support $out/share/doc ''; + disallowGhcReference = true; }); /* Build a source distribution tarball instead of using the source files From 3ae13a2f42abeea55f100cc804c195760593d5e6 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Mon, 27 May 2024 16:21:16 +0200 Subject: [PATCH 2/3] haskellPackages.mkDerivation: no rebuild w/o disallowedRequisites This change ensures that packages won't be rebuild compared to before the introduction of disallowedRequisites/disallowGhcReference unless they use one of those arguments. It may be nice to revert this in the future (via staging) for greater simplicity, but will help with initial regression testing. (cherry picked from commit 0454f7b8ee334ddcb3dd1b690719f5acfb625d2f) --- .../haskell-modules/generic-builder.nix | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkgs/development/haskell-modules/generic-builder.nix b/pkgs/development/haskell-modules/generic-builder.nix index 280d42b812892..8c639262b2c6b 100644 --- a/pkgs/development/haskell-modules/generic-builder.nix +++ b/pkgs/development/haskell-modules/generic-builder.nix @@ -667,14 +667,6 @@ stdenv.mkDerivation ({ runHook postInstallIntermediates ''; - disallowedRequisites = - disallowedRequisites - ++ ( - if disallowGhcReference - then [ghc] - else [] - ); - passthru = passthru // rec { inherit pname version disallowGhcReference; @@ -838,10 +830,19 @@ stdenv.mkDerivation ({ // optionalAttrs (args ? dontStrip) { inherit dontStrip; } // optionalAttrs (postPhases != []) { inherit postPhases; } // optionalAttrs (stdenv.buildPlatform.libc == "glibc"){ LOCALE_ARCHIVE = "${glibcLocales}/lib/locale/locale-archive"; } +// optionalAttrs (disallowedRequisites != [] || disallowGhcReference) { + disallowedRequisites = + disallowedRequisites + ++ ( + if disallowGhcReference + then [ghc] + else [] + ); +} # Implicit pointer to integer conversions are errors by default since clang 15. # Works around https://gitlab.haskell.org/ghc/ghc/-/issues/23456. -// lib.optionalAttrs (stdenv.hasCC && stdenv.cc.isClang) { +// optionalAttrs (stdenv.hasCC && stdenv.cc.isClang) { NIX_CFLAGS_COMPILE = "-Wno-error=int-conversion"; } ) From df030aca8b941a123104e949fc7d245bf9be6612 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 12 Jun 2024 12:21:54 +0200 Subject: [PATCH 3/3] haskell.lib.compose.justStaticExecutables: Unforbid references to GHC for backport --- doc/languages-frameworks/haskell.section.md | 25 +++++++++++-------- .../haskell-modules/generic-builder.nix | 5 ++-- .../haskell-modules/lib/compose.nix | 3 ++- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/doc/languages-frameworks/haskell.section.md b/doc/languages-frameworks/haskell.section.md index e363e90854ad3..3843f3a36eeef 100644 --- a/doc/languages-frameworks/haskell.section.md +++ b/doc/languages-frameworks/haskell.section.md @@ -934,25 +934,28 @@ for this to work. unable to remove the references to the dependency's store path that module contains. (See [nixpkgs#164630][164630] for more information.) - Importing the `Paths_*` module may cause builds to fail with this message: - - ``` - error: output '/nix/store/64k8iw0ryz76qpijsnl9v87fb26v28z8-my-haskell-package-1.0.0.0' is not allowed to refer to the following paths: - /nix/store/5q5s4a07gaz50h04zpfbda8xjs8wrnhg-ghc-9.6.3 - ``` - - If that happens, first disable the check for GHC references and rebuild the - derivation: + Nixpkgs unstable and releases from 24.11 onwards will verify that such + references are eliminated and fail the build if they are not. + You can opt in to this new behavior as follows: ```nix pkgs.haskell.lib.overrideCabal (pkgs.haskell.lib.justStaticExecutables my-haskell-package) (drv: { - disallowGhcReference = false; + disallowGhcReference = true; }) ``` - Then use `strings` to determine which libraries are responsible: + on the package you are statically linking with `justStaticExecutables`. + After doing so, importing the `Paths_*` module may cause builds to fail with this message: + + ``` + error: output '/nix/store/64k8iw0ryz76qpijsnl9v87fb26v28z8-my-haskell-package-1.0.0.0' is not allowed to refer to the following paths: + /nix/store/5q5s4a07gaz50h04zpfbda8xjs8wrnhg-ghc-9.6.3 + ``` + + If that happens, temporarily disable the check for GHC references and rebuild the + derivation. Then use `strings` to determine which libraries are responsible: ``` $ nix-build ... diff --git a/pkgs/development/haskell-modules/generic-builder.nix b/pkgs/development/haskell-modules/generic-builder.nix index 8c639262b2c6b..e383af0b42281 100644 --- a/pkgs/development/haskell-modules/generic-builder.nix +++ b/pkgs/development/haskell-modules/generic-builder.nix @@ -114,8 +114,9 @@ in , disallowedRequisites ? [] # Whether to allow the produced output to refer to `ghc`. # - # This is used by `haskell.lib.justStaticExecutables` to help prevent static - # Haskell binaries from having erroneous dependencies on GHC. + # This can be used in conjunction with `haskell.lib.justStaticExecutables` + # to help prevent static Haskell binaries from having erroneous dependencies + # on GHC. # # Generated `Paths_*` modules include paths for the runtime library # directory (and similar) of the package being built. If the `Paths_*` diff --git a/pkgs/development/haskell-modules/lib/compose.nix b/pkgs/development/haskell-modules/lib/compose.nix index 492091ef35fc2..3717ed98d4a1d 100644 --- a/pkgs/development/haskell-modules/lib/compose.nix +++ b/pkgs/development/haskell-modules/lib/compose.nix @@ -300,7 +300,8 @@ rec { # Remove every directory which could have links to other store paths. rm -rf $out/lib $out/nix-support $out/share/doc ''; - disallowGhcReference = true; + # We don't do this yet in 24.05. Unstable has it, and 24.11 will have it. + # disallowGhcReference = true; }); /* Build a source distribution tarball instead of using the source files