diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f83a2c0..8efa100 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -51,3 +51,12 @@ steps: command: | NIX_PATH=nixpkgs=nixpkgs nix-build --no-link survey/default.nix \ --argstr compiler ghc865 -A workingStackageExecutables + + # Stack via stack2nix + + - label: static-stack + command: | + cd static-stack/ + mkdir -p static-stack-test-dir + curl -L https://github.com/commercialhaskell/stack/archive/v2.1.3.tar.gz | tar -xz -C static-stack-test-dir + $(nix-build --no-link -A fullBuildScript --argstr stackDir $PWD/static-stack-test-dir/stack-*) diff --git a/.in-static-haskell-nix b/.in-static-haskell-nix new file mode 100644 index 0000000..cdbd22d --- /dev/null +++ b/.in-static-haskell-nix @@ -0,0 +1 @@ +The presence of this file indicates that we're inside the static-haskell-nix project. diff --git a/default.nix b/default.nix index fd6a931..ef58216 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ # Note: This is just a minimal example. For proper usage, see the README. -{ nixpkgs ? (import {}).pkgsMusl, compiler ? "ghc864", strip ? true }: +{ nixpkgs ? (import ./nixpkgs.nix).pkgsMusl, compiler ? "ghc864", strip ? true }: let diff --git a/nixpkgs b/nixpkgs index a2d7e9b..b577340 160000 --- a/nixpkgs +++ b/nixpkgs @@ -1 +1 @@ -Subproject commit a2d7e9b875e8ba7fd15b989cf2d80be4e183dc72 +Subproject commit b577340eb5bc3b72549f0544b50e2e37df78bf12 diff --git a/nixpkgs.nix b/nixpkgs.nix new file mode 100644 index 0000000..fd2687e --- /dev/null +++ b/nixpkgs.nix @@ -0,0 +1,10 @@ +# If a `./nixpkgs` submodule exists, use that. +# Note that this will take precedence over setting NIX_PATH! +# We prefer this such that `static-stack2nix-builder` and specifically +# `static-stack2nix-builder-example` can just import `nixpkgs.nix` +# in CI and when called during development to get the right version of +# nixpkgs. +if builtins.pathExists ./nixpkgs/pkgs + then import ./nixpkgs {} + # Pinned nixpkgs version; should be kept up-to-date with our submodule. + else import (fetchTarball https://github.com/nh2/nixpkgs/archive/b577340eb5bc3b72549f0544b50e2e37df78bf12.tar.gz) {} diff --git a/static-stack/default.nix b/static-stack/default.nix index e16e9c7..a643e34 100644 --- a/static-stack/default.nix +++ b/static-stack/default.nix @@ -16,7 +16,7 @@ let stack2nix-script = import ../static-stack2nix-builder/stack2nix-script.nix { pkgs = pkgs; stack-project-dir = stackDir; # where stack.yaml is - hackageSnapshot = "2019-05-08T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions + hackageSnapshot = "2019-08-17T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions }; static-stack2nix-builder = import ../static-stack2nix-builder/default.nix { diff --git a/static-stack2nix-builder-example/default.nix b/static-stack2nix-builder-example/default.nix index 575d2a8..7fda7b9 100644 --- a/static-stack2nix-builder-example/default.nix +++ b/static-stack2nix-builder-example/default.nix @@ -8,11 +8,18 @@ let cabalPackageName = "example-project"; compiler = "ghc864"; # matching stack.yaml - # Pin nixpkgs version. - pkgs = import (fetchTarball https://github.com/nh2/nixpkgs/archive/a2d7e9b875e8ba7fd15b989cf2d80be4e183dc72.tar.gz) {}; - # Pin static-haskell-nix version. - static-haskell-nix = fetchTarball https://github.com/nh2/static-haskell-nix/archive/1d37d9a83e570eceef9c7dad5c89557f8179a076.tar.gz; + static-haskell-nix = + if builtins.pathExists ../.in-static-haskell-nix + then toString ../. # for the case that we're in static-haskell-nix itself, so that CI always builds the latest version. + # Update this hash to use a different `static-haskell-nix` version: + else fetchTarball https://github.com/nh2/static-haskell-nix/archive/4521a448dda7a613e29d3bcd8e725c96202c5728.tar.gz; + + # Pin nixpkgs version + # By default to the one `static-haskell-nix` provides, but you may also give + # your own as long as it has the necessary patches, using e.g. + # pkgs = import (fetchTarball https://github.com/nh2/nixpkgs/archive/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa123.tar.gz) {}; + pkgs = import "${static-haskell-nix}/nixpkgs.nix"; stack2nix-script = import "${static-haskell-nix}/static-stack2nix-builder/stack2nix-script.nix" { inherit pkgs; @@ -31,6 +38,7 @@ let #!/usr/bin/env bash set -eu -o pipefail STACK2NIX_OUTPUT_PATH=$(${stack2nix-script}) + export NIX_PATH=nixpkgs=${pkgs.path} ${pkgs.nix}/bin/nix-build --no-link -A static_package --argstr stack2nix-output-path "$STACK2NIX_OUTPUT_PATH" "$@" ''; diff --git a/static-stack2nix-builder/stack2nix-script.nix b/static-stack2nix-builder/stack2nix-script.nix index 6025c06..d7e045d 100644 --- a/static-stack2nix-builder/stack2nix-script.nix +++ b/static-stack2nix-builder/stack2nix-script.nix @@ -48,6 +48,34 @@ "${pkgs.nix}/bin" # various `nix-*` commands "${pkgs.wget}/bin" # `wget` ]; + + fixed_stack2nix = + let + # stack2nix isn't compatible with Stack >= 2.0, see + # https://github.com/input-output-hk/stack2nix/issues/168. + # Current versions of nixpkgs master have Stack >= 2.0, see + # https://github.com/NixOS/nixpkgs/issues/63691. + # We thus fetch the `stack2nix` binary from an older nixpkgs version + # that doesn't have Stack >= 2.0. + # This means that `static-stack2nix-builder` may not work on `stack.yaml` + # files that aren't compatible with Stack < 2.0. + stack2nix_pkgs = import (fetchTarball https://github.com/NixOS/nixpkgs/archive/e36f91fa86109fa93cac2516a9365af57233a3a6.tar.gz) {}; + in + # Some older stack2nix versions have fundamental problems that prevent + # stack2nix from running correctly. Fix them here, until these old versions + # are faded out of current nixpkgs. Especially: + # * "Make sure output is written in UTF-8." + # https://github.com/input-output-hk/stack2nix/commit/cb05818ef8b58899f15641f50cb04e5473b4f9b0 + # + # Versions < 0.2.3 aren't supported, force-upgrade them to 0.2.3. + if stack2nix_pkgs.lib.versionOlder stack2nix_pkgs.stack2nix.version "0.2.3" + then stack2nix_pkgs.haskellPackages.callCabal2nix "stack2nix" (stack2nix_pkgs.fetchFromGitHub { + owner = "input-output-hk"; + repo = "stack2nix"; + rev = "v0.2.3"; + sha256 = "1b4g7800hvhr97cjssy5ffd097n2z0fvk9cm31a5jh66pkxys0mq"; + }) {} + else stack2nix_pkgs.stack2nix; in pkgs.writeScript "stack2nix-build-script.sh" '' #!/usr/bin/env bash @@ -56,6 +84,6 @@ export PATH=${pkgs.lib.concatStringsSep ":" add_to_PATH}:$PATH OUT_DIR=$(mktemp --directory -t stack2nix-output-dir.XXXXXXXXXX) set -x - ${pkgs.stack2nix}/bin/stack2nix "${stack-project-dir}" --stack-yaml "${stack-yaml}" --hackage-snapshot "${hackageSnapshot}" -o "$OUT_DIR/stack2nix-output.nix" "$@" 1>&2 + ${fixed_stack2nix}/bin/stack2nix "${stack-project-dir}" --stack-yaml "${stack-yaml}" --hackage-snapshot "${hackageSnapshot}" -o "$OUT_DIR/stack2nix-output.nix" "$@" 1>&2 nix-store --add "$OUT_DIR/stack2nix-output.nix" '' diff --git a/survey/default.nix b/survey/default.nix index 1f55f67..b6f82f7 100644 --- a/survey/default.nix +++ b/survey/default.nix @@ -1,6 +1,6 @@ let - cython-disable-tests-overlay = pkgs: final: previous: { - python27 = pkgs.python27.override { + cython-disable-tests-overlay = final: previous: { + python27 = previous.python27.override { packageOverrides = self: super: { cython = super.cython.overridePythonAttrs (old: rec { # TODO Remove once Cython tests are no longer flaky. See @@ -28,7 +28,7 @@ in # Note that we must NOT use something like `import normalPkgs.path {}`. # It is bad because it removes previous overlays. pkgs ? (normalPkgs.appendOverlays [ - (cython-disable-tests-overlay normalPkgs) + cython-disable-tests-overlay ])."${approach}", # When changing this, also change the default version of Cabal declared below @@ -432,6 +432,11 @@ let # directly to GHC. # Tip: If you want to debug this when it's failing, see # https://github.com/NixOS/nixpkgs/issues/65210#issuecomment-513515829 + # A common reason for it to fail is when the wrong `compiler` is given; + # in that case, the build log of the `Cabal` package involved will show + # two different ghc versions, and the output's `lib` directory will also + # contain 2 different ghc versions (one with the `.o` files and one with + # the `.conf` file). preCompileBuildDriver = '' cabalPackageId=$(basename --suffix=.conf ${fixedCabal}/lib/ghc-*/package.conf.d/*.conf) echo "Determined cabalPackageId as $cabalPackageId" @@ -445,9 +450,80 @@ let then static_package else throw "If you see this, nixpkgs #61682 has been fixed and ${name} should be overridden"; + # Takes a zlib derivation and overrides it to have both .a and .so files. + statify_zlib = zlib_drv: + (zlib_drv.override { + shared = true; + static = true; + splitStaticOutput = false; + }).overrideAttrs (old: { dontDisableStatic = true; }); + + # Takes a curl derivation and overrides it to have both .a and .so files, + # and have the `curl` executable be statically linked. + statify_curl_including_exe = curl_drv: + (curl_drv.override (old: { + # Disable gss support, because that requires `krb5`, which + # (as mentioned in note [krb5 can only be static XOR shared]) is a + # library that cannot build both .a and .so files in its build system. + # That means that if we enable it, we can no longer build the + # dynamically-linked `curl` binary from the overlay + # `archiveFilesOverlay` below where `statify_curl_including_exe` is used. + gssSupport = false; + zlib = statify_zlib old.zlib; + })).overrideAttrs (old: { + dontDisableStatic = true; + + # Additionally, flags to also build a static `curl` executable: + + # Note: It is important that in the eventual `libtool` invocation, + # `-all-static` comes before (or instead of) `-static`. + # This is because the first of them "wins setting the mode". + # See https://lists.gnu.org/archive/html/libtool/2006-12/msg00047.html + # libtool makes various problems with static linking. + # Some of them are is well-described by + # https://github.com/sabotage-linux/sabotage/commit/57a989a2e23c9e46501da1227f371da59d212ae4 + # However, so far, `-all-static` seems to have the same effect + # of convincing libtool to NOT drop the `-static` flag. + # Other places where this was dicussed (in case you have to debug this in + # the future) are: + # https://debbugs.gnu.org/cgi/bugreport.cgi?bug=11064 + # https://github.com/esnet/iperf/issues/632 + # Another common thing that people do is to pass `-static --static`, + # with the intent that `--static` isn't eaten by libtool but still + # accepted by e.g. gcc. In our case as of writing (nixpkgs commit bc94dcf50), + # this isn't enough. That is because: + # * The `--with-*=/path` options given to curl's `./configure` + # are usually `.lib` split outputs that contain only headers and + # pkg-config `.pc` files. OK so far. + # * For some of these, e.g. for libssh2, curl's `./configure` turns them + # into `LDFLAGS=-L/...libssh2-dev/lib`, which doesn't do anything to + # libtool, gcc or ld, because `*-dev/lib` contains only `lib/pkgconfig` + # and no libraries. + # * But for others, e.g. for libnghttp2, curl's `./configure` resolves + # them by taking the actual `-L` flags out of the `.pc` file, and turns + # them into e.g. `LDFLAGS=-L/...nghttp2-lib/lib`, which contains + # `{ *.la, *.a, *.so }`. + # * When libtool is invoked with such `LDFLAGS`, it adds two entries to + # `./lib/libcurl.la`'s `dependency_libs=`: `-L/...nghttp2-lib/lib` and + # `/...nghttp2-lib/lib/*.la`. + # When the `.la` path is given, libtool will read it, and pass the + # `.so` file referred to within as a positional argument to e.g. gcc, + # even when linking statically, which will result in linker error + # ld: attempted static link of dynamic object `/...-nghttp2-lib/lib/libnghttp2.so' + # I believe this is what + # https://github.com/sabotage-linux/sabotage/commit/57a989a2e23c9e46501da1227f371da59d212ae4 + # fixes. + # If we pass `-all-static` to libtool, it won't do the things in the last + # bullet point, causing static linking to succeed. + makeFlags = [ "curl_LDFLAGS=-all-static" ]; + }); + # Overlay that enables `.a` files for as many system packages as possible. # This is in *addition* to `.so` files. # See also https://github.com/NixOS/nixpkgs/issues/61575 + # TODO Instead of overriding each individual package manually, + # override them all at once similar to how `makeStaticLibraries` + # in `adapters.nix` does it (but without disabling shared). archiveFilesOverlay = final: previous: { libffi = previous.libffi.overrideAttrs (old: { dontDisableStatic = true; }); @@ -483,12 +559,7 @@ let patchelf_static = previous.patchelf.overrideAttrs (old: { dontDisableStatic = true; }); pcre_static = previous.pcre.overrideAttrs (old: { dontDisableStatic = true; }); xz_static = previous.xz.overrideAttrs (old: { dontDisableStatic = true; }); - zlib_static = (previous.zlib.override { - static = true; - shared = true; - # If both `static` and `dynamic` are enabled, then the zlib package - # puts the static libs into the `.static` split-output. - }).static; + zlib_both = statify_zlib previous.zlib; # Also override the original packages with a throw (which as of writing # has no effect) so we can know when the bug gets fixed in the future. acl = issue_61682_throw "acl" previous.acl; @@ -525,6 +596,15 @@ let enableSystemd = false; }; + pixman = previous.pixman.overrideAttrs (old: { dontDisableStatic = true; }); + fontconfig = previous.fontconfig.overrideAttrs (old: { + dontDisableStatic = true; + configureFlags = (old.configureFlags or []) ++ [ + "--enable-static" + ]; + }); + cairo = previous.cairo.overrideAttrs (old: { dontDisableStatic = true; }); + expat = previous.expat.overrideAttrs (old: { dontDisableStatic = true; }); mpfr = previous.mpfr.overrideAttrs (old: { dontDisableStatic = true; }); @@ -555,20 +635,62 @@ let openblas = previous.openblas.override { enableStatic = true; }; + openssl = previous.openssl.override { static = true; }; + krb5 = previous.krb5.override { - # Note krb5 does not support building both static and shared at the same time. + # Note [krb5 can only be static XOR shared] + # krb5 does not support building both static and shared at the same time. + # That means *anything* on top of this overlay trying to link krb5 + # dynamically from this overlay will fail with linker errors. staticOnly = true; }; - openssl = previous.openssl.override { static = true; }; + # See comments on `statify_curl_including_exe` for the interaction with krb5! + curl = statify_curl_including_exe previous.curl; + + # TODO: All of this can be removed once https://github.com/NixOS/nixpkgs/pull/66506 + # is available. + # Given that we override `krb5` (above) in this overlay so that it has + # static libs only, the `curl` used by `fetchurl` (e.g. via `fetchpatch`, + # which some packages may use) cannot be dynamically linked against it. + # Note this `curl` via `fetchurl` is NOT EXACTLY the same curl as our `curl` above + # in the overlay, but has a peculiarity: + # It forces `gssSupport = true` on Linux, undoing us setting it to `false` above! + # See https://github.com/NixOS/nixpkgs/blob/73493b2a2df75b487c6056e577b6cf3e6aa9fc91/pkgs/top-level/all-packages.nix#L295 + # So we have to turn it back off again here, *inside* `fetchurl`. + # Because `fetchurl` is a form of boostrap package, + # (which make ssense, as `curl`'s source code itself must be fetchurl'd), + # we can't just `fetchurl.override { curl = the_curl_from_the_overlay_above; }`; + # that would give an infinite evaluation loop. + # Instead, we have override the `curl` *after* `all-packages.nix` has force-set + # `gssSupport = false`. + # Other alternatives are to just use a statically linked `curl` binary for + # `fetchurl`, or to keep `gssSupport = true` and give it a `krb5` that has + # static libs switched off again. + # + # Note: This needs the commit from https://github.com/NixOS/nixpkgs/pull/66503 to work, + # which allows us to do `fetchurl.override`. + fetchurl = previous.fetchurl.override (old: { + curl = + # We have the 3 choices mentioned above: - # Disable gss support, because that requires `krb5`, - # which (as mentioned above) is a library that cannot build both - # .a and .so files in its build system. - # That means that if we enable it, we can no longer build the - # dynamically-linked `curl` binary from this overlay. - curl = (previous.curl.override { gssSupport = false; }).overrideAttrs (old: { - dontDisableStatic = true; + # 1) Turning `gssSupport` back off: + + (old.curl.override { gssSupport = false; }).overrideAttrs (old: { + makeFlags = builtins.filter (x: x != "curl_LDFLAGS=-all-static") (old.makeFlags or []); + }); + + # 2) Static `curl` binary: + + # statify_curl old.curl; + + # 3) Non-statick krb5: + + # (old.curl.override (old: { + # libkrb5 = old.libkrb5.override { staticOnly = false; }; + # })).overrideAttrs (old: { + # makeFlags = builtins.filter (x: x != "curl_LDFLAGS=-all-static") old.makeFlags; + # }); }); }; @@ -681,6 +803,12 @@ let # If we unconditionally add packages here, we will override # whatever packages they've passed us in. + # Override zlib Haskell package to use the system zlib package + # that has `.a` files added. + # This is because the system zlib package can't be overridden accordingly, + # see note [Packages that can't be overridden by overlays]. + zlib = super.zlib.override { zlib = final.zlib_both; }; + # `criterion`'s test suite fails with a timeout if its dependent # libraries (apparently `bytestring`) are compiled with `-O0`. # Even increasing the timeout 5x did not help! @@ -988,8 +1116,6 @@ let # obviously doesn' thave our patches. statify = drv: with final.haskell.lib; final.lib.foldl appendConfigureFlag (disableLibraryProfiling (disableSharedExecutables (useFixedCabal drv))) ([ "--enable-executable-static" # requires `useFixedCabal` - # TODO These probably shouldn't be here but only for packages that actually need them - "--extra-lib-dirs=${if approach == "pkgsMusl" then final.zlib_static else final.zlib}/lib" "--extra-lib-dirs=${final.ncurses.override { enableStatic = true; }}/lib" # TODO Figure out why this and the below libffi are necessary. # `working` and `workingStackageExecutables` don't seem to need that,