nixpkgs: Update submodule to latest master

.buildkite/pipeline.yml

@@ -51,3 +51,12 @@ steps:
     command: |
       NIX_PATH=nixpkgs=nixpkgs nix-build --no-link survey/default.nix \
         --argstr compiler ghc865 -A workingStackageExecutables
+
+  # Stack via stack2nix
+
+  - label: static-stack
+    command: |
+      cd static-stack/
+      mkdir -p static-stack-test-dir
+      curl -L https://github.com/commercialhaskell/stack/archive/v2.1.3.tar.gz | tar -xz -C static-stack-test-dir
+      $(nix-build --no-link -A fullBuildScript --argstr stackDir $PWD/static-stack-test-dir/stack-*)

.in-static-haskell-nix

@@ -0,0 +1 @@
+The presence of this file indicates that we're inside the static-haskell-nix project.

default.nix

@@ -1,6 +1,6 @@
 # Note: This is just a minimal example. For proper usage, see the README.
 
-{ nixpkgs ? (import <nixpkgs> {}).pkgsMusl, compiler ? "ghc864", strip ? true }:
+{ nixpkgs ? (import ./nixpkgs.nix).pkgsMusl, compiler ? "ghc864", strip ? true }:
 
 
 let

nixpkgs.nix

@@ -0,0 +1,10 @@
+# If a `./nixpkgs` submodule exists, use that.
+# Note that this will take precedence over setting NIX_PATH!
+# We prefer this such that `static-stack2nix-builder` and specifically
+# `static-stack2nix-builder-example` can just import `nixpkgs.nix`
+# in CI and when called during development to get the right version of
+# nixpkgs.
+if builtins.pathExists ./nixpkgs/pkgs
+  then import ./nixpkgs {}
+  # Pinned nixpkgs version; should be kept up-to-date with our submodule.
+  else import (fetchTarball https://github.com/nh2/nixpkgs/archive/b577340eb5bc3b72549f0544b50e2e37df78bf12.tar.gz) {}

static-stack/default.nix

@@ -16,7 +16,7 @@ let
   stack2nix-script = import ../static-stack2nix-builder/stack2nix-script.nix {
     pkgs = pkgs;
     stack-project-dir = stackDir; # where stack.yaml is
-    hackageSnapshot = "2019-05-08T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions
+    hackageSnapshot = "2019-08-17T00:00:00Z"; # pins e.g. extra-deps without hashes or revisions
   };
 
   static-stack2nix-builder = import ../static-stack2nix-builder/default.nix {

static-stack2nix-builder-example/default.nix

@@ -8,11 +8,18 @@ let
   cabalPackageName = "example-project";
   compiler = "ghc864"; # matching stack.yaml
 
-  # Pin nixpkgs version.
-  pkgs = import (fetchTarball https://github.com/nh2/nixpkgs/archive/a2d7e9b875e8ba7fd15b989cf2d80be4e183dc72.tar.gz) {};
-
   # Pin static-haskell-nix version.
-  static-haskell-nix = fetchTarball https://github.com/nh2/static-haskell-nix/archive/1d37d9a83e570eceef9c7dad5c89557f8179a076.tar.gz;
+  static-haskell-nix =
+    if builtins.pathExists ../.in-static-haskell-nix
+      then toString ../. # for the case that we're in static-haskell-nix itself, so that CI always builds the latest version.
+      # Update this hash to use a different `static-haskell-nix` version:
+      else fetchTarball https://github.com/nh2/static-haskell-nix/archive/4521a448dda7a613e29d3bcd8e725c96202c5728.tar.gz;
+
+  # Pin nixpkgs version
+  # By default to the one `static-haskell-nix` provides, but you may also give
+  # your own as long as it has the necessary patches, using e.g.
+  #     pkgs = import (fetchTarball https://github.com/nh2/nixpkgs/archive/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa123.tar.gz) {};
+  pkgs = import "${static-haskell-nix}/nixpkgs.nix";
 
   stack2nix-script = import "${static-haskell-nix}/static-stack2nix-builder/stack2nix-script.nix" {
     inherit pkgs;
@@ -31,6 +38,7 @@ let
     #!/usr/bin/env bash
     set -eu -o pipefail
     STACK2NIX_OUTPUT_PATH=$(${stack2nix-script})
+    export NIX_PATH=nixpkgs=${pkgs.path}
     ${pkgs.nix}/bin/nix-build --no-link -A static_package --argstr stack2nix-output-path "$STACK2NIX_OUTPUT_PATH" "$@"
   '';
 

static-stack2nix-builder/stack2nix-script.nix

@@ -48,6 +48,34 @@
       "${pkgs.nix}/bin" # various `nix-*` commands
       "${pkgs.wget}/bin" # `wget`
     ];
+
+    fixed_stack2nix =
+      let
+        # stack2nix isn't compatible with Stack >= 2.0, see
+        # https://github.com/input-output-hk/stack2nix/issues/168.
+        # Current versions of nixpkgs master have Stack >= 2.0, see
+        # https://github.com/NixOS/nixpkgs/issues/63691.
+        # We thus fetch the `stack2nix` binary from an older nixpkgs version
+        # that doesn't have Stack >= 2.0.
+        # This means that `static-stack2nix-builder` may not work on `stack.yaml`
+        # files that aren't compatible with Stack < 2.0.
+        stack2nix_pkgs = import (fetchTarball https://github.com/NixOS/nixpkgs/archive/e36f91fa86109fa93cac2516a9365af57233a3a6.tar.gz) {};
+      in
+        # Some older stack2nix versions have fundamental problems that prevent
+        # stack2nix from running correctly. Fix them here, until these old versions
+        # are faded out of current nixpkgs. Especially:
+        #   * "Make sure output is written in UTF-8."
+        #     https://github.com/input-output-hk/stack2nix/commit/cb05818ef8b58899f15641f50cb04e5473b4f9b0
+        #
+        # Versions < 0.2.3 aren't supported, force-upgrade them to 0.2.3.
+        if stack2nix_pkgs.lib.versionOlder stack2nix_pkgs.stack2nix.version "0.2.3"
+          then stack2nix_pkgs.haskellPackages.callCabal2nix "stack2nix" (stack2nix_pkgs.fetchFromGitHub {
+            owner = "input-output-hk";
+            repo = "stack2nix";
+            rev = "v0.2.3";
+            sha256 = "1b4g7800hvhr97cjssy5ffd097n2z0fvk9cm31a5jh66pkxys0mq";
+          }) {}
+          else stack2nix_pkgs.stack2nix;
   in
   pkgs.writeScript "stack2nix-build-script.sh" ''
     #!/usr/bin/env bash
@@ -56,6 +84,6 @@
     export PATH=${pkgs.lib.concatStringsSep ":" add_to_PATH}:$PATH
     OUT_DIR=$(mktemp --directory -t stack2nix-output-dir.XXXXXXXXXX)
     set -x
-    ${pkgs.stack2nix}/bin/stack2nix "${stack-project-dir}" --stack-yaml "${stack-yaml}" --hackage-snapshot "${hackageSnapshot}" -o "$OUT_DIR/stack2nix-output.nix" "$@" 1>&2
+    ${fixed_stack2nix}/bin/stack2nix "${stack-project-dir}" --stack-yaml "${stack-yaml}" --hackage-snapshot "${hackageSnapshot}" -o "$OUT_DIR/stack2nix-output.nix" "$@" 1>&2
     nix-store --add "$OUT_DIR/stack2nix-output.nix"
   ''

survey/default.nix

@@ -1,6 +1,6 @@
 let
-  cython-disable-tests-overlay = pkgs: final: previous: {
-    python27 = pkgs.python27.override {
+  cython-disable-tests-overlay = final: previous: {
+    python27 = previous.python27.override {
       packageOverrides = self: super: {
         cython = super.cython.overridePythonAttrs (old: rec {
           # TODO Remove once Cython tests are no longer flaky. See
@@ -28,7 +28,7 @@ in
   # Note that we must NOT use something like `import normalPkgs.path {}`.
   # It is bad because it removes previous overlays.
   pkgs ? (normalPkgs.appendOverlays [
-    (cython-disable-tests-overlay normalPkgs)
+    cython-disable-tests-overlay
   ])."${approach}",
 
   # When changing this, also change the default version of Cabal declared below
@@ -432,6 +432,11 @@ let
         # directly to GHC.
         # Tip: If you want to debug this when it's failing, see
         #      https://github.com/NixOS/nixpkgs/issues/65210#issuecomment-513515829
+        # A common reason for it to fail is when the wrong `compiler` is given;
+        # in that case, the build log of the `Cabal` package involved will show
+        # two different ghc versions, and the output's `lib` directory will also
+        # contain 2 different ghc versions (one with the `.o` files and one with
+        # the `.conf` file).
         preCompileBuildDriver = ''
           cabalPackageId=$(basename --suffix=.conf ${fixedCabal}/lib/ghc-*/package.conf.d/*.conf)
           echo "Determined cabalPackageId as $cabalPackageId"
@@ -445,9 +450,80 @@ let
       then static_package
       else throw "If you see this, nixpkgs #61682 has been fixed and ${name} should be overridden";
 
+  # Takes a zlib derivation and overrides it to have both .a and .so files.
+  statify_zlib = zlib_drv:
+    (zlib_drv.override {
+      shared = true;
+      static = true;
+      splitStaticOutput = false;
+    }).overrideAttrs (old: { dontDisableStatic = true; });
+
+  # Takes a curl derivation and overrides it to have both .a and .so files,
+  # and have the `curl` executable be statically linked.
+  statify_curl_including_exe = curl_drv:
+    (curl_drv.override (old: {
+      # Disable gss support, because that requires `krb5`, which
+      # (as mentioned in note [krb5 can only be static XOR shared]) is a
+      # library that cannot build both .a and .so files in its build system.
+      # That means that if we enable it, we can no longer build the
+      # dynamically-linked `curl` binary from the overlay
+      # `archiveFilesOverlay` below where `statify_curl_including_exe` is used.
+      gssSupport = false;
+      zlib = statify_zlib old.zlib;
+    })).overrideAttrs (old: {
+      dontDisableStatic = true;
+
+      # Additionally, flags to also build a static `curl` executable:
+
+      # Note: It is important that in the eventual `libtool` invocation,
+      # `-all-static` comes before (or instead of) `-static`.
+      # This is because the first of them "wins setting the mode".
+      # See https://lists.gnu.org/archive/html/libtool/2006-12/msg00047.html
+      # libtool makes various problems with static linking.
+      # Some of them are is well-described by
+      #   https://github.com/sabotage-linux/sabotage/commit/57a989a2e23c9e46501da1227f371da59d212ae4
+      # However, so far, `-all-static` seems to have the same effect
+      # of convincing libtool to NOT drop the `-static` flag.
+      # Other places where this was dicussed (in case you have to debug this in
+      # the future) are:
+      #   https://debbugs.gnu.org/cgi/bugreport.cgi?bug=11064
+      #   https://github.com/esnet/iperf/issues/632
+      # Another common thing that people do is to pass `-static --static`,
+      # with the intent that `--static` isn't eaten by libtool but still
+      # accepted by e.g. gcc. In our case as of writing (nixpkgs commit bc94dcf50),
+      # this isn't enough. That is because:
+      #   * The `--with-*=/path` options given to curl's `./configure`
+      #     are usually `.lib` split outputs that contain only headers and
+      #     pkg-config `.pc` files. OK so far.
+      #   * For some of these, e.g. for libssh2, curl's `./configure` turns them
+      #     into `LDFLAGS=-L/...libssh2-dev/lib`, which doesn't do anything to
+      #     libtool, gcc or ld, because `*-dev/lib` contains only `lib/pkgconfig`
+      #     and no libraries.
+      #   * But for others, e.g. for libnghttp2, curl's `./configure` resolves
+      #     them by taking the actual `-L` flags out of the `.pc` file, and turns
+      #     them into e.g. `LDFLAGS=-L/...nghttp2-lib/lib`, which contains
+      #     `{ *.la, *.a, *.so }`.
+      #   * When libtool is invoked with such `LDFLAGS`, it adds two entries to
+      #     `./lib/libcurl.la`'s `dependency_libs=`: `-L/...nghttp2-lib/lib` and
+      #     `/...nghttp2-lib/lib/*.la`.
+      #     When the `.la` path is given, libtool will read it, and pass the
+      #     `.so` file referred to within as a positional argument to e.g. gcc,
+      #     even when linking statically, which will result in linker error
+      #         ld: attempted static link of dynamic object `/...-nghttp2-lib/lib/libnghttp2.so'
+      #     I believe this is what
+      #         https://github.com/sabotage-linux/sabotage/commit/57a989a2e23c9e46501da1227f371da59d212ae4
+      #     fixes.
+      # If we pass `-all-static` to libtool, it won't do the things in the last
+      # bullet point, causing static linking to succeed.
+      makeFlags = [ "curl_LDFLAGS=-all-static" ];
+    });
+
   # Overlay that enables `.a` files for as many system packages as possible.
   # This is in *addition* to `.so` files.
   # See also https://github.com/NixOS/nixpkgs/issues/61575
+  # TODO Instead of overriding each individual package manually,
+  #      override them all at once similar to how `makeStaticLibraries`
+  #      in `adapters.nix` does it (but without disabling shared).
   archiveFilesOverlay = final: previous: {
 
     libffi = previous.libffi.overrideAttrs (old: { dontDisableStatic = true; });
@@ -483,12 +559,7 @@ let
     patchelf_static = previous.patchelf.overrideAttrs (old: { dontDisableStatic = true; });
     pcre_static = previous.pcre.overrideAttrs (old: { dontDisableStatic = true; });
     xz_static = previous.xz.overrideAttrs (old: { dontDisableStatic = true; });
-    zlib_static = (previous.zlib.override {
-      static = true;
-      shared = true;
-      # If both `static` and `dynamic` are enabled, then the zlib package
-      # puts the static libs into the `.static` split-output.
-    }).static;
+    zlib_both = statify_zlib previous.zlib;
     # Also override the original packages with a throw (which as of writing
     # has no effect) so we can know when the bug gets fixed in the future.
     acl = issue_61682_throw "acl" previous.acl;
@@ -525,6 +596,15 @@ let
       enableSystemd = false;
     };
 
+    pixman = previous.pixman.overrideAttrs (old: { dontDisableStatic = true; });
+    fontconfig = previous.fontconfig.overrideAttrs (old: {
+      dontDisableStatic = true;
+      configureFlags = (old.configureFlags or []) ++ [
+        "--enable-static"
+      ];
+    });
+    cairo = previous.cairo.overrideAttrs (old: { dontDisableStatic = true; });
+
     expat = previous.expat.overrideAttrs (old: { dontDisableStatic = true; });
 
     mpfr = previous.mpfr.overrideAttrs (old: { dontDisableStatic = true; });
@@ -555,20 +635,62 @@ let
 
     openblas = previous.openblas.override { enableStatic = true; };
 
+    openssl = previous.openssl.override { static = true; };
+
     krb5 = previous.krb5.override {
-      # Note krb5 does not support building both static and shared at the same time.
+      # Note [krb5 can only be static XOR shared]
+      # krb5 does not support building both static and shared at the same time.
+      # That means *anything* on top of this overlay trying to link krb5
+      # dynamically from this overlay will fail with linker errors.
       staticOnly = true;
     };
 
-    openssl = previous.openssl.override { static = true; };
+    # See comments on `statify_curl_including_exe` for the interaction with krb5!
+    curl = statify_curl_including_exe previous.curl;
+
+    # TODO: All of this can be removed once https://github.com/NixOS/nixpkgs/pull/66506
+    #       is available.
+    # Given that we override `krb5` (above) in this overlay so that it has
+    # static libs only, the `curl` used by `fetchurl` (e.g. via `fetchpatch`,
+    # which some packages may use) cannot be dynamically linked against it.
+    # Note this `curl` via `fetchurl` is NOT EXACTLY the same curl as our `curl` above
+    # in the overlay, but has a peculiarity:
+    # It forces `gssSupport = true` on Linux, undoing us setting it to `false` above!
+    # See https://github.com/NixOS/nixpkgs/blob/73493b2a2df75b487c6056e577b6cf3e6aa9fc91/pkgs/top-level/all-packages.nix#L295
+    # So we have to turn it back off again here, *inside* `fetchurl`.
+    # Because `fetchurl` is a form of boostrap package,
+    # (which make ssense, as `curl`'s source code itself must be fetchurl'd),
+    # we can't just `fetchurl.override { curl = the_curl_from_the_overlay_above; }`;
+    # that would give an infinite evaluation loop.
+    # Instead, we have override the `curl` *after* `all-packages.nix` has force-set
+    # `gssSupport = false`.
+    # Other alternatives are to just use a statically linked `curl` binary for
+    # `fetchurl`, or to keep `gssSupport = true` and give it a `krb5` that has
+    # static libs switched off again.
+    #
+    # Note: This needs the commit from https://github.com/NixOS/nixpkgs/pull/66503 to work,
+    # which allows us to do `fetchurl.override`.
+    fetchurl = previous.fetchurl.override (old: {
+      curl =
+        # We have the 3 choices mentioned above:
 
-    # Disable gss support, because that requires `krb5`,
-    # which (as mentioned above) is a library that cannot build both
-    # .a and .so files in its build system.
-    # That means that if we enable it, we can no longer build the
-    # dynamically-linked `curl` binary from this overlay.
-    curl = (previous.curl.override { gssSupport = false; }).overrideAttrs (old: {
-      dontDisableStatic = true;
+        # 1) Turning `gssSupport` back off:
+
+        (old.curl.override { gssSupport = false; }).overrideAttrs (old: {
+          makeFlags = builtins.filter (x: x != "curl_LDFLAGS=-all-static") (old.makeFlags or []);
+        });
+
+        # 2) Static `curl` binary:
+
+        # statify_curl old.curl;
+
+        # 3) Non-statick krb5:
+
+        # (old.curl.override (old: {
+        #   libkrb5 = old.libkrb5.override { staticOnly = false; };
+        # })).overrideAttrs (old: {
+        #   makeFlags = builtins.filter (x: x != "curl_LDFLAGS=-all-static") old.makeFlags;
+        # });
     });
   };
 
@@ -681,6 +803,12 @@ let
               # If we unconditionally add packages here, we will override
               # whatever packages they've passed us in.
 
+              # Override zlib Haskell package to use the system zlib package
+              # that has `.a` files added.
+              # This is because the system zlib package can't be overridden accordingly,
+              # see note [Packages that can't be overridden by overlays].
+              zlib = super.zlib.override { zlib = final.zlib_both; };
+
               # `criterion`'s test suite fails with a timeout if its dependent
               # libraries (apparently `bytestring`) are compiled with `-O0`.
               # Even increasing the timeout 5x did not help!
@@ -988,8 +1116,6 @@ let
           # obviously doesn' thave our patches.
           statify = drv: with final.haskell.lib; final.lib.foldl appendConfigureFlag (disableLibraryProfiling (disableSharedExecutables (useFixedCabal drv))) ([
             "--enable-executable-static" # requires `useFixedCabal`
-            # TODO These probably shouldn't be here but only for packages that actually need them
-            "--extra-lib-dirs=${if approach == "pkgsMusl" then final.zlib_static else final.zlib}/lib"
             "--extra-lib-dirs=${final.ncurses.override { enableStatic = true; }}/lib"
           # TODO Figure out why this and the below libffi are necessary.
           #      `working` and `workingStackageExecutables` don't seem to need that,