Sane cross-compiling through bootstrapping

doc/cross-compilation.xml

@@ -124,6 +124,11 @@
       Because of this, a best-of-both-worlds solution is in the works with no splicing or explicit access of <varname>buildPackages</varname> needed.
       For now, feel free to use either method.
     </para>
+    <note><para>
+      There is also a "backlink" <varname>__targetPackages</varname>, yielding a package set which whose <varname>buildPackages<varname> is the current package set.
+      This is a hack, though, to accommodate compilers with lousy build systems.
+      Please do not use this unless you are absolutely sure you are packaging such a compiler and there is no other way.
+    </para></note>
   </section>
 
 </section>

lib/lists.nix

@@ -60,6 +60,38 @@ rec {
   */
   foldl' = builtins.foldl' or foldl;
 
+  /* "Fold" a ternary function `op' between successive elements of `list' as if
+     it was a doubly-linked list with `lnul' and `rnul` base cases at either
+     end. In precise terms, `fold op lnul rnul [x_0 x_1 x_2 ... x_n-1]` is the
+     same as
+
+       let
+         f_-1  = lnul;
+         f_0   = op f_-1   x_0  f_1;
+         f_1   = op f_0    x_1  f_2;
+         f_2   = op f_1    x_2  f_3;
+         ...
+         f_n   = op f_n-1  x_n  f_n+1;
+         f_n+1 = rnul;
+       in
+         f_0
+
+     New in 17.03 --- Until that release is cut, feel free to modify this
+     function (and update its uses accordingly).
+  */
+  dfold = op: lnul: rnul: list:
+    let
+      len = length list;
+      go = pred: n:
+        if n == len
+        then rnul
+        else let
+          # Note the cycle -- call-by-need ensures finite fold.
+          cur  = op pred (elemAt list n) succ;
+          succ = go cur (n + 1);
+        in cur;
+    in go lnul 0;
+
   /* Map with index
 
      FIXME(zimbatm): why does this start to count at 1?

pkgs/build-support/build-fhs-userenv/env.nix

@@ -37,13 +37,13 @@ let
   # base packages of the chroot
   # these match the host's architecture, glibc_multi is used for multilib
   # builds.
-  basePkgs = with pkgs;
+  basePkgs = with (pkgs.buildPackages // pkgs);
     [ (if isMultiBuild then glibc_multi else glibc)
       (toString gcc.cc.lib) bashInteractive coreutils less shadow su
       gawk diffutils findutils gnused gnugrep
       gnutar gzip bzip2 xz glibcLocales
     ];
-  baseMultiPkgs = with pkgsi686Linux;
+  baseMultiPkgs = with (pkgsi686Linux.buildPackages // pkgsi686Linux);
     [ (toString gcc.cc.lib)
     ];
 

pkgs/build-support/cc-wrapper/default.nix

@@ -280,8 +280,8 @@ stdenv.mkDerivation {
     else "";
 
   crossAttrs = {
-    shell = shell.crossDrv + shell.crossDrv.shellPath;
-    libc = stdenv.ccCross.libc;
+    #shell = shell.crossDrv + shell.crossDrv.shellPath;
+    #libc = stdenv.ccCross.libc;
     #
     # This is not the best way to do this. I think the reference should be
     # the style in the gcc-cross-wrapper, but to keep a stable stdenv now I

pkgs/development/compilers/go/1.4.nix

@@ -1,12 +1,8 @@
-{ stdenv, lib, fetchurl, tzdata, iana_etc, libcCross
+{ stdenv, lib, fetchurl, tzdata, iana_etc, libc
 , pkgconfig
 , pcre
 , Security }:
 
-let
-  libc = if stdenv ? "cross" then libcCross else stdenv.cc.libc;
-in
-
 stdenv.mkDerivation rec {
   name = "go-${version}";
   version = "1.4-bootstrap-20161024";

pkgs/development/haskell-modules/default.nix

@@ -78,7 +78,8 @@ let
       };
 
     in
-      import ./hackage-packages.nix { inherit pkgs stdenv callPackage; } self // {
+      # TODO(@Ericson2314) think harder about build-time dependencies
+      import ./hackage-packages.nix { inherit stdenv callPackage; pkgs = pkgs.buildPackages // pkgs;  } self // {
 
         inherit mkDerivation callPackage;
 

pkgs/stdenv/adapters.nix

@@ -58,6 +58,10 @@ rec {
   # builds.
   makeStdenvCross = stdenv: cross: binutilsCross: gccCross: stdenv // {
 
+    # Overrides are surely not valid as packages built with this run on a
+    # different platform.
+    overrides = _: _: {};
+
     mkDerivation =
       { name ? "", buildInputs ? [], nativeBuildInputs ? []
       , propagatedBuildInputs ? [], propagatedNativeBuildInputs ? []

pkgs/stdenv/booter.nix

@@ -55,19 +55,21 @@ stageFuns: let
 
   # Adds the stdenv to the arguments, and sticks in it the previous stage for
   # debugging purposes.
-  folder = stageFun: finalSoFar: let
-    args = stageFun finalSoFar;
+  folder = nextStage: stageFun: prevStage: let
+    args = stageFun prevStage;
     args' = args // {
       stdenv = args.stdenv // {
         # For debugging
-        __bootPackages = finalSoFar;
+        __bootPackages = prevStage;
+        __hatPackages = nextStage;
       };
     };
   in
     if args.__raw or false
     then args'
     else allPackages ((builtins.removeAttrs args' ["selfBuild"]) // {
-      buildPackages = if args.selfBuild or true then null else finalSoFar;
+      buildPackages = if args.selfBuild or true then null else prevStage;
+      __targetPackages = if args.selfBuild or true then null else nextStage;
     });
 
-in lib.lists.fold folder {} withAllowCustomOverrides
+in lib.lists.dfold folder {} {} withAllowCustomOverrides

pkgs/stdenv/cross/default.nix

@@ -21,9 +21,7 @@ in bootStages ++ [
     selfBuild = false;
     # It's OK to change the built-time dependencies
     allowCustomOverrides = true;
-    stdenv = vanillaPackages.stdenv // {
-      overrides = _: _: {};
-    };
+    inherit (vanillaPackages) stdenv;
   })
 
   # Run Packages

pkgs/stdenv/darwin/default.nix

@@ -253,7 +253,7 @@ in rec {
     inherit
       gnumake gzip gnused bzip2 gawk ed xz patch bash
       libcxxabi libcxx ncurses libffi zlib icu llvm gmp pcre gnugrep
-      coreutils findutils diffutils patchutils binutils binutils-raw;
+      coreutils findutils diffutils patchutils;
 
     llvmPackages = super.llvmPackages // {
       inherit (llvmPackages) llvm clang-unwrapped;
@@ -262,6 +262,9 @@ in rec {
     darwin = super.darwin // {
       inherit (darwin) dyld Libsystem cctools libiconv;
     };
+  } // lib.optionalAttrs (super.targetPlatform == localSystem) {
+    # Need to get rid of these when cross-compiling.
+    inherit binutils binutils-raw;
   };
 
   stdenvDarwin = prevStage: let pkgs = prevStage; in import ../generic rec {

pkgs/stdenv/linux/default.nix

@@ -232,7 +232,8 @@ in
       # other purposes (binutils and top-level pkgs) too.
       inherit (prevStage) gettext gnum4 bison gmp perl glibc zlib linuxHeaders;
 
-      gcc = lib.makeOverridable (import ../../build-support/cc-wrapper) {
+      # Can't be called plain `gcc` or will be clobbered by build wrappers
+      gcc' = lib.makeOverridable (import ../../build-support/cc-wrapper) {
         nativeTools = false;
         nativeLibc = false;
         isGNU = true;
@@ -277,7 +278,7 @@ in
         # Many tarballs come with obsolete config.sub/config.guess that don't recognize aarch64.
         lib.optional (system == "aarch64-linux") prevStage.updateAutotoolsGnuConfigScriptsHook;
 
-      cc = prevStage.gcc;
+      cc = prevStage.gcc';
 
       shell = cc.shell;
 
@@ -298,12 +299,13 @@ in
         */
 
       overrides = self: super: {
-        gcc = cc;
-
         inherit (prevStage)
-          gzip bzip2 xz bash binutils coreutils diffutils findutils gawk
+          gzip bzip2 xz bash coreutils diffutils findutils gawk
           glibc gnumake gnused gnutar gnugrep gnupatch patchelf
           attr acl paxctl zlib pcre;
+      } // lib.optionalAttrs (super.targetPlatform == localSystem) {
+        # Need to get rid of these when cross-compiling.
+        inherit (prevStage) binutils gcc-unwrapped;
       };
     };
   })

pkgs/stdenv/linux/make-bootstrap-tools.nix

@@ -94,24 +94,24 @@ rec {
         cp -d ${gnugrep.pcre.out}/lib/libpcre*.so* $out/lib # needed by grep
 
         # Copy what we need of GCC.
-        cp -d ${gcc.cc.out}/bin/gcc $out/bin
-        cp -d ${gcc.cc.out}/bin/cpp $out/bin
-        cp -d ${gcc.cc.out}/bin/g++ $out/bin
-        cp -d ${gcc.cc.lib}/lib*/libgcc_s.so* $out/lib
-        cp -d ${gcc.cc.lib}/lib*/libstdc++.so* $out/lib
-        cp -rd ${gcc.cc.out}/lib/gcc $out/lib
+        cp -d ${gcc-unwrapped.out}/bin/gcc $out/bin
+        cp -d ${gcc-unwrapped.out}/bin/cpp $out/bin
+        cp -d ${gcc-unwrapped.out}/bin/g++ $out/bin
+        cp -d ${gcc-unwrapped.lib}/lib*/libgcc_s.so* $out/lib
+        cp -d ${gcc-unwrapped.lib}/lib*/libstdc++.so* $out/lib
+        cp -rd ${gcc-unwrapped.out}/lib/gcc $out/lib
         chmod -R u+w $out/lib
         rm -f $out/lib/gcc/*/*/include*/linux
         rm -f $out/lib/gcc/*/*/include*/sound
         rm -rf $out/lib/gcc/*/*/include*/root
         rm -f $out/lib/gcc/*/*/include-fixed/asm
         rm -rf $out/lib/gcc/*/*/plugin
         #rm -f $out/lib/gcc/*/*/*.a
-        cp -rd ${gcc.cc.out}/libexec/* $out/libexec
+        cp -rd ${gcc-unwrapped.out}/libexec/* $out/libexec
         chmod -R u+w $out/libexec
         rm -rf $out/libexec/gcc/*/*/plugin
         mkdir $out/include
-        cp -rd ${gcc.cc.out}/include/c++ $out/include
+        cp -rd ${gcc-unwrapped.out}/include/c++ $out/include
         chmod -R u+w $out/include
         rm -rf $out/include/c++/*/ext/pb_ds
         rm -rf $out/include/c++/*/ext/parallel

pkgs/stdenv/nix/default.nix

@@ -43,10 +43,12 @@ bootStages ++ [
 
       overrides = self: super: {
         inherit cc;
-        inherit (cc) binutils;
         inherit (prevStage)
           gzip bzip2 xz bash coreutils diffutils findutils gawk
           gnumake gnused gnutar gnugrep gnupatch perl;
+      } // lib.optionalAttrs (super.targetPlatform == localSystem) {
+        # Need to get rid of these when cross-compiling.
+        inherit (prevStage) binutils;
       };
     };
   })