wrapCC, wrapBintools: correctly cross-compile if buildPlatform != hostPlatform

pkgs/build-support/bintools-wrapper/default.nix

@@ -8,7 +8,8 @@
 { name ? ""
 , lib
 , stdenvNoCC
-, bintools ? null, libc ? null, coreutils ? null, shell ? stdenvNoCC.shell, gnugrep ? null
+, runtimeShell
+, bintools ? null, libc ? null, coreutils ? null, gnugrep ? null
 , netbsd ? null, netbsdCross ? null
 , sharedLibraryLoader ?
   if libc == null then
@@ -28,7 +29,7 @@
 , isGNU ? bintools.isGNU or false
 , isLLVM ? bintools.isLLVM or false
 , isCCTools ? bintools.isCCTools or false
-, buildPackages ? {}
+, expand-response-params
 , targetPackages ? {}
 , useMacosReexportHack ? false
 , wrapGas ? false
@@ -131,10 +132,6 @@ let
     else if hasSuffix "pc-gnu" targetPlatform.config then "ld.so.1"
     else "";
 
-  expand-response-params =
-    optionalString (buildPackages ? stdenv && buildPackages.stdenv.hasCC && buildPackages.stdenv.cc != "/dev/null")
-    (import ../expand-response-params { inherit (buildPackages) stdenv; });
-
 in
 
 stdenvNoCC.mkDerivation {
@@ -418,8 +415,10 @@ stdenvNoCC.mkDerivation {
 
   env = {
     # for substitution in utils.bash
+    # TODO(@sternenseemann): invent something cleaner than passing in "" in case of absence
     expandResponseParams = "${expand-response-params}/bin/expand-response-params";
-    shell = getBin shell + shell.shellPath or "";
+    # TODO(@sternenseemann): rename env var via stdenv rebuild
+    shell = (getBin runtimeShell + runtimeShell.shellPath or "");
     gnugrep_bin = optionalString (!nativeTools) gnugrep;
     wrapperName = "BINTOOLS_WRAPPER";
     inherit dynamicLinker targetPrefix suffixSalt coreutils_bin;

pkgs/build-support/cc-wrapper/default.nix

@@ -8,14 +8,15 @@
 { name ? ""
 , lib
 , stdenvNoCC
-, cc ? null, libc ? null, bintools, coreutils ? null, shell ? stdenvNoCC.shell
+, runtimeShell
+, cc ? null, libc ? null, bintools, coreutils ? null
 , zlib ? null
 , nativeTools, noLibc ? false, nativeLibc, nativePrefix ? ""
 , propagateDoc ? cc != null && cc ? man
 , extraTools ? [], extraPackages ? [], extraBuildCommands ? ""
 , nixSupport ? {}
 , isGNU ? false, isClang ? cc.isClang or false, isCcache ? cc.isCcache or false, gnugrep ? null
-, buildPackages ? {}
+, expand-response-params
 , libcxx ? null
 
 # Whether or not to add `-B` and `-L` to `nix-support/cc-{c,ld}flags`
@@ -112,9 +113,6 @@ let
   # unstable implementation detail, however.
   suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config;
 
-  expand-response-params =
-    optionalString ((buildPackages.stdenv.hasCC or false) && buildPackages.stdenv.cc != "/dev/null") (import ../expand-response-params { inherit (buildPackages) stdenv; });
-
   useGccForLibs = useCcForLibs
     && libcxx == null
     && !targetPlatform.isDarwin
@@ -297,6 +295,9 @@ stdenvNoCC.mkDerivation {
         '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)}))
     '';
 
+    # Expose expand-response-params we are /actually/ using. In stdenv
+    # bootstrapping, expand-response-params usually comes from an earlier stage,
+    # so it is important to expose this for reference checking.
     inherit expand-response-params;
 
     inherit nixSupport;
@@ -738,8 +739,10 @@ stdenvNoCC.mkDerivation {
     inherit isClang;
 
     # for substitution in utils.bash
+    # TODO(@sternenseemann): invent something cleaner than passing in "" in case of absence
     expandResponseParams = "${expand-response-params}/bin/expand-response-params";
-    shell = getBin shell + shell.shellPath or "";
+    # TODO(@sternenseemann): rename env var via stdenv rebuild
+    shell = getBin runtimeShell + runtimeShell.shellPath or "";
     gnugrep_bin = optionalString (!nativeTools) gnugrep;
     # stdenv.cc.cc should not be null and we have nothing better for now.
     # if the native impure bootstrap is gotten rid of this can become `inherit cc;` again.

pkgs/build-support/expand-response-params/default.nix

@@ -1,4 +1,4 @@
-{ stdenv }:
+{ stdenv, lib }:
 
 # A "response file" is a sequence of arguments that is passed via a
 # file, rather than via argv[].
@@ -25,4 +25,18 @@ stdenv.mkDerivation {
     mkdir -p $prefix/bin
     mv expand-response-params $prefix/bin/
   '';
+
+  meta = {
+    description = "Internal tool used by the nixpkgs wrapper scripts for processing response files";
+    longDescription = ''
+      expand-response-params is a tool that allows for obtaining a full list of all
+      arguments passed in a given compiler command line including those passed via
+      so-called response files. The nixpkgs wrapper scripts for bintools and C
+      compilers use it for processing compiler flags. As it is developed in
+      conjunction with the nixpkgs wrapper scripts, it should be considered as
+      unstable and subject to change.
+    '';
+    license = lib.licenses.mit;
+    platforms = lib.platforms.all;
+  };
 }

pkgs/stdenv/darwin/default.nix

@@ -76,9 +76,9 @@ let
         nativeTools = false;
         nativeLibc = false;
 
-        buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
-          inherit (prevStage) stdenv;
-        };
+        expand-response-params = lib.optionalString
+          (prevStage.stdenv.hasCC or false && prevStage.stdenv.cc != "/dev/null")
+          prevStage.expand-response-params;
 
         extraPackages = [
           prevStage.llvmPackages.compiler-rt
@@ -117,6 +117,7 @@ let
         inherit (prevStage) coreutils gnugrep;
 
         stdenvNoCC = prevStage.ccWrapperStdenv;
+        runtimeShell = prevStage.ccWrapperStdenv.shell;
       };
 
       bash = prevStage.bash or bootstrapTools;
@@ -253,11 +254,12 @@ in
           nativeTools = false;
           nativeLibc = false;
 
-          buildPackages = { };
+          expand-response-params = "";
           libc = selfDarwin.Libsystem;
 
           inherit lib;
           inherit (self) stdenvNoCC coreutils gnugrep;
+          runtimeShell = self.stdenvNoCC.shell;
 
           bintools = selfDarwin.binutils-unwrapped;
 
@@ -457,6 +459,8 @@ in
 
           bintools = selfDarwin.binutils-unwrapped;
           libc = selfDarwin.Libsystem;
+          # TODO(@sternenseemann): can this be removed?
+          runtimeShell = "${bootstrapTools}/bin/bash";
         };
 
         binutils-unwrapped = superDarwin.binutils-unwrapped.override {
@@ -837,9 +841,7 @@ in
 
         # Rewrap binutils so it uses the rebuilt Libsystem.
         binutils = superDarwin.binutils.override {
-          buildPackages = {
-            inherit (prevStage) stdenv;
-          };
+          inherit (prevStage) expand-response-params;
           libc = selfDarwin.Libsystem;
         } // {
           passthru = { inherit (prevStage.bintools.passthru) isFromBootstrapFiles; };
@@ -1044,11 +1046,7 @@ in
         };
 
         binutils = superDarwin.binutils.override {
-          shell = self.bash + "/bin/bash";
-
-          buildPackages = {
-            inherit (prevStage) stdenv;
-          };
+          inherit (prevStage) expand-response-params;
 
           bintools = selfDarwin.binutils-unwrapped;
           libc = selfDarwin.Libsystem;
@@ -1085,9 +1083,7 @@ in
               nativeTools = false;
               nativeLibc = false;
 
-              buildPackages = {
-                inherit (prevStage) stdenv;
-              };
+              inherit (prevStage) expand-response-params;
 
               extraPackages = [
                 self.llvmPackages.compiler-rt
@@ -1124,9 +1120,7 @@ in
               inherit (self.llvmPackages) libcxx;
 
               inherit lib;
-              inherit (self) stdenvNoCC coreutils gnugrep;
-
-              shell = self.bash + "/bin/bash";
+              inherit (self) stdenvNoCC coreutils gnugrep runtimeShell;
             };
           });
           libraries = super.llvmPackages.libraries.extend (_: _:{

pkgs/stdenv/linux/default.nix

@@ -185,9 +185,9 @@ let
           name = "${name}-gcc-wrapper";
           nativeTools = false;
           nativeLibc = false;
-          buildPackages = lib.optionalAttrs (prevStage ? stdenv) {
-            inherit (prevStage) stdenv;
-          };
+          expand-response-params = lib.optionalString
+            (prevStage.stdenv.hasCC or false && prevStage.stdenv.cc != "/dev/null")
+            prevStage.expand-response-params;
           cc = prevStage.gcc-unwrapped;
           bintools = prevStage.binutils;
           isGNU = true;
@@ -196,6 +196,7 @@ let
           inherit (prevStage) coreutils gnugrep;
           stdenvNoCC = prevStage.ccWrapperStdenv;
           fortify-headers = prevStage.fortify-headers;
+          runtimeShell = prevStage.ccWrapperStdenv.shell;
         }).overrideAttrs(a: lib.optionalAttrs (prevStage.gcc-unwrapped.passthru.isXgcc or false) {
           # This affects only `xgcc` (the compiler which compiles the final compiler).
           postFixup = (a.postFixup or "") + ''
@@ -260,11 +261,12 @@ in
         name = "bootstrap-stage0-binutils-wrapper";
         nativeTools = false;
         nativeLibc = false;
-        buildPackages = { };
+        expand-response-params = "";
         libc = getLibc self;
         inherit lib;
         inherit (self) stdenvNoCC coreutils gnugrep;
         bintools = bootstrapTools;
+        runtimeShell = "${bootstrapTools}/bin/bash";
       };
       coreutils = bootstrapTools;
       gnugrep = bootstrapTools;
@@ -332,6 +334,14 @@ in
         inherit (prevStage) ccWrapperStdenv coreutils gnugrep gettext bison texinfo zlib gnum4 perl patchelf;
         ${localSystem.libc} = getLibc prevStage;
         gmp = super.gmp.override { cxx = false; };
+        # This stage also rebuilds binutils which will of course be used only in the next stage.
+        # We inherit this until stage3, in stage4 it will be rebuilt using the adjacent bash/runtimeShell pkg.
+        # TODO(@sternenseemann): Can we already build the wrapper with the actual runtimeShell here?
+        # Historically, the wrapper didn't use runtimeShell, so the used shell had to be changed explicitly
+        # (or stdenvNoCC.shell would be used) which happened in stage4.
+        binutils = super.binutils.override {
+          runtimeShell = "${bootstrapTools}/bin/bash";
+        };
         gcc-unwrapped =
           (super.gcc-unwrapped.override (commonGccOverrides // {
             # The most logical name for this package would be something like
@@ -544,13 +554,10 @@ in
       # other purposes (binutils and top-level pkgs) too.
       inherit (prevStage) gettext gnum4 bison perl texinfo zlib linuxHeaders libidn2 libunistring;
       ${localSystem.libc} = getLibc prevStage;
+      # Since this is the first fresh build of binutils since stage2, our own runtimeShell will be used.
       binutils = super.binutils.override {
-        # Don't use stdenv's shell but our own
-        shell = self.bash + "/bin/bash";
         # Build expand-response-params with last stage like below
-        buildPackages = {
-          inherit (prevStage) stdenv;
-        };
+        inherit (prevStage) expand-response-params;
       };
 
       # To allow users' overrides inhibit dependencies too heavy for
@@ -561,15 +568,12 @@ in
         nativeTools = false;
         nativeLibc = false;
         isGNU = true;
-        buildPackages = {
-          inherit (prevStage) stdenv;
-        };
+        inherit (prevStage) expand-response-params;
         cc = prevStage.gcc-unwrapped;
         bintools = self.binutils;
         libc = getLibc self;
         inherit lib;
-        inherit (self) stdenvNoCC coreutils gnugrep;
-        shell = self.bash + "/bin/bash";
+        inherit (self) stdenvNoCC coreutils gnugrep runtimeShell;
         fortify-headers = self.fortify-headers;
       };
     };
@@ -646,7 +650,9 @@ in
         # More complicated cases
         ++ (map (x: getOutput x (getLibc prevStage)) [ "out" "dev" "bin" ] )
         ++  [ linuxHeaders # propagated from .dev
-            binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params gcc.cc.libgcc glibc.passthru.libgcc
+              binutils gcc gcc.cc gcc.cc.lib
+              gcc.expand-response-params # != (prevStage.)expand-response-params
+              gcc.cc.libgcc glibc.passthru.libgcc
           ]
         ++ lib.optionals (localSystem.libc == "musl") [ fortify-headers ]
         ++ [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ]

pkgs/top-level/all-packages.nix

@@ -656,6 +656,8 @@ with pkgs;
 
   evhz = callPackage ../tools/misc/evhz { };
 
+  expand-response-params = callPackage ../build-support/expand-response-params { };
+
   expressvpn = callPackage ../applications/networking/expressvpn { };
 
   faq = callPackage ../development/tools/faq { };