From 92147a69855371cd065d64152956f2c4815cb4d0 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Fri, 12 Apr 2024 00:14:16 +0200 Subject: [PATCH 1/3] wrapCC, wrapBintools: use runtimeShell instead of stdenv shell We don't need to artificially make sure that we can execute the wrapper scripts on the build platform by using stdenv's shell (which comes from buildPackages) since our cross infrastructure will get us the wrapper from buildPackages. The upside of this change is that cross-compiled wrappers (e.g. pkgsCross.aarch64-multiplatform.gcc) will actually work when executed! For bootstrapping this is also not a problem, since we have a long build->build platform chain so runtimeShell is just as good as stdenvNoCC.shell. We do fall back to old ways, though, by explicitly using the bootstrap-tools shell in stage2, so the adjacent bash is only used from stage4 onwards. This is unnecessary in principle (I'll try removing this hack in the future), but ensures this change causes zero rebuilds. --- pkgs/build-support/bintools-wrapper/default.nix | 6 ++++-- pkgs/build-support/cc-wrapper/default.nix | 6 ++++-- pkgs/stdenv/darwin/default.nix | 10 +++++----- pkgs/stdenv/linux/default.nix | 16 ++++++++++++---- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix index 3c553f181f0dd..7bb96b470687d 100644 --- a/pkgs/build-support/bintools-wrapper/default.nix +++ b/pkgs/build-support/bintools-wrapper/default.nix @@ -8,7 +8,8 @@ { name ? "" , lib , stdenvNoCC -, bintools ? null, libc ? null, coreutils ? null, shell ? stdenvNoCC.shell, gnugrep ? null +, runtimeShell +, bintools ? null, libc ? null, coreutils ? null, gnugrep ? null , netbsd ? null, netbsdCross ? null , sharedLibraryLoader ? if libc == null then @@ -419,7 +420,8 @@ stdenvNoCC.mkDerivation { env = { # for substitution in utils.bash expandResponseParams = "${expand-response-params}/bin/expand-response-params"; - shell = getBin shell + shell.shellPath or ""; + # TODO(@sternenseemann): rename env var via stdenv rebuild + shell = (getBin runtimeShell + runtimeShell.shellPath or ""); gnugrep_bin = optionalString (!nativeTools) gnugrep; wrapperName = "BINTOOLS_WRAPPER"; inherit dynamicLinker targetPrefix suffixSalt coreutils_bin; diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 11ae9868ce016..17aac3170971d 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -8,7 +8,8 @@ { name ? "" , lib , stdenvNoCC -, cc ? null, libc ? null, bintools, coreutils ? null, shell ? stdenvNoCC.shell +, runtimeShell +, cc ? null, libc ? null, bintools, coreutils ? null , zlib ? null , nativeTools, noLibc ? false, nativeLibc, nativePrefix ? "" , propagateDoc ? cc != null && cc ? man @@ -739,7 +740,8 @@ stdenvNoCC.mkDerivation { # for substitution in utils.bash expandResponseParams = "${expand-response-params}/bin/expand-response-params"; - shell = getBin shell + shell.shellPath or ""; + # TODO(@sternenseemann): rename env var via stdenv rebuild + shell = getBin runtimeShell + runtimeShell.shellPath or ""; gnugrep_bin = optionalString (!nativeTools) gnugrep; # stdenv.cc.cc should not be null and we have nothing better for now. # if the native impure bootstrap is gotten rid of this can become `inherit cc;` again. diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index b1094630dd465..ef5314d92d774 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -117,6 +117,7 @@ let inherit (prevStage) coreutils gnugrep; stdenvNoCC = prevStage.ccWrapperStdenv; + runtimeShell = prevStage.ccWrapperStdenv.shell; }; bash = prevStage.bash or bootstrapTools; @@ -258,6 +259,7 @@ in inherit lib; inherit (self) stdenvNoCC coreutils gnugrep; + runtimeShell = self.stdenvNoCC.shell; bintools = selfDarwin.binutils-unwrapped; @@ -457,6 +459,8 @@ in bintools = selfDarwin.binutils-unwrapped; libc = selfDarwin.Libsystem; + # TODO(@sternenseemann): can this be removed? + runtimeShell = "${bootstrapTools}/bin/bash"; }; binutils-unwrapped = superDarwin.binutils-unwrapped.override { @@ -1044,8 +1048,6 @@ in }; binutils = superDarwin.binutils.override { - shell = self.bash + "/bin/bash"; - buildPackages = { inherit (prevStage) stdenv; }; @@ -1124,9 +1126,7 @@ in inherit (self.llvmPackages) libcxx; inherit lib; - inherit (self) stdenvNoCC coreutils gnugrep; - - shell = self.bash + "/bin/bash"; + inherit (self) stdenvNoCC coreutils gnugrep runtimeShell; }; }); libraries = super.llvmPackages.libraries.extend (_: _:{ diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 0e1852805cfd2..4f2e3b3cb0bce 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -196,6 +196,7 @@ let inherit (prevStage) coreutils gnugrep; stdenvNoCC = prevStage.ccWrapperStdenv; fortify-headers = prevStage.fortify-headers; + runtimeShell = prevStage.ccWrapperStdenv.shell; }).overrideAttrs(a: lib.optionalAttrs (prevStage.gcc-unwrapped.passthru.isXgcc or false) { # This affects only `xgcc` (the compiler which compiles the final compiler). postFixup = (a.postFixup or "") + '' @@ -265,6 +266,7 @@ in inherit lib; inherit (self) stdenvNoCC coreutils gnugrep; bintools = bootstrapTools; + runtimeShell = "${bootstrapTools}/bin/bash"; }; coreutils = bootstrapTools; gnugrep = bootstrapTools; @@ -332,6 +334,14 @@ in inherit (prevStage) ccWrapperStdenv coreutils gnugrep gettext bison texinfo zlib gnum4 perl patchelf; ${localSystem.libc} = getLibc prevStage; gmp = super.gmp.override { cxx = false; }; + # This stage also rebuilds binutils which will of course be used only in the next stage. + # We inherit this until stage3, in stage4 it will be rebuilt using the adjacent bash/runtimeShell pkg. + # TODO(@sternenseemann): Can we already build the wrapper with the actual runtimeShell here? + # Historically, the wrapper didn't use runtimeShell, so the used shell had to be changed explicitly + # (or stdenvNoCC.shell would be used) which happened in stage4. + binutils = super.binutils.override { + runtimeShell = "${bootstrapTools}/bin/bash"; + }; gcc-unwrapped = (super.gcc-unwrapped.override (commonGccOverrides // { # The most logical name for this package would be something like @@ -544,9 +554,8 @@ in # other purposes (binutils and top-level pkgs) too. inherit (prevStage) gettext gnum4 bison perl texinfo zlib linuxHeaders libidn2 libunistring; ${localSystem.libc} = getLibc prevStage; + # Since this is the first fresh build of binutils since stage2, our own runtimeShell will be used. binutils = super.binutils.override { - # Don't use stdenv's shell but our own - shell = self.bash + "/bin/bash"; # Build expand-response-params with last stage like below buildPackages = { inherit (prevStage) stdenv; @@ -568,8 +577,7 @@ in bintools = self.binutils; libc = getLibc self; inherit lib; - inherit (self) stdenvNoCC coreutils gnugrep; - shell = self.bash + "/bin/bash"; + inherit (self) stdenvNoCC coreutils gnugrep runtimeShell; fortify-headers = self.fortify-headers; }; }; From cb680bda6faaadaecf9d711ea6d3d315ebc631f4 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Tue, 16 Apr 2024 17:44:08 +0200 Subject: [PATCH 2/3] wrapCC, wrapBintools: move expand-response-params bootstrapping out The cc and bintools wrapper contained ad hoc bootstrapping logic for expand-response-params (which was callPackage-ed in a let binding). This lead to the strange situation that the bootstrapping logic related to expand-response-params is split between the wrapper derivations (where it is duplicated) and the actual stdenv bootstrapping. To clean this up, the wrappers simply should take expand-response-params as an ordinary input: They need an adjacent expand-response-params (i.e. one that runs on their host platform), but don't care about the how. Providing this is only problematic during stdenv bootstrapping where we have to pull it from the previous stage at times. --- .../bintools-wrapper/default.nix | 7 ++----- pkgs/build-support/cc-wrapper/default.nix | 9 +++++---- pkgs/stdenv/darwin/default.nix | 20 +++++++------------ pkgs/stdenv/linux/default.nix | 20 +++++++++---------- pkgs/top-level/all-packages.nix | 2 ++ 5 files changed, 25 insertions(+), 33 deletions(-) diff --git a/pkgs/build-support/bintools-wrapper/default.nix b/pkgs/build-support/bintools-wrapper/default.nix index 7bb96b470687d..5ca5bc3f5eb3b 100644 --- a/pkgs/build-support/bintools-wrapper/default.nix +++ b/pkgs/build-support/bintools-wrapper/default.nix @@ -29,7 +29,7 @@ , isGNU ? bintools.isGNU or false , isLLVM ? bintools.isLLVM or false , isCCTools ? bintools.isCCTools or false -, buildPackages ? {} +, expand-response-params , targetPackages ? {} , useMacosReexportHack ? false , wrapGas ? false @@ -132,10 +132,6 @@ let else if hasSuffix "pc-gnu" targetPlatform.config then "ld.so.1" else ""; - expand-response-params = - optionalString (buildPackages ? stdenv && buildPackages.stdenv.hasCC && buildPackages.stdenv.cc != "/dev/null") - (import ../expand-response-params { inherit (buildPackages) stdenv; }); - in stdenvNoCC.mkDerivation { @@ -419,6 +415,7 @@ stdenvNoCC.mkDerivation { env = { # for substitution in utils.bash + # TODO(@sternenseemann): invent something cleaner than passing in "" in case of absence expandResponseParams = "${expand-response-params}/bin/expand-response-params"; # TODO(@sternenseemann): rename env var via stdenv rebuild shell = (getBin runtimeShell + runtimeShell.shellPath or ""); diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix index 17aac3170971d..4adc1dcb1f8d8 100644 --- a/pkgs/build-support/cc-wrapper/default.nix +++ b/pkgs/build-support/cc-wrapper/default.nix @@ -16,7 +16,7 @@ , extraTools ? [], extraPackages ? [], extraBuildCommands ? "" , nixSupport ? {} , isGNU ? false, isClang ? cc.isClang or false, isCcache ? cc.isCcache or false, gnugrep ? null -, buildPackages ? {} +, expand-response-params , libcxx ? null # Whether or not to add `-B` and `-L` to `nix-support/cc-{c,ld}flags` @@ -113,9 +113,6 @@ let # unstable implementation detail, however. suffixSalt = replaceStrings ["-" "."] ["_" "_"] targetPlatform.config; - expand-response-params = - optionalString ((buildPackages.stdenv.hasCC or false) && buildPackages.stdenv.cc != "/dev/null") (import ../expand-response-params { inherit (buildPackages) stdenv; }); - useGccForLibs = useCcForLibs && libcxx == null && !targetPlatform.isDarwin @@ -298,6 +295,9 @@ stdenvNoCC.mkDerivation { '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)})) ''; + # Expose expand-response-params we are /actually/ using. In stdenv + # bootstrapping, expand-response-params usually comes from an earlier stage, + # so it is important to expose this for reference checking. inherit expand-response-params; inherit nixSupport; @@ -739,6 +739,7 @@ stdenvNoCC.mkDerivation { inherit isClang; # for substitution in utils.bash + # TODO(@sternenseemann): invent something cleaner than passing in "" in case of absence expandResponseParams = "${expand-response-params}/bin/expand-response-params"; # TODO(@sternenseemann): rename env var via stdenv rebuild shell = getBin runtimeShell + runtimeShell.shellPath or ""; diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index ef5314d92d774..2335a8d18de58 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -76,9 +76,9 @@ let nativeTools = false; nativeLibc = false; - buildPackages = lib.optionalAttrs (prevStage ? stdenv) { - inherit (prevStage) stdenv; - }; + expand-response-params = lib.optionalString + (prevStage.stdenv.hasCC or false && prevStage.stdenv.cc != "/dev/null") + prevStage.expand-response-params; extraPackages = [ prevStage.llvmPackages.compiler-rt @@ -254,7 +254,7 @@ in nativeTools = false; nativeLibc = false; - buildPackages = { }; + expand-response-params = ""; libc = selfDarwin.Libsystem; inherit lib; @@ -841,9 +841,7 @@ in # Rewrap binutils so it uses the rebuilt Libsystem. binutils = superDarwin.binutils.override { - buildPackages = { - inherit (prevStage) stdenv; - }; + inherit (prevStage) expand-response-params; libc = selfDarwin.Libsystem; } // { passthru = { inherit (prevStage.bintools.passthru) isFromBootstrapFiles; }; @@ -1048,9 +1046,7 @@ in }; binutils = superDarwin.binutils.override { - buildPackages = { - inherit (prevStage) stdenv; - }; + inherit (prevStage) expand-response-params; bintools = selfDarwin.binutils-unwrapped; libc = selfDarwin.Libsystem; @@ -1087,9 +1083,7 @@ in nativeTools = false; nativeLibc = false; - buildPackages = { - inherit (prevStage) stdenv; - }; + inherit (prevStage) expand-response-params; extraPackages = [ self.llvmPackages.compiler-rt diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 4f2e3b3cb0bce..e1801abcb485b 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -185,9 +185,9 @@ let name = "${name}-gcc-wrapper"; nativeTools = false; nativeLibc = false; - buildPackages = lib.optionalAttrs (prevStage ? stdenv) { - inherit (prevStage) stdenv; - }; + expand-response-params = lib.optionalString + (prevStage.stdenv.hasCC or false && prevStage.stdenv.cc != "/dev/null") + prevStage.expand-response-params; cc = prevStage.gcc-unwrapped; bintools = prevStage.binutils; isGNU = true; @@ -261,7 +261,7 @@ in name = "bootstrap-stage0-binutils-wrapper"; nativeTools = false; nativeLibc = false; - buildPackages = { }; + expand-response-params = ""; libc = getLibc self; inherit lib; inherit (self) stdenvNoCC coreutils gnugrep; @@ -557,9 +557,7 @@ in # Since this is the first fresh build of binutils since stage2, our own runtimeShell will be used. binutils = super.binutils.override { # Build expand-response-params with last stage like below - buildPackages = { - inherit (prevStage) stdenv; - }; + inherit (prevStage) expand-response-params; }; # To allow users' overrides inhibit dependencies too heavy for @@ -570,9 +568,7 @@ in nativeTools = false; nativeLibc = false; isGNU = true; - buildPackages = { - inherit (prevStage) stdenv; - }; + inherit (prevStage) expand-response-params; cc = prevStage.gcc-unwrapped; bintools = self.binutils; libc = getLibc self; @@ -654,7 +650,9 @@ in # More complicated cases ++ (map (x: getOutput x (getLibc prevStage)) [ "out" "dev" "bin" ] ) ++ [ linuxHeaders # propagated from .dev - binutils gcc gcc.cc gcc.cc.lib gcc.expand-response-params gcc.cc.libgcc glibc.passthru.libgcc + binutils gcc gcc.cc gcc.cc.lib + gcc.expand-response-params # != (prevStage.)expand-response-params + gcc.cc.libgcc glibc.passthru.libgcc ] ++ lib.optionals (localSystem.libc == "musl") [ fortify-headers ] ++ [ prevStage.updateAutotoolsGnuConfigScriptsHook prevStage.gnu-config ] diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index f07cc19c7b3e4..088dd4c389e88 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -656,6 +656,8 @@ with pkgs; evhz = callPackage ../tools/misc/evhz { }; + expand-response-params = callPackage ../build-support/expand-response-params { }; + expressvpn = callPackage ../applications/networking/expressvpn { }; faq = callPackage ../development/tools/faq { }; From b544fd9605ae011e851e30c1684b804cf05e1204 Mon Sep 17 00:00:00 2001 From: sternenseemann Date: Tue, 16 Apr 2024 17:56:13 +0200 Subject: [PATCH 3/3] expand-response-params: add description Since the tool is exposed more prominently now, we should clear up what it is and note that it is to be considered unstable, i.e. we may change it if the necessity arises. (In practice it is probably going to be fairly stable though, as compiler interfaces tend to be quite stable.) Should we add a version? --- .../expand-response-params/default.nix | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/expand-response-params/default.nix b/pkgs/build-support/expand-response-params/default.nix index 7ce15e98c8d99..6868ab97d8964 100644 --- a/pkgs/build-support/expand-response-params/default.nix +++ b/pkgs/build-support/expand-response-params/default.nix @@ -1,4 +1,4 @@ -{ stdenv }: +{ stdenv, lib }: # A "response file" is a sequence of arguments that is passed via a # file, rather than via argv[]. @@ -25,4 +25,18 @@ stdenv.mkDerivation { mkdir -p $prefix/bin mv expand-response-params $prefix/bin/ ''; + + meta = { + description = "Internal tool used by the nixpkgs wrapper scripts for processing response files"; + longDescription = '' + expand-response-params is a tool that allows for obtaining a full list of all + arguments passed in a given compiler command line including those passed via + so-called response files. The nixpkgs wrapper scripts for bintools and C + compilers use it for processing compiler flags. As it is developed in + conjunction with the nixpkgs wrapper scripts, it should be considered as + unstable and subject to change. + ''; + license = lib.licenses.mit; + platforms = lib.platforms.all; + }; }