diff --git a/pkgs/development/compilers/gcc/11/default.nix b/pkgs/development/compilers/gcc/11/default.nix index de764b9ccc7c2..429ac9df902da 100644 --- a/pkgs/development/compilers/gcc/11/default.nix +++ b/pkgs/development/compilers/gcc/11/default.nix @@ -190,7 +190,7 @@ stdenv.mkDerivation ({ preConfigure = (import ../common/pre-configure.nix { inherit lib; inherit version targetPlatform hostPlatform gnatboot langAda langGo langJit crossStageStatic enableMultilib; - }) + '' + }) + lib.optionalString (libxcrypt!=null) '' ln -sf ${libxcrypt}/include/crypt.h libsanitizer/sanitizer_common/crypt.h ''; diff --git a/pkgs/development/interpreters/perl/default.nix b/pkgs/development/interpreters/perl/default.nix index 4d95ebf2052bf..b35c7b6e9b322 100644 --- a/pkgs/development/interpreters/perl/default.nix +++ b/pkgs/development/interpreters/perl/default.nix @@ -1,7 +1,7 @@ { config, lib, stdenv, fetchurl, fetchFromGitHub, pkgs, buildPackages , callPackage , enableThreading ? true, coreutils, makeWrapper -, enableCrypt ? true, libxcrypt ? null +, enableCrypt ? libxcrypt!=null, libxcrypt ? null , zlib }: diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index 388802fca8c66..3a85582c10bca 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -384,7 +384,7 @@ in with passthru; stdenv.mkDerivation { postInstall = let # References *not* to nuke from (sys)config files - keep-references = concatMapStringsSep " " (val: "-e ${val}") ([ + keep-references = concatMapStringsSep " " (val: "-e ${val}") (optionals (libxcrypt!=null) [ (placeholder "out") libxcrypt ] ++ optionals tzdataSupport [ tzdata diff --git a/pkgs/stdenv/cross/default.nix b/pkgs/stdenv/cross/default.nix index bf410ec0a6841..ddafa3d2c9aeb 100644 --- a/pkgs/stdenv/cross/default.nix +++ b/pkgs/stdenv/cross/default.nix @@ -1,7 +1,10 @@ { lib , localSystem, crossSystem, config, overlays, crossOverlays ? [] +, rebootstrap ? false }: +assert rebootstrap -> throw "bootstrapping is irrelevant to stdenv/cross"; + let bootStages = import ../. { inherit lib localSystem overlays; diff --git a/pkgs/stdenv/custom/default.nix b/pkgs/stdenv/custom/default.nix index 4c7380118f7d5..9fc4a069f0c65 100644 --- a/pkgs/stdenv/custom/default.nix +++ b/pkgs/stdenv/custom/default.nix @@ -1,7 +1,10 @@ { lib , localSystem, crossSystem, config, overlays, crossOverlays ? [] +, rebootstrap ? false }: +assert rebootstrap -> throw "bootstrapping is not relevant to stdenv/custom"; + assert crossSystem == localSystem; let diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix index 76c44870f4ad6..aad704f324d20 100644 --- a/pkgs/stdenv/darwin/default.nix +++ b/pkgs/stdenv/darwin/default.nix @@ -34,9 +34,11 @@ cpio = fetch { file = "cpio"; sha256 = "sha256-SWkwvLaFyV44kLKL2nx720SvcL4ej/p2V/bX3uqAGO0="; }; tarball = fetch { file = "bootstrap-tools.cpio.bz2"; sha256 = "sha256-kRC/bhCmlD4L7KAvJQgcukk7AinkMz4IwmG1rqlh5tA="; executable = false; }; } +, rebootstrap ? false }: assert crossSystem == localSystem; +assert rebootstrap -> throw "darwin does not yet support automatic rebootstrapping"; let inherit (localSystem) system; diff --git a/pkgs/stdenv/default.nix b/pkgs/stdenv/default.nix index 7a2ad665e09d7..ccee907f8bb60 100644 --- a/pkgs/stdenv/default.nix +++ b/pkgs/stdenv/default.nix @@ -8,6 +8,13 @@ lib # Args to pass on to the pkgset builder, too , localSystem, crossSystem, config, overlays, crossOverlays ? [] + +# If true, the bootstrapFiles will be rebuilt, resulting in a stdenv +# built by the same compiler as its bootstrap tools. Turning this +# feature on will considerably increase the rebuild cycle, but may +# help with troubleshooting the bootstrap. +, rebootstrap ? false + } @ args: let diff --git a/pkgs/stdenv/freebsd/default.nix b/pkgs/stdenv/freebsd/default.nix index de66085876052..3eafa152302c8 100644 --- a/pkgs/stdenv/freebsd/default.nix +++ b/pkgs/stdenv/freebsd/default.nix @@ -1,7 +1,10 @@ { lib , localSystem, crossSystem, config, overlays, crossOverlays ? [] +, rebootstrap ? false }: +assert rebootstrap -> throw "stdenv/freebsd does not yet support automatic rebootstrapping"; + assert crossSystem == localSystem; let inherit (localSystem) system; fetchURL = import ; diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix index 870fb04c3883f..5eb03f3dd8d4e 100644 --- a/pkgs/stdenv/linux/default.nix +++ b/pkgs/stdenv/linux/default.nix @@ -59,6 +59,16 @@ { lib , localSystem, crossSystem, config, overlays, crossOverlays ? [] +# ARMv8.0 backward compatibility (`-moutline-intrinsics`) currently +# requires regenerating bootstrapFiles; see: +# +# https://github.com/NixOS/nixpkgs/issues/108111 +# https://github.com/NixOS/nixpkgs/pull/108200 +# https://bugzilla.redhat.com/show_bug.cgi?id=1830472 +, rebootstrap ? localSystem.isAarch64 + +, minimal ? false + , bootstrapFiles ? let table = { glibc = { @@ -93,7 +103,7 @@ files = archLookupTable.${localSystem.system} or (if getCompatibleTools != null then getCompatibleTools else (abort "unsupported platform for the pure Linux stdenv")); in files -}: +} @args: assert crossSystem == localSystem; @@ -117,7 +127,7 @@ let # Download and unpack the bootstrap tools (coreutils, GCC, Glibc, ...). - bootstrapTools = import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) { + unpackBootstrapTools = bootstrapFiles: import (if localSystem.libc == "musl" then ./bootstrap-tools-musl else ./bootstrap-tools) { inherit system bootstrapFiles; extraAttrs = lib.optionalAttrs config.contentAddressedByDefault @@ -135,10 +145,9 @@ let # the bootstrap. In all stages, we build an stdenv and the package # set that can be built with that stdenv. stageFun = prevStage: - { name, overrides ? (self: super: {}), extraNativeBuildInputs ? [] }: + { name, overrides ? (self: super: {}), extraNativeBuildInputs ? [], bootstrapTools ? prevStage.bootstrapTools }: let - thisStdenv = import ../generic { name = "${name}-stdenv-linux"; buildPlatform = localSystem; @@ -177,9 +186,11 @@ let stdenvNoCC = prevStage.ccWrapperStdenv; }; - overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; }; + overrides = self: super: (overrides self super) // { + fetchurl = thisStdenv.fetchurlBoot; + inherit bootstrapTools; + }; }; - in { inherit config overlays; stdenv = thisStdenv; @@ -187,15 +198,42 @@ let in -[ +# when rebootstrapping, we first prepend a copy of the stdenv stages +# that use the fetched bootstrapFiles: +lib.optionals rebootstrap (import ./. (args // { + rebootstrap = false; + minimal = true; + overlays = overlays ++ [ (final: prev: { + gettext = null; + help2man = null; + texinfo = null; + python3Minimal = null; + python3 = null; + libxcrypt = null; + coreutils = prev.coreutilsMinimal; + }) ]; +})) ++ - ({}: { +[ + (prevStage: { __raw = true; - gcc-unwrapped = null; binutils = null; coreutils = null; gnugrep = null; + bootstrapTools = + unpackBootstrapTools + (if rebootstrap + then (prevStage.freshBootstrapTools.override { + # Copy busybox from the original bootstrapFiles; it is + # safe to do this because it is statically linked. This + # avoids having to rebuild a musl-gcc compiler (glibc + # doesn't know how to do static linking). + busyboxMinimal = prevStage.stdenv.bootstrapFiles.busybox; + # don't rebuild a custom compiler + bootGCC = prevStage.gcc.cc; + nukeReferences = prevStage.nukeReferences.override { perl = prevStage.stdenv.bootstrapTools; }; + }).bootstrapFiles else bootstrapFiles); }) # Build a dummy stdenv with no GCC or working fetchurl. This is @@ -203,7 +241,7 @@ in # # resulting stage0 stdenv: # - coreutils, binutils, glibc, gcc: from bootstrapFiles - (prevStage: stageFun prevStage { + ({ bootstrapTools, ...}@prevStage: stageFun prevStage { name = "bootstrap-stage0"; overrides = self: super: { @@ -244,6 +282,8 @@ in }; coreutils = bootstrapTools; gnugrep = bootstrapTools; + } // lib.optionalAttrs minimal { + nukeReferences = prevStage.nukeReferences.override { perl = prevStage.bootstrapTools; }; }; }) @@ -280,7 +320,7 @@ in # This is not an issue for the final stdenv, because this perl # won't be included in the final stdenv and won't be exported to # top-level pkgs as an override either. - perl = super.perl.override { enableThreading = false; enableCrypt = false; }; + perl = if minimal then prevStage.bootstrapTools else super.perl.override { enableThreading = false; enableCrypt = false; }; }; }) @@ -332,7 +372,8 @@ in # and that can fail to load. Therefore we upgrade `ld` to use newer libc; # apparently the interpreter needs to match libc, too. bintools = self.stdenvNoCC.mkDerivation { - inherit (prevStage.bintools.bintools) name; + pname = prevStage.bintools.bintools.pname + "-patchelfed"; + inherit (prevStage.bintools.bintools) version; enableParallelBuilding = true; dontUnpack = true; dontBuild = true; @@ -477,7 +518,7 @@ in # and the bootstrapTools-built, statically-linked # lib{mpfr,mpc,gmp,isl}.a which are linked into the final gcc # (see commit cfde88976ba4cddd01b1bb28b40afd12ea93a11d). - (prevStage: { + ({ bootstrapTools, ...}@prevStage: { inherit config overlays; stdenv = import ../generic rec { name = "stdenv-linux"; @@ -504,7 +545,7 @@ in inherit (prevStage.stdenv) fetchurlBoot; extraAttrs = { - inherit bootstrapTools; + inherit bootstrapTools bootstrapFiles; shellPackage = prevStage.bash; }; @@ -544,7 +585,12 @@ in inherit (self) stdenv runCommandLocal patchelf libunistring; }; + } // lib.optionalAttrs (!minimal) { gnumake = super.gnumake.override { inBootstrap = false; }; + } // lib.optionalAttrs minimal { + fetchurl = prevStage.stdenv.fetchurlBoot; + cpio = prevStage.bootstrapTools; + nukeReferences = prevStage.nukeReferences.override { perl = prevStage.bootstrapTools; }; } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) { # Need to get rid of these when cross-compiling. inherit (prevStage) binutils binutils-unwrapped; diff --git a/pkgs/stdenv/linux/make-bootstrap-tools.nix b/pkgs/stdenv/linux/make-bootstrap-tools.nix index c6112766781ee..4f9d65e8f118f 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools.nix @@ -1,22 +1,6 @@ -{ pkgs ? import ../../.. {} }: - -let - libc = pkgs.stdenv.cc.libc; -in with pkgs; rec { - - - coreutilsMinimal = coreutils.override (args: { - # We want coreutils without ACL/attr support. - aclSupport = false; - attrSupport = false; - # Our tooling currently can't handle scripts in bin/, only ELFs and symlinks. - singleBinary = "symlinks"; - }); - - tarMinimal = gnutar.override { acl = null; }; - - busyboxMinimal = busybox.override { - useMusl = !stdenv.targetPlatform.isRiscV; +{ pkgs ? import ../../.. {} +, busyboxMinimal ? "${pkgs.busybox.override { + useMusl = !pkgs.stdenv.targetPlatform.isRiscV; enableStatic = true; enableMinimal = true; extraConfig = '' @@ -28,9 +12,21 @@ in with pkgs; rec { CONFIG_TAR y CONFIG_UNXZ y ''; - }; + }}/bin/busybox" +, bootGCC ? pkgs.gcc.cc.override { enableLTO = false; } +, nukeReferences ? pkgs.buildPackages.nukeReferences +, cpio ? pkgs.buildPackages.cpio +, rebootstrap ? false +}: + +let + libc = pkgs.stdenv.cc.libc; +in with pkgs; rec { + + inherit (pkgs) coreutilsMinimal; + + tarMinimal = gnutar.override { acl = null; }; - bootGCC = gcc.cc.override { enableLTO = false; }; bootBinutils = binutils.bintools.override { withAllTargets = false; # Don't need two linkers, disable whatever's not primary/default. @@ -50,7 +46,7 @@ in with pkgs; rec { schedulingPriority = 200; }; - nativeBuildInputs = [ buildPackages.nukeReferences buildPackages.cpio ]; + nativeBuildInputs = [ nukeReferences cpio ]; buildCommand = '' set -x @@ -196,7 +192,7 @@ in with pkgs; rec { mkdir $out/on-server XZ_OPT="-9 -e" tar cvJf $out/on-server/bootstrap-tools.tar.xz --hard-dereference --sort=name --numeric-owner --owner=0 --group=0 --mtime=@1 -C $out/pack . - cp ${busyboxMinimal}/bin/busybox $out/on-server + cp ${busyboxMinimal} $out/on-server/busybox chmod u+w $out/on-server/busybox nuke-refs $out/on-server/busybox ''; # */ diff --git a/pkgs/stdenv/native/default.nix b/pkgs/stdenv/native/default.nix index 87862b84bc1b2..c50799db1e51d 100644 --- a/pkgs/stdenv/native/default.nix +++ b/pkgs/stdenv/native/default.nix @@ -1,8 +1,10 @@ { lib , localSystem, crossSystem, config, overlays, crossOverlays ? [] +, rebootstrap }: assert crossSystem == localSystem; +assert rebootstrap -> throw "stdenv/native does not yet support automatic rebootstrapping"; let inherit (localSystem) system; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e4dc26675b41f..b4f0ce16e2924 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6139,6 +6139,13 @@ with pkgs; coreutils = callPackage ../tools/misc/coreutils { }; coreutils-full = coreutils.override { minimal = false; }; coreutils-prefixed = coreutils.override { withPrefix = true; singleBinary = false; }; + coreutilsMinimal = callPackage ../tools/misc/coreutils { # this is for the bootstrapFiles + # We want coreutils without ACL/attr support. + aclSupport = false; + attrSupport = false; + # Our tooling currently can't handle scripts in bin/, only ELFs and symlinks. + singleBinary = "symlinks"; + }; corkscrew = callPackage ../tools/networking/corkscrew { }; @@ -14052,7 +14059,6 @@ with pkgs; inherit (let num = if (with stdenv.targetPlatform; isVc4 || libc == "relibc") then 6 - else if (stdenv.targetPlatform.isAarch64 && stdenv.isLinux) then 9 else 11; numS = toString num; in {