From 27606dd81a6f6fc65f190a618e72dcb054f3eb26 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Thu, 11 Apr 2024 08:47:48 +0200 Subject: [PATCH 1/7] pkgs/top-level: rewrite some outdated comments This removes all specific references to pkgsCross or pkgsi686Linux, because they have become outdated with the addition of many more package sets. --- pkgs/top-level/default.nix | 16 ++++++---------- pkgs/top-level/stage.nix | 7 +------ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index 12329d0af82fd..71c510cbd0e97 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -134,22 +134,18 @@ let config = lib.showWarnings configEval.config.warnings configEval.config; # A few packages make a new package set to draw their dependencies from. - # (Currently to get a cross tool chain, or forced-i686 package.) Rather than - # give `all-packages.nix` all the arguments to this function, even ones that - # don't concern it, we give it this function to "re-call" nixpkgs, inheriting - # whatever arguments it doesn't explicitly provide. This way, - # `all-packages.nix` doesn't know more than it needs too. + # Rather than give `all-packages.nix` all the arguments to this function, + # even ones that don't concern it, we give it this function to "re-call" + # nixpkgs, inheriting whatever arguments it doesn't explicitly provide. This + # way, `all-packages.nix` doesn't know more than it needs to. # # It's OK that `args` doesn't include default arguments from this file: # they'll be deterministically inferred. In fact we must *not* include them, # because it's important that if some parameter which affects the default is # substituted with a different argument, the default is re-inferred. # - # To put this in concrete terms, this function is basically just used today to - # use package for a different platform for the current platform (namely cross - # compiling toolchains and 32-bit packages on x86_64). In both those cases we - # want the provided non-native `localSystem` argument to affect the stdenv - # chosen. + # To put this in concrete terms, we want the provided non-native `localSystem` + # and `crossSystem` arguments to affect the stdenv chosen. # # NB!!! This thing gets its `config` argument from `args`, i.e. it's actually # `config0`. It is important to keep it to `config0` format (as opposed to the diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 2af30f3de4ec0..b2c33c61e7f3f 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -212,12 +212,7 @@ let lib.optionalAttrs allowCustomOverrides ((config.packageOverrides or (super: { })) super); # Convenience attributes for instantitating package sets. Each of - # these will instantiate a new version of allPackages. Currently the - # following package sets are provided: - # - # - pkgsCross. where system is a member of lib.systems.examples - # - pkgsMusl - # - pkgsi686Linux + # these will instantiate a new version of allPackages. otherPackageSets = self: super: { # This maps each entry in lib.systems.examples to its own package # set. Each of these will contain all packages cross compiled for From dd9e0ff0828cf75610df8287934490d2e705a31f Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 26 May 2024 21:43:20 +0200 Subject: [PATCH 2/7] pkgs/top-level/stage: refactor moving more generic package sets to the bottom No change, just move appendOverlays and extend to the bottom, since they will be changed much less often. This makes it easier to compare the other package sets side-by-side. --- pkgs/top-level/stage.nix | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index b2c33c61e7f3f..5844cbb1b1d9f 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -353,22 +353,6 @@ let localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; }; - # Extend the package set with zero or more overlays. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - appendOverlays = - extraOverlays: - if extraOverlays == [ ] then self else nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; - - # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB - # of allocations. DO NOT USE THIS IN NIXPKGS. - # - # Extend the package set with a single overlay. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - # Prefer appendOverlays if used repeatedly. - extend = f: self.appendOverlays [ f ]; - # Fully static packages. # Currently uses Musl on Linux (couldn’t get static glibc to work). pkgsStatic = nixpkgsFun ({ @@ -419,6 +403,22 @@ let ) ] ++ overlays; }; + + # Extend the package set with zero or more overlays. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + appendOverlays = + extraOverlays: + if extraOverlays == [ ] then self else nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; + + # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB + # of allocations. DO NOT USE THIS IN NIXPKGS. + # + # Extend the package set with a single overlay. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + # Prefer appendOverlays if used repeatedly. + extend = f: self.appendOverlays [ f ]; }; # The complete chain of package set builders, applied from top to bottom. From 9ebbec34574ebc9be9feb15d6152f2e51c413939 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sun, 15 Dec 2024 15:05:05 +0100 Subject: [PATCH 3/7] pkgs/top-level: refactor mkPkgs Sharing a first piece of common code between all package sets makes it easier to maintain and less likely to introduce a new package set without this. --- pkgs/top-level/stage.nix | 360 ++++++++++++++++++--------------------- 1 file changed, 168 insertions(+), 192 deletions(-) diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 5844cbb1b1d9f..750d8a9631fd3 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -213,214 +213,190 @@ let # Convenience attributes for instantitating package sets. Each of # these will instantiate a new version of allPackages. - otherPackageSets = self: super: { - # This maps each entry in lib.systems.examples to its own package - # set. Each of these will contain all packages cross compiled for - # that target system. For instance, pkgsCross.raspberryPi.hello, - # will refer to the "hello" package built for the ARM6-based - # Raspberry Pi. - pkgsCross = lib.mapAttrs (n: crossSystem: nixpkgsFun { inherit crossSystem; }) lib.systems.examples; - - pkgsLLVM = nixpkgsFun { - overlays = [ - (self': super': { - pkgsLLVM = super'; - }) - ] ++ overlays; - # Bootstrap a cross stdenv using the LLVM toolchain. - # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { - useLLVM = true; - linker = "lld"; + otherPackageSets = + let + mkPkgs = + name: nixpkgsArgs: + nixpkgsFun ( + nixpkgsArgs + // { + overlays = + [ + (self': super': { + "${name}" = super'; + }) + ] + ++ nixpkgsArgs.overlays or [ ] + ++ overlays; + } + ); + in + self: super: { + # This maps each entry in lib.systems.examples to its own package + # set. Each of these will contain all packages cross compiled for + # that target system. For instance, pkgsCross.raspberryPi.hello, + # will refer to the "hello" package built for the ARM6-based + # Raspberry Pi. + pkgsCross = lib.mapAttrs (n: crossSystem: nixpkgsFun { inherit crossSystem; }) lib.systems.examples; + + pkgsLLVM = mkPkgs "pkgsLLVM" { + # Bootstrap a cross stdenv using the LLVM toolchain. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + useLLVM = true; + linker = "lld"; + }; }; - }; - pkgsLLVMLibc = nixpkgsFun { - overlays = [ - (self': super': { - pkgsLLVMLibc = super'; - }) - ] ++ overlays; - # Bootstrap a cross stdenv using LLVM libc. - # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { - config = lib.systems.parse.tripleFromSystem (makeLLVMParsedPlatform stdenv.hostPlatform.parsed); - libc = "llvm"; + pkgsLLVMLibc = mkPkgs "pkgsLLVMLibc" { + # Bootstrap a cross stdenv using LLVM libc. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + config = lib.systems.parse.tripleFromSystem (makeLLVMParsedPlatform stdenv.hostPlatform.parsed); + libc = "llvm"; + }; }; - }; - pkgsArocc = nixpkgsFun { - overlays = [ - (self': super': { - pkgsArocc = super'; - }) - ] ++ overlays; - # Bootstrap a cross stdenv using the Aro C compiler. - # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { - useArocc = true; - linker = "lld"; + pkgsArocc = mkPkgs "pkgsArocc" { + # Bootstrap a cross stdenv using the Aro C compiler. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + useArocc = true; + linker = "lld"; + }; }; - }; - pkgsZig = nixpkgsFun { - overlays = [ - (self': super': { - pkgsZig = super'; - }) - ] ++ overlays; - # Bootstrap a cross stdenv using the Zig toolchain. - # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { - useZig = true; - linker = "lld"; + pkgsZig = mkPkgs "pkgsZig" { + # Bootstrap a cross stdenv using the Zig toolchain. + # This is currently not possible when compiling natively, + # so we don't need to check hostPlatform != buildPlatform. + crossSystem = stdenv.hostPlatform // { + useZig = true; + linker = "lld"; + }; }; - }; - # All packages built with the Musl libc. This will override the - # default GNU libc on Linux systems. Non-Linux systems are not - # supported. 32-bit is also not supported. - pkgsMusl = - if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then - nixpkgsFun { - overlays = [ - (self': super': { - pkgsMusl = super'; - }) - ] ++ overlays; - ${if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" else "crossSystem"} = { - config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); - }; - } - else - throw "Musl libc only supports 64-bit Linux systems."; - - # All packages built for i686 Linux. - # Used by wine, firefox with debugging version of Flash, ... - pkgsi686Linux = - if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then - nixpkgsFun { - overlays = [ - (self': super': { - pkgsi686Linux = super'; - }) - ] ++ overlays; - ${if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" else "crossSystem"} = { - config = lib.systems.parse.tripleFromSystem ( - stdenv.hostPlatform.parsed - // { - cpu = lib.systems.parse.cpuTypes.i686; - } - ); + # All packages built with the Musl libc. This will override the + # default GNU libc on Linux systems. Non-Linux systems are not + # supported. 32-bit is also not supported. + pkgsMusl = + if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then + mkPkgs "pkgsMusl" { + ${if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" else "crossSystem"} = { + config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); + }; + } + else + throw "Musl libc only supports 64-bit Linux systems."; + + # All packages built for i686 Linux. + # Used by wine, firefox with debugging version of Flash, ... + pkgsi686Linux = + if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then + mkPkgs "pkgsi686Linux" { + ${if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" else "crossSystem"} = { + config = lib.systems.parse.tripleFromSystem ( + stdenv.hostPlatform.parsed + // { + cpu = lib.systems.parse.cpuTypes.i686; + } + ); + }; + } + else + throw "i686 Linux package set can only be used with the x86 family."; + + # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages + pkgsx86_64Darwin = + if stdenv.hostPlatform.isDarwin then + mkPkgs "pkgsx86_64Darwin" { + localSystem = { + config = lib.systems.parse.tripleFromSystem ( + stdenv.hostPlatform.parsed + // { + cpu = lib.systems.parse.cpuTypes.x86_64; + } + ); + }; + } + else + throw "x86_64 Darwin package set can only be used on Darwin systems."; + + # If already linux: the same package set unaltered + # Otherwise, return a natively built linux package set for the current cpu architecture string. + pkgsLinux = + if stdenv.hostPlatform.isLinux then + self + else + mkPkgs "pkgsLinux" { + localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; }; - } - else - throw "i686 Linux package set can only be used with the x86 family."; - - # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages - pkgsx86_64Darwin = - if stdenv.hostPlatform.isDarwin then - nixpkgsFun { - overlays = [ - (self': super': { - pkgsx86_64Darwin = super'; - }) - ] ++ overlays; - localSystem = { - config = lib.systems.parse.tripleFromSystem ( + + # Fully static packages. + # Currently uses Musl on Linux (couldn’t get static glibc to work). + pkgsStatic = mkPkgs "pkgsStatic" { + crossSystem = { + isStatic = true; + config = lib.systems.parse.tripleFromSystem ( + if stdenv.hostPlatform.isLinux then + makeMuslParsedPlatform stdenv.hostPlatform.parsed + else stdenv.hostPlatform.parsed - // { - cpu = lib.systems.parse.cpuTypes.x86_64; - } - ); - }; - } - else - throw "x86_64 Darwin package set can only be used on Darwin systems."; - - # If already linux: the same package set unaltered - # Otherwise, return a natively built linux package set for the current cpu architecture string. - # (ABI and other details will be set to the default for the cpu/os pair) - pkgsLinux = - if stdenv.hostPlatform.isLinux then - self - else - nixpkgsFun { - localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; + ); + gcc = + lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } + // stdenv.hostPlatform.gcc or { }; }; + }; - # Fully static packages. - # Currently uses Musl on Linux (couldn’t get static glibc to work). - pkgsStatic = nixpkgsFun ({ - overlays = [ - (self': super': { - pkgsStatic = super'; - }) - ] ++ overlays; - crossSystem = { - isStatic = true; - config = lib.systems.parse.tripleFromSystem ( - if stdenv.hostPlatform.isLinux then - makeMuslParsedPlatform stdenv.hostPlatform.parsed - else - stdenv.hostPlatform.parsed - ); - gcc = - lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } - // stdenv.hostPlatform.gcc or { }; + pkgsExtraHardening = mkPkgs "pkgsExtraHardening" { + overlays = [ + ( + self': super': + { + stdenv = super'.withDefaultHardeningFlags ( + super'.stdenv.cc.defaultHardeningFlags + ++ [ + "shadowstack" + "pacret" + "trivialautovarinit" + ] + ) super'.stdenv; + glibc = super'.glibc.override rec { + enableCET = if self'.stdenv.hostPlatform.isx86_64 then "permissive" else false; + enableCETRuntimeDefault = enableCET != false; + }; + } + // lib.optionalAttrs (with super'.stdenv.hostPlatform; isx86_64 && isLinux) { + # causes shadowstack disablement + pcre = super'.pcre.override { enableJit = false; }; + pcre-cpp = super'.pcre-cpp.override { enableJit = false; }; + pcre16 = super'.pcre16.override { enableJit = false; }; + } + ) + ]; }; - }); - pkgsExtraHardening = nixpkgsFun { - overlays = [ - ( - self': super': - { - pkgsExtraHardening = super'; - stdenv = super'.withDefaultHardeningFlags ( - super'.stdenv.cc.defaultHardeningFlags - ++ [ - "shadowstack" - "pacret" - "trivialautovarinit" - ] - ) super'.stdenv; - glibc = super'.glibc.override rec { - enableCET = if self'.stdenv.hostPlatform.isx86_64 then "permissive" else false; - enableCETRuntimeDefault = enableCET != false; - }; - } - // lib.optionalAttrs (with super'.stdenv.hostPlatform; isx86_64 && isLinux) { - # causes shadowstack disablement - pcre = super'.pcre.override { enableJit = false; }; - pcre-cpp = super'.pcre-cpp.override { enableJit = false; }; - pcre16 = super'.pcre16.override { enableJit = false; }; - } - ) - ] ++ overlays; + # Extend the package set with zero or more overlays. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + appendOverlays = + extraOverlays: + if extraOverlays == [ ] then self else nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; + + # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB + # of allocations. DO NOT USE THIS IN NIXPKGS. + # + # Extend the package set with a single overlay. This preserves + # preexisting overlays. Prefer to initialize with the right overlays + # in one go when calling Nixpkgs, for performance and simplicity. + # Prefer appendOverlays if used repeatedly. + extend = f: self.appendOverlays [ f ]; }; - # Extend the package set with zero or more overlays. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - appendOverlays = - extraOverlays: - if extraOverlays == [ ] then self else nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; - - # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB - # of allocations. DO NOT USE THIS IN NIXPKGS. - # - # Extend the package set with a single overlay. This preserves - # preexisting overlays. Prefer to initialize with the right overlays - # in one go when calling Nixpkgs, for performance and simplicity. - # Prefer appendOverlays if used repeatedly. - extend = f: self.appendOverlays [ f ]; - }; - # The complete chain of package set builders, applied from top to bottom. # stdenvOverlays must be last as it brings package forward from the # previous bootstrapping phases which have already been overlayed. From 46c5e9d8dcd461f43f3fc0b448dfaddc21948a92 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Wed, 10 Apr 2024 22:18:25 +0200 Subject: [PATCH 4/7] pkgs/test/top-level: add tests for package set composability This adds some basic tests to compose package sets. The cases that are currently broken are commented out, they include things like: - pkgsStatic.pkgsMusl losing the isStatic flag - pkgsCross.ppc64-musl.pkgsMusl losing the gcc.abi setting - pkgsCross.mingwW64.pkgsStatic losing the config string - pkgsLLVM.pkgsMusl losing the useLLVM flag - pkgsLLVM.pkgsStatic losing the useLLVM flag - pkgsLLVM.pkgsi686Linux losing the useLLVM flag And probably more. --- pkgs/test/top-level/stage.nix | 122 ++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 pkgs/test/top-level/stage.nix diff --git a/pkgs/test/top-level/stage.nix b/pkgs/test/top-level/stage.nix new file mode 100644 index 0000000000000..84985c62d4250 --- /dev/null +++ b/pkgs/test/top-level/stage.nix @@ -0,0 +1,122 @@ +# run like this: +# nix-build pkgs/test/top-level/stage.nix +{ + localSystem ? { + system = builtins.currentSystem; + }, +}: + +with import ../../top-level { inherit localSystem; }; + +let + # To silence platform specific evaluation errors + discardEvaluationErrors = e: (builtins.tryEval e).success -> e; + + # Basic test for idempotency of the package set, i.e: + # Applying the same package set twice should work and + # not change anything. + isIdempotent = set: discardEvaluationErrors (pkgs.${set}.stdenv == pkgs.${set}.${set}.stdenv); + + # Some package sets should be noops in certain circumstances. + # This is very similar to the idempotency test, but not going + # via the super' overlay. + isNoop = + parent: child: + discardEvaluationErrors ( + (lib.getAttrFromPath parent pkgs).stdenv == (lib.getAttrFromPath parent pkgs).${child}.stdenv + ); + + allMuslExamples = builtins.attrNames ( + lib.filterAttrs (_: system: lib.hasSuffix "-musl" system.config) lib.systems.examples + ); + + allLLVMExamples = builtins.attrNames ( + lib.filterAttrs (_: system: system.useLLVM or false) lib.systems.examples + ); + + # A package set should only change specific configuration, but needs + # to keep all other configuration from previous layers in place. + # Each package set has one or more key characteristics for which we + # test here. Those should be kept, even when applying the "set" package + # set. + isComposable = + set: + lib.all discardEvaluationErrors ( + [ + (pkgsCross.ppc64-musl.${set}.stdenv.hostPlatform.gcc.abi == "elfv2") + (pkgs.pkgsLLVM.${set}.stdenv.hostPlatform.useLLVM) + (pkgs.pkgsArocc.${set}.stdenv.hostPlatform.useArocc) + (pkgs.pkgsZig.${set}.stdenv.hostPlatform.useZig) + (pkgs.pkgsLinux.${set}.stdenv.buildPlatform.isLinux) + (pkgs.pkgsStatic.${set}.stdenv.hostPlatform.isStatic) + (pkgs.pkgsi686Linux.${set}.stdenv.hostPlatform.isx86_32) + (pkgs.pkgsx86_64Darwin.${set}.stdenv.hostPlatform.isx86_64) + (builtins.elem "trivialautovarinit" pkgs.pkgsExtraHardening.${set}.stdenv.cc.defaultHardeningFlags) + ] # Can't compose two different libcs... + ++ lib.optionals (!builtins.elem set [ "pkgsLLVMLibc" ]) [ + (pkgsCross.mingwW64.${set}.stdenv.hostPlatform.config == "x86_64-w64-mingw32") + (pkgsCross.mingwW64.${set}.stdenv.hostPlatform.libc == "msvcrt") + (pkgs.pkgsMusl.${set}.stdenv.hostPlatform.isMusl) + ] + ++ + lib.optionals + ( + !builtins.elem set [ + "pkgsMusl" + "pkgsStatic" + ] + ) + [ + (pkgs.pkgsLLVMLibc.${set}.stdenv.hostPlatform.isLLVMLibc) + ] + ); +in + +# Appends same defaultHardeningFlags again on each .pkgsExtraHardening - thus not idempotent. +# assert isIdempotent "pkgsExtraHardening"; +# TODO: Remove the isDarwin condition, which currently results in infinite recursion. +# Also see https://github.com/NixOS/nixpkgs/pull/330567#discussion_r1894653309 +assert (stdenv.hostPlatform.isDarwin || isIdempotent "pkgsLLVM"); +# TODO: This currently results in infinite recursion, even on Linux +# assert isIdempotent "pkgsLLVMLibc"; +assert isIdempotent "pkgsArocc"; +assert isIdempotent "pkgsZig"; +assert isIdempotent "pkgsLinux"; +assert isIdempotent "pkgsMusl"; +assert isIdempotent "pkgsStatic"; +assert isIdempotent "pkgsi686Linux"; +assert isIdempotent "pkgsx86_64Darwin"; + +# TODO: fails +# assert isNoop [ "pkgsStatic" ] "pkgsMusl"; +# TODO: fails because of ppc64-musl +# assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples; +# This will fail on all systems which are not handled in pkgs/os-specific/bsd/openbsd/pkgs/mkDerivation.nix, +# for example armv6l-linux, powerpc64le-linux, riscv64-linux +assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples; + +# TODO: fails +# assert isComposable "pkgsExtraHardening"; +# assert isComposable "pkgsLLVM"; +# assert isComposable "pkgsLLVMLibc"; +# assert isComposable "pkgsArocc"; +# assert isComposable "pkgsZig"; +# TODO: attribute 'abi' missing +# assert isComposable "pkgsMusl"; +# TODO: fails +# assert isComposable "pkgsStatic"; +# assert isComposable "pkgsi686Linux"; + +# Special cases regarding buildPlatform vs hostPlatform +# TODO: fails +# assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl); +# assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32); +# assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); +# assert discardEvaluationErrors (pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64); + +# pkgsCross should keep upper cross settings +# TODO: fails +# assert discardEvaluationErrors (with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic); +# assert discardEvaluationErrors (with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM); + +emptyFile From acbabb9bdece306cda9ba1a084f33112c39f5765 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 9 Apr 2024 08:41:18 +0200 Subject: [PATCH 5/7] pkgs/top-level: make package sets composable The various pkgsXYZ top-level package sets did not pass localSystem / crossSystem to lower levels, so far. This change propagates original arguments to lower levels, which include the overrides made by an upper package sets. There is an extensive test-suite to test various combinations of package sets in pkgs/test/top-level. There are a few basic promises made: - Package sets must be idempotent. pkgsMusl.pkgsMusl === pkgsMusl. - Once pkgsCross is used any subsequent package sets should affect the **host platform** and not the build platform. Examples: - pkgsMusl.pkgsCross.aarch64-multiplatform is a cross compilation from musl to glibc/aarch64 - pkgsCross.aarch64-multiplatform.pkgsMusl is a cross compilation to musl/aarch64 - Modifications from an earlier layer should not be lost, unless explicitly overwritten. Examples: - pkgsStatic.pkgsMusl should still be static. - pkgsStatic.pkgsCross.gnu64 should be static, but with glibc instead of musl. Exceptions / TODOs: - pkgsExtraHardening is currently not idempotent, because it applies the same flags over and over again. Resolves #136549 Resolves #114510 Resolves #212494 Resolves #281596 --- lib/systems/default.nix | 3 ++ lib/tests/systems.nix | 1 + pkgs/test/top-level/stage.nix | 39 +++++++++++++------------- pkgs/top-level/stage.nix | 52 +++++++++++++++++++---------------- 4 files changed, 53 insertions(+), 42 deletions(-) diff --git a/lib/systems/default.nix b/lib/systems/default.nix index 67bd9826465c0..f3df685f9c303 100644 --- a/lib/systems/default.nix +++ b/lib/systems/default.nix @@ -534,6 +534,9 @@ let "-uefi" ]; }; + } + // { + override = allArgs.override or (newArgs: elaborate (allArgs // newArgs)); }; in assert final.useAndroidPrebuilt -> final.isAndroid; diff --git a/lib/tests/systems.nix b/lib/tests/systems.nix index 941f887304d3f..d5189e1b65063 100644 --- a/lib/tests/systems.nix +++ b/lib/tests/systems.nix @@ -281,6 +281,7 @@ lib.runTests ( emulatorAvailable = null; staticEmulatorAvailable = null; isCompatible = null; + override = null; } ? ${platformAttrName}; }; diff --git a/pkgs/test/top-level/stage.nix b/pkgs/test/top-level/stage.nix index 84985c62d4250..9b21871db4848 100644 --- a/pkgs/test/top-level/stage.nix +++ b/pkgs/test/top-level/stage.nix @@ -87,36 +87,37 @@ assert isIdempotent "pkgsStatic"; assert isIdempotent "pkgsi686Linux"; assert isIdempotent "pkgsx86_64Darwin"; -# TODO: fails -# assert isNoop [ "pkgsStatic" ] "pkgsMusl"; -# TODO: fails because of ppc64-musl -# assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples; +assert isNoop [ "pkgsStatic" ] "pkgsMusl"; +assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples; # This will fail on all systems which are not handled in pkgs/os-specific/bsd/openbsd/pkgs/mkDerivation.nix, # for example armv6l-linux, powerpc64le-linux, riscv64-linux assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples; -# TODO: fails -# assert isComposable "pkgsExtraHardening"; -# assert isComposable "pkgsLLVM"; +assert isComposable "pkgsExtraHardening"; +assert isComposable "pkgsLLVM"; +# TODO: Results in infinite recursion # assert isComposable "pkgsLLVMLibc"; -# assert isComposable "pkgsArocc"; -# assert isComposable "pkgsZig"; -# TODO: attribute 'abi' missing -# assert isComposable "pkgsMusl"; -# TODO: fails -# assert isComposable "pkgsStatic"; -# assert isComposable "pkgsi686Linux"; +assert isComposable "pkgsArocc"; +assert isComposable "pkgsZig"; +assert isComposable "pkgsMusl"; +assert isComposable "pkgsStatic"; +assert isComposable "pkgsi686Linux"; # Special cases regarding buildPlatform vs hostPlatform # TODO: fails # assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl); # assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32); -# assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); -# assert discardEvaluationErrors (pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64); +assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); +assert discardEvaluationErrors ( + pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64 +); # pkgsCross should keep upper cross settings -# TODO: fails -# assert discardEvaluationErrors (with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic); -# assert discardEvaluationErrors (with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM); +assert discardEvaluationErrors ( + with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic +); +assert discardEvaluationErrors ( + with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM +); emptyFile diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 750d8a9631fd3..bd4a2ac8f5aba 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -215,6 +215,7 @@ let # these will instantiate a new version of allPackages. otherPackageSets = let + isNative = lib.systems.equals stdenv.hostPlatform stdenv.buildPlatform; mkPkgs = name: nixpkgsArgs: nixpkgsFun ( @@ -237,13 +238,15 @@ let # that target system. For instance, pkgsCross.raspberryPi.hello, # will refer to the "hello" package built for the ARM6-based # Raspberry Pi. - pkgsCross = lib.mapAttrs (n: crossSystem: nixpkgsFun { inherit crossSystem; }) lib.systems.examples; + pkgsCross = lib.mapAttrs ( + n: crossSystem: nixpkgsFun { crossSystem = stdenv.hostPlatform.override crossSystem; } + ) lib.systems.examples; pkgsLLVM = mkPkgs "pkgsLLVM" { # Bootstrap a cross stdenv using the LLVM toolchain. # This is currently not possible when compiling natively, # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + crossSystem = stdenv.hostPlatform.override { useLLVM = true; linker = "lld"; }; @@ -253,7 +256,7 @@ let # Bootstrap a cross stdenv using LLVM libc. # This is currently not possible when compiling natively, # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + crossSystem = stdenv.hostPlatform.override { config = lib.systems.parse.tripleFromSystem (makeLLVMParsedPlatform stdenv.hostPlatform.parsed); libc = "llvm"; }; @@ -263,7 +266,7 @@ let # Bootstrap a cross stdenv using the Aro C compiler. # This is currently not possible when compiling natively, # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + crossSystem = stdenv.hostPlatform.override { useArocc = true; linker = "lld"; }; @@ -273,7 +276,7 @@ let # Bootstrap a cross stdenv using the Zig toolchain. # This is currently not possible when compiling natively, # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + crossSystem = stdenv.hostPlatform.override { useZig = true; linker = "lld"; }; @@ -285,7 +288,7 @@ let pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkPkgs "pkgsMusl" { - ${if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" else "crossSystem"} = { + ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override { config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); }; } @@ -297,7 +300,7 @@ let pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkPkgs "pkgsi686Linux" { - ${if stdenv.hostPlatform == stdenv.buildPlatform then "localSystem" else "crossSystem"} = { + ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override { config = lib.systems.parse.tripleFromSystem ( stdenv.hostPlatform.parsed // { @@ -313,9 +316,9 @@ let pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkPkgs "pkgsx86_64Darwin" { - localSystem = { + ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override { config = lib.systems.parse.tripleFromSystem ( - stdenv.hostPlatform.parsed + stdenv.buildPlatform.parsed // { cpu = lib.systems.parse.cpuTypes.x86_64; } @@ -326,30 +329,33 @@ let throw "x86_64 Darwin package set can only be used on Darwin systems."; # If already linux: the same package set unaltered - # Otherwise, return a natively built linux package set for the current cpu architecture string. + # Otherwise, return a linux package set for the current cpu architecture string. + # ABI and other details will be set to the default for the cpu/os pair. pkgsLinux = if stdenv.hostPlatform.isLinux then self else mkPkgs "pkgsLinux" { - localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; + ${if isNative then "localSystem" else "crossSystem"} = + lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; }; # Fully static packages. # Currently uses Musl on Linux (couldn’t get static glibc to work). pkgsStatic = mkPkgs "pkgsStatic" { - crossSystem = { - isStatic = true; - config = lib.systems.parse.tripleFromSystem ( - if stdenv.hostPlatform.isLinux then - makeMuslParsedPlatform stdenv.hostPlatform.parsed - else - stdenv.hostPlatform.parsed - ); - gcc = - lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } - // stdenv.hostPlatform.gcc or { }; - }; + crossSystem = stdenv.hostPlatform.override ( + { + isStatic = true; + } + // lib.optionalAttrs stdenv.hostPlatform.isLinux { + config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); + } + // lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { + gcc = { + abi = "elfv2"; + } // stdenv.hostPlatform.gcc or { }; + } + ); }; pkgsExtraHardening = mkPkgs "pkgsExtraHardening" { From 245125468337be7319dced4a413dd692181c150b Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 14 Dec 2024 19:01:31 +0100 Subject: [PATCH 6/7] pkgs/top-level: refactor mkHybridPkgs --- pkgs/top-level/stage.nix | 50 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index bd4a2ac8f5aba..8a3407bd15a16 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -231,6 +231,12 @@ let ++ overlays; } ); + # This is only cross when we are already cross, otherwise local. + mkHybridPkgs = + name: hybridAttrs: + mkPkgs name { + ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override hybridAttrs; + }; in self: super: { # This maps each entry in lib.systems.examples to its own package @@ -287,10 +293,8 @@ let # supported. 32-bit is also not supported. pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then - mkPkgs "pkgsMusl" { - ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override { - config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); - }; + mkHybridPkgs "pkgsMusl" { + config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); } else throw "Musl libc only supports 64-bit Linux systems."; @@ -299,15 +303,13 @@ let # Used by wine, firefox with debugging version of Flash, ... pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then - mkPkgs "pkgsi686Linux" { - ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override { - config = lib.systems.parse.tripleFromSystem ( - stdenv.hostPlatform.parsed - // { - cpu = lib.systems.parse.cpuTypes.i686; - } - ); - }; + mkHybridPkgs "pkgsi686Linux" { + config = lib.systems.parse.tripleFromSystem ( + stdenv.hostPlatform.parsed + // { + cpu = lib.systems.parse.cpuTypes.i686; + } + ); } else throw "i686 Linux package set can only be used with the x86 family."; @@ -315,15 +317,13 @@ let # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then - mkPkgs "pkgsx86_64Darwin" { - ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override { - config = lib.systems.parse.tripleFromSystem ( - stdenv.buildPlatform.parsed - // { - cpu = lib.systems.parse.cpuTypes.x86_64; - } - ); - }; + mkHybridPkgs "pkgsx86_64Darwin" { + config = lib.systems.parse.tripleFromSystem ( + stdenv.hostPlatform.parsed + // { + cpu = lib.systems.parse.cpuTypes.x86_64; + } + ); } else throw "x86_64 Darwin package set can only be used on Darwin systems."; @@ -335,9 +335,9 @@ let if stdenv.hostPlatform.isLinux then self else - mkPkgs "pkgsLinux" { - ${if isNative then "localSystem" else "crossSystem"} = - lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; + mkHybridPkgs "pkgsLinux" { + config = lib.systems.parse.tripleFromSystem (lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux") + .parsed; }; # Fully static packages. From 3d17de9242d0d9769a96329dc90836d98fcd8e84 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 21 Dec 2024 16:42:40 +0100 Subject: [PATCH 7/7] pkgs/top-level: fix composing "native cross" package sets When using pkgsCross with a system that ends up the same as the localSystem, then modifications for package sets like pksgMusl need to be done for **both** localSystem and crossSystem. Consider the following on x86_64-linux: pkgsCross.gnu64.pkgsMusl Before this change, this would result in a musl buildPlatform, but a gnu hostPlatform. This breaks the promise of "stacking" package sets on top of each other. After this change, it results in a musl buildPlatform and a musl hostPlatform. This works better. One could expect this to result in the same as pkgsCross.musl64, i.e. a gnu buildPlatform and a musl hostPlatform, however I couldn't get this to work without increasing memory usage for ci/eval by many, many GB. This is caused by usage of pkgsi686Linux inside the main package set, which follows the same hybrid pattern. --- pkgs/test/top-level/stage.nix | 5 ++--- pkgs/top-level/stage.nix | 11 ++++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkgs/test/top-level/stage.nix b/pkgs/test/top-level/stage.nix index 9b21871db4848..a542b6a82a694 100644 --- a/pkgs/test/top-level/stage.nix +++ b/pkgs/test/top-level/stage.nix @@ -104,9 +104,8 @@ assert isComposable "pkgsStatic"; assert isComposable "pkgsi686Linux"; # Special cases regarding buildPlatform vs hostPlatform -# TODO: fails -# assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl); -# assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32); +assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl); +assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32); assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); assert discardEvaluationErrors ( pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64 diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 8a3407bd15a16..d16c559b7a349 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -232,11 +232,16 @@ let } ); # This is only cross when we are already cross, otherwise local. + # For the case of "native cross", i.e. pkgsCross.gnu64 on a x86_64-linux system, we need to adjust **both** + # localSystem **and** crossSystem, otherwise they're out of sync. mkHybridPkgs = name: hybridAttrs: - mkPkgs name { - ${if isNative then "localSystem" else "crossSystem"} = stdenv.hostPlatform.override hybridAttrs; - }; + mkPkgs name ( + let + newSystem = stdenv.hostPlatform.override hybridAttrs; + in + { crossSystem = newSystem; } // lib.optionalAttrs isNative { localSystem = newSystem; } + ); in self: super: { # This maps each entry in lib.systems.examples to its own package