diff --git a/ci/eval/attrpaths.nix b/ci/eval/attrpaths.nix new file mode 100644 index 0000000000000..6938e5dfe3cb1 --- /dev/null +++ b/ci/eval/attrpaths.nix @@ -0,0 +1,83 @@ +# This expression will, as efficiently as possible, dump a +# *superset* of all attrpaths of derivations which might be +# part of a release on *any* platform. +# +# This expression runs single-threaded under all current Nix +# implementations, but much faster and with much less memory +# used than ./outpaths.nix itself. +# +# Once you have the list of attrnames you can split it up into +# $NUM_CORES batches and evaluate the outpaths separately for each +# batch, in parallel. +# +# To dump the attrnames: +# +# nix-instantiate --eval --strict --json ci/eval/attrpaths.nix -A names +# +{ + lib ? import (path + "/lib"), + trace ? false, + path ? ./../.., +}: +let + + # TODO: Use mapAttrsToListRecursiveCond when this PR lands: + # https://github.com/NixOS/nixpkgs/pull/395160 + justAttrNames = + path: value: + let + result = + if path == [ "AAAAAASomeThingsFailToEvaluate" ] || !(lib.isAttrs value) then + [ ] + else if lib.isDerivation value then + [ path ] + else + lib.pipe value [ + (lib.mapAttrsToList ( + name: value: + lib.addErrorContext "while evaluating package set attribute path '${ + lib.showAttrPath (path ++ [ name ]) + }'" (justAttrNames (path ++ [ name ]) value) + )) + lib.concatLists + ]; + in + lib.traceIf trace "** ${lib.showAttrPath path}" result; + + outpaths = import ./outpaths.nix { + inherit path; + attrNamesOnly = true; + }; + + paths = [ + # Some of the following are based on variants, which are disabled with `attrNamesOnly = true`. + # Until these have been removed from release.nix / hydra, we manually add them to the list. + [ + "pkgsLLVM" + "stdenv" + ] + [ + "pkgsArocc" + "stdenv" + ] + [ + "pkgsZig" + "stdenv" + ] + [ + "pkgsStatic" + "stdenv" + ] + [ + "pkgsMusl" + "stdenv" + ] + ] + ++ justAttrNames [ ] outpaths; + + names = map lib.showAttrPath paths; + +in +{ + inherit paths names; +} diff --git a/pkgs/top-level/release-outpaths-parallel.nix b/ci/eval/chunk.nix similarity index 82% rename from pkgs/top-level/release-outpaths-parallel.nix rename to ci/eval/chunk.nix index 46d10c6a95041..43c684074602f 100644 --- a/pkgs/top-level/release-outpaths-parallel.nix +++ b/ci/eval/chunk.nix @@ -1,5 +1,4 @@ -# This file works in tandem with ../../ci/eval/default.nix -# It turns ./release-outpaths.nix into chunks of a fixed size +# This turns ./outpaths.nix into chunks of a fixed size. { lib ? import ../../lib, path ? ../.., @@ -7,7 +6,6 @@ attrpathFile, chunkSize, myChunk, - checkMeta, includeBroken, systems, }: @@ -16,9 +14,9 @@ let attrpaths = lib.importJSON attrpathFile; myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths; - unfiltered = import ./release-outpaths.nix { + unfiltered = import ./outpaths.nix { inherit path; - inherit checkMeta includeBroken systems; + inherit includeBroken systems; }; # Turns the unfiltered recursive attribute set into one that is limited to myAttrpaths diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix index 9060d357d6620..e632e87e2fee9 100644 --- a/ci/eval/compare/default.nix +++ b/ci/eval/compare/default.nix @@ -103,7 +103,6 @@ let ) rebuildsByKernel # Set the "11.by: package-maintainer" label to whether all packages directly # changed are maintained by the PR's author. - # (https://github.com/NixOS/ofborg/blob/df400f44502d4a4a80fa283d33f2e55a4e43ee90/ofborg/src/tagger.rs#L83-L88) // { "11.by: package-maintainer" = maintainers ? ${githubAuthorId} diff --git a/ci/eval/compare/maintainers.nix b/ci/eval/compare/maintainers.nix index 43ae050acba4f..d862e23efca12 100644 --- a/ci/eval/compare/maintainers.nix +++ b/ci/eval/compare/maintainers.nix @@ -1,7 +1,6 @@ { lib, }: -# Almost directly vendored from https://github.com/NixOS/ofborg/blob/5a4e743f192fb151915fcbe8789922fa401ecf48/ofborg/src/maintainers.nix { changedattrs, changedpathsjson, diff --git a/ci/eval/default.nix b/ci/eval/default.nix index 14005ea401afb..a379594dbc6d2 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -26,6 +26,11 @@ let root = ../..; fileset = unions ( map (lib.path.append ../..) [ + ".version" + "ci/supportedSystems.json" + "ci/eval/attrpaths.nix" + "ci/eval/chunk.nix" + "ci/eval/outpaths.nix" "default.nix" "doc" "lib" @@ -33,8 +38,6 @@ let "modules" "nixos" "pkgs" - ".version" - "ci/supportedSystems.json" ] ); }; @@ -60,7 +63,7 @@ let export GC_INITIAL_HEAP_SIZE=4g command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \ nix-instantiate --eval --strict --json --show-trace \ - "$src/pkgs/top-level/release-attrpaths-superset.nix" \ + "$src/ci/eval/attrpaths.nix" \ -A paths \ -I "$src" \ --option restrict-eval true \ @@ -78,7 +81,6 @@ let attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json", # The number of attributes per chunk, see ./README.md for more info. chunkSize ? 5000, - checkMeta ? true, # Don't try to eval packages marked as broken. includeBroken ? false, @@ -99,7 +101,7 @@ let set +e command time -o "$outputDir/timestats/$myChunk" \ -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \ - nix-env -f "${nixpkgs}/pkgs/top-level/release-outpaths-parallel.nix" \ + nix-env -f "${nixpkgs}/ci/eval/chunk.nix" \ --eval-system "$system" \ --option restrict-eval true \ --option allow-import-from-derivation false \ @@ -110,7 +112,6 @@ let --arg myChunk "$myChunk" \ --arg attrpathFile "${attrpathFile}" \ --arg systems "[ \"$system\" ]" \ - --arg checkMeta ${lib.boolToString checkMeta} \ --arg includeBroken ${lib.boolToString includeBroken} \ -I ${nixpkgs} \ -I ${attrpathFile} \ diff --git a/pkgs/top-level/release-outpaths.nix b/ci/eval/outpaths.nix similarity index 85% rename from pkgs/top-level/release-outpaths.nix rename to ci/eval/outpaths.nix index 18840848c1857..4efe8d1dc0610 100644 --- a/pkgs/top-level/release-outpaths.nix +++ b/ci/eval/outpaths.nix @@ -1,19 +1,16 @@ #!/usr/bin/env nix-shell # When using as a callable script, passing `--argstr path some/path` overrides $PWD. -#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path --arg checkMeta true -f pkgs/top-level/release-outpaths.nix" +#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path -f ci/eval/outpaths.nix" -# Vendored from: -# https://raw.githubusercontent.com/NixOS/ofborg/74f38efa7ef6f0e8e71ec3bfc675ae4fb57d7491/ofborg/src/outpaths.nix { - checkMeta, includeBroken ? true, # set this to false to exclude meta.broken packages from the output path ? ./../.., - # used by pkgs/top-level/release-attrnames-superset.nix + # used by ./attrpaths.nix attrNamesOnly ? false, # Set this to `null` to build for builtins.currentSystem only - systems ? builtins.fromJSON (builtins.readFile ../../ci/supportedSystems.json), + systems ? builtins.fromJSON (builtins.readFile ../supportedSystems.json), }: let lib = import (path + "/lib"); @@ -30,7 +27,7 @@ let allowUnfree = true; allowInsecurePredicate = x: true; allowVariants = !attrNamesOnly; - checkMeta = checkMeta; + checkMeta = true; handleEvalIssue = reason: errormsg: diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index dd1e30d09fa5a..cf7327c8242dd 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -152,9 +152,7 @@ with pkgs; php = recurseIntoAttrs (callPackages ./php { }); - pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { }) // { - __recurseIntoDerivationForReleaseJobs = true; - }; + pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { }); buildRustCrate = recurseIntoAttrs (callPackage ../build-support/rust/build-rust-crate/test { }); importCargoLock = recurseIntoAttrs (callPackage ../build-support/rust/test/import-cargo-lock { }); diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 22a0064925060..b6d11ebc41244 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5177,10 +5177,7 @@ with pkgs; haskell.packages.ghc96 else haskell.packages.ghc98 - ) - // { - __recurseIntoDerivationForReleaseJobs = true; - }; + ); # haskellPackages.ghc is build->host (it exposes the compiler used to build the # set, similarly to stdenv.cc), but pkgs.ghc should be host->target to be more @@ -8868,22 +8865,18 @@ with pkgs; } ); - libsForQt5 = - (recurseIntoAttrs ( - import ./qt5-packages.nix { - inherit - lib - config - __splicedPackages - makeScopeWithSplicing' - generateSplicesForMkScope - pkgsHostTarget - ; - } - )) - // { - __recurseIntoDerivationForReleaseJobs = true; - }; + libsForQt5 = recurseIntoAttrs ( + import ./qt5-packages.nix { + inherit + lib + config + __splicedPackages + makeScopeWithSplicing' + generateSplicesForMkScope + pkgsHostTarget + ; + } + ); # plasma5Packages maps to the Qt5 packages set that is used to build the plasma5 desktop plasma5Packages = libsForQt5; diff --git a/pkgs/top-level/linux-kernels.nix b/pkgs/top-level/linux-kernels.nix index 92e7265ed62b7..f03f4bb76b8d6 100644 --- a/pkgs/top-level/linux-kernels.nix +++ b/pkgs/top-level/linux-kernels.nix @@ -789,7 +789,6 @@ in linux_libre = recurseIntoAttrs (packagesFor kernels.linux_libre); linux_latest_libre = recurseIntoAttrs (packagesFor kernels.linux_latest_libre); - __recurseIntoDerivationForReleaseJobs = true; } // lib.optionalAttrs config.allowAliases { diff --git a/pkgs/top-level/pkg-config/tests.nix b/pkgs/top-level/pkg-config/tests.nix index 345283f83d816..4f3e6c8fb7cfd 100644 --- a/pkgs/top-level/pkg-config/tests.nix +++ b/pkgs/top-level/pkg-config/tests.nix @@ -22,7 +22,5 @@ let }; in lib.recurseIntoAttrs { - defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { } // { - __recurseIntoDerivationForReleaseJobs = true; - }; + defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { }; } diff --git a/pkgs/top-level/release-attrpaths-superset.nix b/pkgs/top-level/release-attrpaths-superset.nix deleted file mode 100644 index 80544ced51656..0000000000000 --- a/pkgs/top-level/release-attrpaths-superset.nix +++ /dev/null @@ -1,114 +0,0 @@ -# This expression will, as efficiently as possible, dump a -# *superset* of all attrpaths of derivations which might be -# part of a release on *any* platform. -# -# Both this expression and what ofborg uses (release-outpaths.nix) -# are essentially single-threaded (under the current cppnix -# implementation). -# -# This expression runs much, much, much faster and uses much, much -# less memory than the ofborg script by skipping the -# platform-relevance checks. The ofborg outpaths.nix script takes -# half an hour on a 3ghz core and peaks at 60gbytes of memory; this -# expression runs on the same machine in 44 seconds with peak memory -# usage of 5gbytes. -# -# Once you have the list of attrnames you can split it up into -# $NUM_CORES batches and run the platform checks separately for each -# batch, in parallel. -# -# To dump the attrnames: -# -# nix-instantiate --eval --strict --json pkgs/top-level/release-attrpaths-superset.nix -A names -# -{ - lib ? import (path + "/lib"), - trace ? false, - checkMeta ? true, - path ? ./../.., -}: -let - - # The intended semantics are that an attrpath rooted at pkgs is - # part of the (unfiltered) release jobset iff both of the following - # are true: - # - # 1. The attrpath leads to a value for which lib.isDerivation is true - # - # 2. Any proper prefix of the attrpath at which lib.isDerivation - # is true also has __recurseIntoDerivationForReleaseJobs=true. - # - # The second condition is unfortunately necessary because there are - # Hydra release jobnames which have proper prefixes which are - # attrnames of derivations (!). We should probably restructure - # the job tree so that this is not the case. - # - justAttrNames = - path: value: - let - result = - if path == [ "AAAAAASomeThingsFailToEvaluate" ] then - [ ] - else if - lib.isDerivation value - && - # in some places we have *derivations* with jobsets as subattributes, ugh - !(value.__recurseIntoDerivationForReleaseJobs or false) - then - [ path ] - - # Even wackier case: we have meta.broken==true jobs with - # !meta.broken jobs as subattributes with license=unfree, and - # check-meta.nix won't throw an "unfree" failure because the - # enclosing derivation is marked broken. Yeah. Bonkers. - # We should just forbid jobsets enclosed by derivations. - else if lib.isDerivation value && !value.meta.available then - [ ] - - else if !(lib.isAttrs value) then - [ ] - else - lib.pipe value [ - (builtins.mapAttrs ( - name: value: - builtins.addErrorContext "while evaluating package set attribute path '${ - lib.showAttrPath (path ++ [ name ]) - }'" (justAttrNames (path ++ [ name ]) value) - )) - builtins.attrValues - builtins.concatLists - ]; - in - if !trace then result else lib.trace "** ${lib.concatStringsSep "." path}" result; - - releaseOutpaths = import ./release-outpaths.nix { - inherit checkMeta; - attrNamesOnly = true; - inherit path; - }; - - paths = [ - # I am not entirely sure why these three packages end up in - # the Hydra jobset. But they do, and they don't meet the - # criteria above, so at the moment they are special-cased. - [ - "pkgsLLVM" - "stdenv" - ] - [ - "pkgsStatic" - "stdenv" - ] - [ - "pkgsMusl" - "stdenv" - ] - ] - ++ justAttrNames [ ] releaseOutpaths; - - names = map (path: (lib.concatStringsSep "." path)) paths; - -in -{ - inherit paths names; -} diff --git a/pkgs/top-level/release-python.nix b/pkgs/top-level/release-python.nix index 18774202503fa..b02b749577a9a 100644 --- a/pkgs/top-level/release-python.nix +++ b/pkgs/top-level/release-python.nix @@ -36,11 +36,7 @@ let res = builtins.tryEval ( if isDerivation value then value.meta.isBuildPythonPackage or [ ] - else if - value.recurseForDerivations or false - || value.recurseForRelease or false - || value.__recurseIntoDerivationForReleaseJobs or false - then + else if value.recurseForDerivations or false || value.recurseForRelease or false then packagePython value else [ ] diff --git a/pkgs/top-level/release-unfree-redistributable.nix b/pkgs/top-level/release-unfree-redistributable.nix index 45940c4d02ccb..cf5c8a4d4987e 100644 --- a/pkgs/top-level/release-unfree-redistributable.nix +++ b/pkgs/top-level/release-unfree-redistributable.nix @@ -69,11 +69,7 @@ let value.meta.hydraPlatforms or (lib.subtractLists (value.meta.badPlatforms or [ ]) (value.meta.platforms or [ "x86_64-linux" ])) ) - else if - value.recurseForDerivations or false - || value.recurseForRelease or false - || value.__recurseIntoDerivationForReleaseJobs or false - then + else if value.recurseForDerivations or false || value.recurseForRelease or false then # Recurse packagesWith attrPath cond value else diff --git a/pkgs/top-level/release.nix b/pkgs/top-level/release.nix index f087876479c84..d6e6184a44e4d 100644 --- a/pkgs/top-level/release.nix +++ b/pkgs/top-level/release.nix @@ -57,12 +57,10 @@ # resulting tree of attributes to *not* have a ".${system}" # suffixed upon every job name like Hydra expects. # - # This flag exists mainly for use by - # pkgs/top-level/release-attrnames-superset.nix; see that file for - # full details. The exact behavior of this flag may change; it - # should be considered an internal implementation detail of - # pkgs/top-level/. - # + # This flag exists mainly for use by ci/eval/attrpaths.nix; see + # that file for full details. The exact behavior of this flag + # may change; it should be considered an internal implementation + # detail of ci/eval. attrNamesOnly ? false, }: