top-level/release-outpaths: move to ci/eval & refactor

ci/eval/attrpaths.nix

@@ -0,0 +1,83 @@
+# This expression will, as efficiently as possible, dump a
+# *superset* of all attrpaths of derivations which might be
+# part of a release on *any* platform.
+#
+# This expression runs single-threaded under all current Nix
+# implementations, but much faster and with much less memory
+# used than ./outpaths.nix itself.
+#
+# Once you have the list of attrnames you can split it up into
+# $NUM_CORES batches and evaluate the outpaths separately for each
+# batch, in parallel.
+#
+# To dump the attrnames:
+#
+#   nix-instantiate --eval --strict --json ci/eval/attrpaths.nix -A names
+#
+{
+  lib ? import (path + "/lib"),
+  trace ? false,
+  path ? ./../..,
+}:
+let
+
+  # TODO: Use mapAttrsToListRecursiveCond when this PR lands:
+  # https://github.com/NixOS/nixpkgs/pull/395160
+  justAttrNames =
+    path: value:
+    let
+      result =
+        if path == [ "AAAAAASomeThingsFailToEvaluate" ] || !(lib.isAttrs value) then
+          [ ]
+        else if lib.isDerivation value then
+          [ path ]
+        else
+          lib.pipe value [
+            (lib.mapAttrsToList (
+              name: value:
+              lib.addErrorContext "while evaluating package set attribute path '${
+                lib.showAttrPath (path ++ [ name ])
+              }'" (justAttrNames (path ++ [ name ]) value)
+            ))
+            lib.concatLists
+          ];
+    in
+    lib.traceIf trace "** ${lib.showAttrPath path}" result;
+
+  outpaths = import ./outpaths.nix {
+    inherit path;
+    attrNamesOnly = true;
+  };
+
+  paths = [
+    # Some of the following are based on variants, which are disabled with `attrNamesOnly = true`.
+    # Until these have been removed from release.nix / hydra, we manually add them to the list.
+    [
+      "pkgsLLVM"
+      "stdenv"
+    ]
+    [
+      "pkgsArocc"
+      "stdenv"
+    ]
+    [
+      "pkgsZig"
+      "stdenv"
+    ]
+    [
+      "pkgsStatic"
+      "stdenv"
+    ]
+    [
+      "pkgsMusl"
+      "stdenv"
+    ]
+  ]
+  ++ justAttrNames [ ] outpaths;
+
+  names = map lib.showAttrPath paths;
+
+in
+{
+  inherit paths names;
+}

pkgs/top-level/release-outpaths-parallel.nix → ci/eval/chunk.nix

@@ -1,13 +1,11 @@
-# This file works in tandem with ../../ci/eval/default.nix
-# It turns ./release-outpaths.nix into chunks of a fixed size
+# This turns ./outpaths.nix into chunks of a fixed size.
 {
   lib ? import ../../lib,
   path ? ../..,
   # The file containing all available attribute paths, which are split into chunks here
   attrpathFile,
   chunkSize,
   myChunk,
-  checkMeta,
   includeBroken,
   systems,
 }:
@@ -16,9 +14,9 @@ let
   attrpaths = lib.importJSON attrpathFile;
   myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths;
 
-  unfiltered = import ./release-outpaths.nix {
+  unfiltered = import ./outpaths.nix {
     inherit path;
-    inherit checkMeta includeBroken systems;
+    inherit includeBroken systems;
   };
 
   # Turns the unfiltered recursive attribute set into one that is limited to myAttrpaths

ci/eval/compare/default.nix

@@ -103,7 +103,6 @@ let
           ) rebuildsByKernel
           # Set the "11.by: package-maintainer" label to whether all packages directly
           # changed are maintained by the PR's author.
-          # (https://github.com/NixOS/ofborg/blob/df400f44502d4a4a80fa283d33f2e55a4e43ee90/ofborg/src/tagger.rs#L83-L88)
           // {
             "11.by: package-maintainer" =
               maintainers ? ${githubAuthorId}

ci/eval/compare/maintainers.nix

@@ -1,7 +1,6 @@
 {
   lib,
 }:
-# Almost directly vendored from https://github.com/NixOS/ofborg/blob/5a4e743f192fb151915fcbe8789922fa401ecf48/ofborg/src/maintainers.nix
 {
   changedattrs,
   changedpathsjson,

ci/eval/default.nix

@@ -26,15 +26,18 @@ let
       root = ../..;
       fileset = unions (
         map (lib.path.append ../..) [
+          ".version"
+          "ci/supportedSystems.json"
+          "ci/eval/attrpaths.nix"
+          "ci/eval/chunk.nix"
+          "ci/eval/outpaths.nix"
           "default.nix"
           "doc"
           "lib"
           "maintainers"
           "modules"
           "nixos"
           "pkgs"
-          ".version"
-          "ci/supportedSystems.json"
         ]
       );
     };
@@ -60,7 +63,7 @@ let
         export GC_INITIAL_HEAP_SIZE=4g
         command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \
           nix-instantiate --eval --strict --json --show-trace \
-            "$src/pkgs/top-level/release-attrpaths-superset.nix" \
+            "$src/ci/eval/attrpaths.nix" \
             -A paths \
             -I "$src" \
             --option restrict-eval true \
@@ -78,7 +81,6 @@ let
       attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json",
       # The number of attributes per chunk, see ./README.md for more info.
       chunkSize ? 5000,
-      checkMeta ? true,
 
       # Don't try to eval packages marked as broken.
       includeBroken ? false,
@@ -99,7 +101,7 @@ let
         set +e
         command time -o "$outputDir/timestats/$myChunk" \
           -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
-          nix-env -f "${nixpkgs}/pkgs/top-level/release-outpaths-parallel.nix" \
+          nix-env -f "${nixpkgs}/ci/eval/chunk.nix" \
           --eval-system "$system" \
           --option restrict-eval true \
           --option allow-import-from-derivation false \
@@ -110,7 +112,6 @@ let
           --arg myChunk "$myChunk" \
           --arg attrpathFile "${attrpathFile}" \
           --arg systems "[ \"$system\" ]" \
-          --arg checkMeta ${lib.boolToString checkMeta} \
           --arg includeBroken ${lib.boolToString includeBroken} \
           -I ${nixpkgs} \
           -I ${attrpathFile} \

pkgs/top-level/release-outpaths.nix → ci/eval/outpaths.nix

@@ -1,19 +1,16 @@
 #!/usr/bin/env nix-shell
 # When using as a callable script, passing `--argstr path some/path` overrides $PWD.
-#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path --arg checkMeta true -f pkgs/top-level/release-outpaths.nix"
+#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path -f ci/eval/outpaths.nix"
 
-# Vendored from:
-#   https://raw.githubusercontent.com/NixOS/ofborg/74f38efa7ef6f0e8e71ec3bfc675ae4fb57d7491/ofborg/src/outpaths.nix
 {
-  checkMeta,
   includeBroken ? true, # set this to false to exclude meta.broken packages from the output
   path ? ./../..,
 
-  # used by pkgs/top-level/release-attrnames-superset.nix
+  # used by ./attrpaths.nix
   attrNamesOnly ? false,
 
   # Set this to `null` to build for builtins.currentSystem only
-  systems ? builtins.fromJSON (builtins.readFile ../../ci/supportedSystems.json),
+  systems ? builtins.fromJSON (builtins.readFile ../supportedSystems.json),
 }:
 let
   lib = import (path + "/lib");
@@ -30,7 +27,7 @@ let
             allowUnfree = true;
             allowInsecurePredicate = x: true;
             allowVariants = !attrNamesOnly;
-            checkMeta = checkMeta;
+            checkMeta = true;
 
             handleEvalIssue =
               reason: errormsg:

pkgs/test/default.nix

@@ -152,9 +152,7 @@ with pkgs;
 
   php = recurseIntoAttrs (callPackages ./php { });
 
-  pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { }) // {
-    __recurseIntoDerivationForReleaseJobs = true;
-  };
+  pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { });
 
   buildRustCrate = recurseIntoAttrs (callPackage ../build-support/rust/build-rust-crate/test { });
   importCargoLock = recurseIntoAttrs (callPackage ../build-support/rust/test/import-cargo-lock { });

pkgs/top-level/all-packages.nix

@@ -5177,10 +5177,7 @@ with pkgs;
           haskell.packages.ghc96
         else
           haskell.packages.ghc98
-      )
-    // {
-      __recurseIntoDerivationForReleaseJobs = true;
-    };
+      );
 
   # haskellPackages.ghc is build->host (it exposes the compiler used to build the
   # set, similarly to stdenv.cc), but pkgs.ghc should be host->target to be more
@@ -8868,22 +8865,18 @@ with pkgs;
     }
   );
 
-  libsForQt5 =
-    (recurseIntoAttrs (
-      import ./qt5-packages.nix {
-        inherit
-          lib
-          config
-          __splicedPackages
-          makeScopeWithSplicing'
-          generateSplicesForMkScope
-          pkgsHostTarget
-          ;
-      }
-    ))
-    // {
-      __recurseIntoDerivationForReleaseJobs = true;
-    };
+  libsForQt5 = recurseIntoAttrs (
+    import ./qt5-packages.nix {
+      inherit
+        lib
+        config
+        __splicedPackages
+        makeScopeWithSplicing'
+        generateSplicesForMkScope
+        pkgsHostTarget
+        ;
+    }
+  );
 
   # plasma5Packages maps to the Qt5 packages set that is used to build the plasma5 desktop
   plasma5Packages = libsForQt5;

pkgs/top-level/linux-kernels.nix

@@ -789,7 +789,6 @@ in
       linux_libre = recurseIntoAttrs (packagesFor kernels.linux_libre);
 
       linux_latest_libre = recurseIntoAttrs (packagesFor kernels.linux_latest_libre);
-      __recurseIntoDerivationForReleaseJobs = true;
     }
     // lib.optionalAttrs config.allowAliases {
 

pkgs/top-level/pkg-config/tests.nix

@@ -22,7 +22,5 @@ let
   };
 in
 lib.recurseIntoAttrs {
-  defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { } // {
-    __recurseIntoDerivationForReleaseJobs = true;
-  };
+  defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { };
 }