ci/eval/compare: add 10.rebuild-nixos-tests label

ci/eval/compare/default.nix

@@ -103,9 +103,15 @@ let
           // lib.mapAttrs' (
             kernel: rebuilds: lib.nameValuePair "10.rebuild-${kernel}-stdenv" (lib.elem "stdenv" rebuilds)
           ) rebuildsByKernel
-          # Set the "11.by: package-maintainer" label to whether all packages directly
-          # changed are maintained by the PR's author.
           // {
+            "10.rebuild-nixos-tests" =
+              lib.elem "nixosTests.simple" (extractPackageNames diffAttrs.rebuilds)
+              &&
+                # Only set this label when no other label with indication for staging has been set.
+                # This avoids confusion whether to target staging or batch this with kernel updates.
+                lib.last (lib.sort lib.lessThan (lib.attrValues rebuildCountByKernel)) <= 500;
+            # Set the "11.by: package-maintainer" label to whether all packages directly
+            # changed are maintained by the PR's author.
             "11.by: package-maintainer" =
               maintainers ? ${githubAuthorId}
               && lib.all (lib.flip lib.elem maintainers.${githubAuthorId}) (

ci/eval/outpaths.nix

@@ -14,7 +14,8 @@
 }:
 let
   lib = import (path + "/lib");
-  hydraJobs =
+
+  nixpkgsJobs =
     import (path + "/pkgs/top-level/release.nix")
       # Compromise: accuracy vs. resources needed for evaluation.
       {
@@ -59,9 +60,15 @@ let
           __allowFileset = false;
         };
       };
+
+  nixosJobs = import (path + "/nixos/release.nix") {
+    inherit attrNamesOnly;
+    supportedSystems = if systems == null then [ builtins.currentSystem ] else systems;
+  };
+
   recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
 
-  # hydraJobs leaves recurseForDerivations as empty attrmaps;
+  # release-lib leaves recurseForDerivations as empty attrmaps;
   # that would break nix-env and we also need to recurse everywhere.
   tweak = lib.mapAttrs (
     name: val:
@@ -87,4 +94,9 @@ let
   ];
 
 in
-tweak (builtins.removeAttrs hydraJobs blacklist)
+tweak (
+  (builtins.removeAttrs nixpkgsJobs blacklist)
+  // {
+    nixosTests.simple = nixosJobs.tests.simple;
+  }
+)

nixos/release.nix

@@ -12,6 +12,16 @@ with import ../lib;
     "aarch64-linux"
   ],
   configuration ? { },
+
+  # This flag, if set to true, causes the resulting tree of attributes
+  # to *not* have a ".${system}" suffixed upon every job name like Hydra
+  # expects. So far, this is only implemented for `tests`.
+  #
+  # This flag exists mainly for use by ci/eval/attrpaths.nix; see
+  # that file for full details.  The exact behavior of this flag
+  # may change; it should be considered an internal implementation
+  # detail of ci/eval.
+  attrNamesOnly ? false,
 }:
 
 with import ../pkgs/top-level/release-lib.nix { inherit supportedSystems; };
@@ -31,23 +41,35 @@ let
     import ./tests/all-tests.nix {
       inherit system;
       pkgs = import ./.. { inherit system; };
-      callTest = config: {
-        ${system} = hydraJob config.test;
-      };
+      callTest =
+        config:
+        if attrNamesOnly then
+          hydraJob config.test
+        else
+          {
+            ${system} = hydraJob config.test;
+          };
     }
     // {
       # for typechecking of the scripts and evaluation of
       # the nodes, without running VMs.
       allDrivers = import ./tests/all-tests.nix {
         inherit system;
         pkgs = import ./.. { inherit system; };
-        callTest = config: {
-          ${system} = hydraJob config.driver;
-        };
+        callTest =
+          config:
+          if attrNamesOnly then
+            hydraJob config.test
+          else
+            {
+              ${system} = hydraJob config.driver;
+            };
       };
     };
 
-  allTests = foldAttrs recursiveUpdate { } (map allTestsForSystem supportedSystems);
+  allTests = foldAttrs recursiveUpdate { } (
+    map allTestsForSystem (if attrNamesOnly then [ (head supportedSystems) ] else supportedSystems)
+  );
 
   pkgs = import ./.. { system = "x86_64-linux"; };
 

nixos/tests/simple.nix

@@ -1,15 +1,7 @@
-{ pkgs, ... }:
 {
   name = "simple";
-  meta = with pkgs.lib.maintainers; {
-    maintainers = [ ];
-  };
 
-  nodes.machine =
-    { ... }:
-    {
-      imports = [ ../modules/profiles/minimal.nix ];
-    };
+  nodes.machine = { };
 
   testScript = ''
     start_all()