diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix
index 46507a492a594..5807f8ef52f7f 100644
--- a/ci/eval/compare/default.nix
+++ b/ci/eval/compare/default.nix
@@ -73,12 +73,11 @@ let
     ;
 
   # Attrs
-  # - keys: "added", "changed" and "removed"
+  # - keys: "added", "changed", "removed" and "rebuilds"
   # - values: lists of `packagePlatformPath`s
   diffAttrs = builtins.fromJSON (builtins.readFile "${combinedDir}/combined-diff.json");
 
-  rebuilds = diffAttrs.added ++ diffAttrs.changed;
-  rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;
+  rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs diffAttrs.rebuilds;
 
   changed-paths =
     let
@@ -90,7 +89,7 @@ let
     in
     writeText "changed-paths.json" (
       builtins.toJSON {
-        attrdiff = lib.mapAttrs (_: extractPackageNames) diffAttrs;
+        attrdiff = lib.mapAttrs (_: extractPackageNames) { inherit (diffAttrs) added changed removed; };
         inherit
           rebuildsByPlatform
           rebuildsByKernel
diff --git a/ci/eval/default.nix b/ci/eval/default.nix
index 56e07f8da6700..21285757a2770 100644
--- a/ci/eval/default.nix
+++ b/ci/eval/default.nix
@@ -218,7 +218,8 @@ let
           reduce .[] as $item ({}; {
             added: (.added + $item.added),
             changed: (.changed + $item.changed),
-            removed: (.removed + $item.removed)
+            removed: (.removed + $item.removed),
+            rebuilds: (.rebuilds + $item.rebuilds)
           })
         ' > $out/combined-diff.json
 
diff --git a/ci/eval/diff.nix b/ci/eval/diff.nix
index 629b4f8d3a6a0..785b0d784308b 100644
--- a/ci/eval/diff.nix
+++ b/ci/eval/diff.nix
@@ -18,13 +18,20 @@ let
       added: [ <keys only in the second object> ],
       removed: [ <keys only in the first object> ],
       changed: [ <keys with different values between the two objects> ],
+      rebuilds: [ <keys in the second object with values not present at all in first object> ],
     }
   */
   diff =
+    old: new:
     let
       filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs);
+      oldOutputs = lib.pipe old [
+        (lib.mapAttrsToList (_: lib.attrValues))
+        lib.concatLists
+        (lib.flip lib.genAttrs (_: true))
+      ];
     in
-    old: new: {
+    {
       added = filterKeys (n: _: !(old ? ${n})) new;
       removed = filterKeys (n: _: !(new ? ${n})) old;
       changed = filterKeys (
@@ -35,6 +42,16 @@ let
         # Filter out attributes that are the same as the new value
         && (v != (new.${n}))
       ) old;
+      # A "rebuild" is every attrpath ...
+      rebuilds = filterKeys (
+        _: pkg:
+        # ... that has at least one output ...
+        lib.any (
+          output:
+          # ... which has not been built in "old" already.
+          !(oldOutputs ? ${output})
+        ) (lib.attrValues pkg)
+      ) new;
     };
 
   getAttrs =