diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
index 90be4670e414b..3a061961a0cdf 100644
--- a/.github/workflows/eval.yml
+++ b/.github/workflows/eval.yml
@@ -168,7 +168,7 @@ jobs:
         run: |
           # Get the latest eval.yml workflow run for the PR's base commit
           if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
-            -f head_sha="$BASE_SHA" \
+            -f head_sha="$BASE_SHA" -f event=push \
             --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
             || [[ -z "$run" ]]; then
             echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison"
@@ -210,7 +210,7 @@ jobs:
             --arg beforeResultDir ./baseResult \
             --arg afterResultDir ./prResult \
             -o comparison
-
+          cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
           # TODO: Request reviews from maintainers for packages whose files are modified in the PR
 
       - name: Upload the combined results
@@ -237,9 +237,33 @@ jobs:
 
       - name: Tagging pull request
         run: |
+          # Get all currently set rebuild labels
           gh api \
-            --method POST \
-            /repos/${{ github.repository }}/issues/${{ github.event.number }}/labels \
-            --input <(jq -c '{ labels: .labels }' comparison/changed-paths.json)
+            /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
+            --jq '.[].name | select(startswith("10.rebuild"))' \
+            | sort > before
+
+          # And the labels that should be there
+          jq -r '.labels[]' comparison/changed-paths.json \
+            | sort > after
+
+          # Remove the ones not needed anymore
+          while read -r toRemove; do
+            echo "Removing label $toRemove"
+            gh api \
+              --method DELETE \
+              /repos/"$REPOSITORY"/issues/"$NUMBER"/labels/"$toRemove"
+          done < <(comm -23 before after)
+
+          # And add the ones that aren't set already
+          while read -r toAdd; do
+            echo "Adding label $toAdd"
+            gh api \
+              --method POST \
+              /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
+              -f "labels[]=$toAdd"
+          done < <(comm -13 before after)
         env:
           GH_TOKEN: ${{ github.token }}
+          REPOSITORY: ${{ github.repository }}
+          NUMBER: ${{ github.event.number }}
diff --git a/ci/eval/default.nix b/ci/eval/default.nix
index 5bd2b48d710d9..ef107d4ce517a 100644
--- a/ci/eval/default.nix
+++ b/ci/eval/default.nix
@@ -5,7 +5,7 @@
   linkFarm,
   time,
   procps,
-  nix,
+  nixVersions,
   jq,
   sta,
 }:
@@ -29,6 +29,8 @@ let
       );
     };
 
+  nix = nixVersions.nix_2_24;
+
   supportedSystems = import ../supportedSystems.nix;
 
   attrpathsSuperset =
@@ -48,8 +50,12 @@ let
         export GC_INITIAL_HEAP_SIZE=4g
         command time -v \
           nix-instantiate --eval --strict --json --show-trace \
-          $src/pkgs/top-level/release-attrpaths-superset.nix -A paths \
-          --arg enableWarnings false > $out/paths.json
+            "$src/pkgs/top-level/release-attrpaths-superset.nix" \
+            -A paths \
+            -I "$src" \
+            --option restrict-eval true \
+            --option allow-import-from-derivation false \
+            --arg enableWarnings false > $out/paths.json
         mv "$supportedSystemsPath" $out/systems.json
       '';
 
@@ -82,6 +88,8 @@ let
         set +e
         command time -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
           nix-env -f "${nixpkgs}/pkgs/top-level/release-attrpaths-parallel.nix" \
+          --option restrict-eval true \
+          --option allow-import-from-derivation false \
           --query --available \
           --no-name --attr-path --out-path \
           --show-trace \
@@ -91,6 +99,8 @@ let
           --arg systems "[ \"$system\" ]" \
           --arg checkMeta ${lib.boolToString checkMeta} \
           --arg includeBroken ${lib.boolToString includeBroken} \
+          -I ${nixpkgs} \
+          -I ${attrpathFile} \
           > "$outputDir/result/$myChunk"
         exitCode=$?
         set -e
@@ -251,6 +261,7 @@ let
           --slurpfile after ${afterResultDir}/outpaths.json \
           > $out/changed-paths.json
 
+        jq -r -f ${./generate-step-summary.jq} < $out/changed-paths.json > $out/step-summary.md
         # TODO: Compare eval stats
       '';
 
diff --git a/ci/eval/generate-step-summary.jq b/ci/eval/generate-step-summary.jq
new file mode 100644
index 0000000000000..28597eaec371f
--- /dev/null
+++ b/ci/eval/generate-step-summary.jq
@@ -0,0 +1,15 @@
+def truncate(xs; n):
+  if xs | length > n then xs[:n] + ["..."]
+  else xs
+  end;
+
+def itemize_packages(xs):
+  # we truncate the list to stay below the GitHub limit of 1MB per step summary.
+  truncate(xs; 3000) | map("- [\(.)](https://search.nixos.org/packages?channel=unstable&show=\(.)&from=0&size=50&sort=relevance&type=packages&query=\(.))") | join("\n");
+
+def section(title; xs):
+  "<details> <summary>" + title + " (" + (xs | length | tostring) + ")</summary>\n\n" + itemize_packages(xs) + "</details>";
+
+section("Added packages"; .attrdiff.added) + "\n\n" +
+section("Removed packages"; .attrdiff.removed) + "\n\n" +
+section("Changed packages"; .attrdiff.changed)