ci/eval/release-checks: init

.github/workflows/release-checks.yml

@@ -0,0 +1,142 @@
+name: Release checks
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/release-checks.yml
+  pull_request_target:
+
+permissions: {}
+
+jobs:
+  get-merge-commit:
+    uses: ./.github/workflows/get-merge-commit.yml
+
+  outpaths:
+    name: Outpaths
+    runs-on: ubuntu-24.04-arm
+    needs: [ get-merge-commit ]
+    strategy:
+      fail-fast: false
+      matrix:
+        system: ${{ fromJSON(needs.get-merge-commit.outputs.systems) }}
+    steps:
+      - name: Enable swap
+        run: |
+          sudo fallocate -l 10G /swap
+          sudo chmod 600 /swap
+          sudo mkswap /swap
+          sudo swapon /swap
+
+      - name: Check out the PR at the test merge commit
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
+          path: nixpkgs
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
+        with:
+          extra_nix_config: sandbox = true
+
+      - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes from a different path
+        env:
+          MATRIX_SYSTEM: ${{ matrix.system }}
+        run: |
+          nix-build nixpkgs/ci -A eval.releaseChecks.secondEval \
+            --argstr evalSystem "$MATRIX_SYSTEM" \
+            --arg chunkSize 10000
+          # If it uses too much memory, slightly decrease chunkSize
+
+      - name: Upload the output paths and eval stats
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: intermediate-${{ matrix.system }}
+          path: result
+
+  release-checks:
+    name: Release checks
+    runs-on: ubuntu-24.04-arm
+    needs: [ outpaths, get-merge-commit ]
+    steps:
+      - name: Download output paths and eval stats from second eval for all systems
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          pattern: intermediate-*
+          path: second
+
+      - name: Check out the PR at the test merge commit
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
+          fetch-depth: 2
+          path: nixpkgs
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
+        with:
+          extra_nix_config: sandbox = true
+
+      - name: Get first eval run id
+        id: evalRunId
+        run: |
+          # Get the latest eval.yml workflow run for the head commit
+          if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
+            -f head_sha="$HEAD_SHA" \
+            --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
+            || [[ -z "$run" ]]; then
+            echo "Could not find an eval.yml workflow run for $HEAD_SHA, cannot make comparison"
+            exit 1
+          fi
+          echo "First eval is $(jq .html_url <<< "$run")"
+          runId=$(jq .id <<< "$run")
+          conclusion=$(jq -r .conclusion <<< "$run")
+
+          while [[ "$conclusion" == null || "$conclusion" == "" ]]; do
+            echo "Workflow not done, waiting 10 seconds before checking again"
+            sleep 10
+            conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
+          done
+
+          if [[ "$conclusion" != "success" ]]; then
+            echo "Workflow was not successful (conclusion: $conclusion), cannot perform release checks"
+            exit 1
+          fi
+
+          echo "evalRunId=$runId" >> "$GITHUB_OUTPUT"
+        env:
+          REPOSITORY: ${{ github.repository }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Download output paths and eval stats from first eval for all systems
+        uses: actions/download-artifact@v4
+        if: steps.evalRunId.outputs.evalRunId
+        with:
+          pattern: intermediate-*
+          path: first
+          github-token: ${{ github.token }}
+          run-id: ${{ steps.evalRunId.outputs.evalRunId }}
+
+      - name: Run release checks
+        if: steps.evalRunId.outputs.evalRunId
+        run: |
+          failed=
+          for system in $(echo -n "$SYSTEMS" | jq -r ".[]"); do
+            echo "::group::$system"
+
+            nix-build nixpkgs/ci -A eval.releaseChecks \
+              --argstr evalSystem "$system" \
+              --arg firstEval "./first/intermediate-$system" \
+              --arg secondEval "./second/intermediate-$system" \
+              --arg chunkSize 10000 || failed=1
+
+            echo "::endgroup::"
+          done
+
+          if [[ $failed ]]; then
+            echo "Release checks failed for at least one system"
+            exit 1
+          fi
+        env:
+          SYSTEMS: ${{ needs.get-merge-commit.outputs.systems }}

ci/OWNERS

@@ -15,11 +15,11 @@
 
 # CI
 /.github/*_TEMPLATE*                    @SigmaSquadron
-/.github/workflows                      @NixOS/Security @Mic92 @zowoq @infinisil @azuwis @wolfgangwalther
+/.github/workflows                      @NixOS/Security @Mic92 @zowoq @infinisil @azuwis @winterqt @wolfgangwalther
 /.github/workflows/check-format.yml     @infinisil @wolfgangwalther
 /.github/workflows/codeowners-v2.yml    @infinisil @wolfgangwalther
 /.github/workflows/nixpkgs-vet.yml      @infinisil @philiptaron @wolfgangwalther
-/ci                                     @infinisil @philiptaron @NixOS/Security @wolfgangwalther
+/ci                                     @infinisil @philiptaron @NixOS/Security @winterqt @wolfgangwalther
 /ci/OWNERS                              @infinisil @philiptaron
 
 # Development support

ci/eval/default.nix

@@ -35,17 +35,23 @@ let
 
   supportedSystems = builtins.fromJSON (builtins.readFile ../supportedSystems.json);
 
+  abortSource = builtins.toFile "dont-use-nix-path-in-nixpkgs.nix" ''
+    builtins.abort "Illegal use of <nixpkgs> in Nixpkgs"
+  '';
+
   attrpathsSuperset =
     {
       evalSystem,
+      nixpkgsPath ? nixpkgs,
     }:
     runCommand "attrpaths-superset.json"
       {
-        src = nixpkgs;
+        src = nixpkgsPath;
         nativeBuildInputs = [
           nix
           time
         ];
+        env.NIX_PATH = "nixpkgs=${abortSource}";
       }
       ''
         export NIX_STATE_DIR=$(mktemp -d)
@@ -69,11 +75,22 @@ let
       # because `--argstr system` would only be passed to the ci/default.nix file!
       evalSystem,
       # The path to the `paths.json` file from `attrpathsSuperset`
-      attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json",
+      attrpathFile ? "${attrpathsSuperset { inherit evalSystem nixpkgsPath; }}/paths.json",
+      nixpkgsPath ? nixpkgs,
       # The number of attributes per chunk, see ./README.md for more info.
       chunkSize,
       checkMeta ? true,
-      includeBroken ? true,
+
+      # Don't try to eval packages marked as broken.
+      includeBroken ? false,
+
+      # This is a misnomer, as we still catch and detect packages that aren't in a
+      # package's `meta.platforms`, but we still want to try and eval it to make
+      # sure that we can even get to the unsupported `throw`. (See
+      # https://github.com/NixOS/nixpkgs/pull/406207 as an example where the broken
+      # `throw` wasn't reached, therefore breaking `nixpkgs-review`.)
+      includeUnsupported ? true,
+
       # Whether to just evaluate a single chunk for quick testing
       quickTest ? false,
     }:
@@ -91,20 +108,22 @@ let
         set +e
         command time -o "$outputDir/timestats/$myChunk" \
           -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
-          nix-env -f "${nixpkgs}/pkgs/top-level/release-attrpaths-parallel.nix" \
+          nix-env -f "${nixpkgsPath}/pkgs/top-level/release-attrpaths-parallel.nix" \
           --eval-system "$system" \
           --option restrict-eval true \
           --option allow-import-from-derivation false \
           --query --available \
           --no-name --attr-path --out-path \
+          --meta --json \
           --show-trace \
           --arg chunkSize "$chunkSize" \
           --arg myChunk "$myChunk" \
           --arg attrpathFile "${attrpathFile}" \
           --arg systems "[ \"$system\" ]" \
           --arg checkMeta ${lib.boolToString checkMeta} \
           --arg includeBroken ${lib.boolToString includeBroken} \
-          -I ${nixpkgs} \
+          --arg includeUnsupported ${lib.boolToString includeUnsupported} \
+          -I ${nixpkgsPath} \
           -I ${attrpathFile} \
           > "$outputDir/result/$myChunk" \
           2> "$outputDir/stderr/$myChunk"
@@ -132,6 +151,7 @@ let
         ];
         env = {
           inherit evalSystem chunkSize;
+          NIX_PATH = "nixpkgs=${abortSource}";
         };
       }
       ''
@@ -190,7 +210,22 @@ let
 
         # Make sure the glob doesn't break when there's no files
         shopt -s nullglob
-        cat "$chunkOutputDir"/result/* > $out/paths
+
+        cat "$chunkOutputDir"/result/* | jq -r '
+          to_entries[]
+          | select(.value.meta.unsupported | not)
+          | .key + " " +
+            .value.outputs as $outputs
+            | $outputs
+            | to_entries
+            | map(select(.key != "out") | .key + "=" + .value)
+            | join(";")
+            + if $outputs.out then
+                (if $outputs | length == 1 then "" else ";" end) + $outputs.out
+              else
+                ""
+              end
+        ' > $out/paths
         cat "$chunkOutputDir"/stats/* > $out/stats.jsonstream
       '';
 
@@ -276,6 +311,16 @@ let
       ;
   };
 
+  releaseChecks = import ./release-checks.nix {
+    inherit
+      lib
+      runCommand
+      nixpkgs
+      singleSystem
+      linkFarm
+      ;
+  };
+
   full =
     {
       # Whether to evaluate on a specific set of systems, by default all are evaluated
@@ -301,11 +346,11 @@ let
 in
 {
   inherit
-    attrpathsSuperset
     singleSystem
     combine
     compare
-    # The above three are used by separate VMs in a GitHub workflow,
+    releaseChecks
+    # The above four are used by separate VMs in a GitHub workflow,
     # while the below is intended for testing on a single local machine
     full
     ;

ci/eval/release-checks.nix

@@ -0,0 +1,69 @@
+{
+  lib,
+  runCommand,
+  nixpkgs,
+  singleSystem,
+  linkFarm,
+}:
+
+let
+  nixpkgs' = builtins.path {
+    name = "nixpkgs-prime";
+    path = nixpkgs;
+  };
+in
+{
+  evalSystem,
+  chunkSize,
+  firstEval ? singleSystem { inherit evalSystem chunkSize; },
+  secondEval ? singleSystem {
+    inherit evalSystem chunkSize;
+    nixpkgsPath = nixpkgs';
+  },
+}:
+
+let
+  checks = {
+    badFiles = runCommand "bad-files-check" { } ''
+      badFiles=$(find ${nixpkgs}/pkgs -type f -name '*.nix' -print | xargs grep -l '^[^#]*<nixpkgs/' || true)
+      if [[ -n $badFiles ]]; then
+        echo "Nixpkgs is not allowed to use <nixpkgs> to refer to itself."
+        echo "The offending files: $badFiles"
+        exit 1
+      fi
+
+      touch "$out"
+    '';
+
+    conflictingPaths = runCommand "conflicting-files-check" { } ''
+      conflictingPaths=$(find ${nixpkgs} | awk '{ print $1 " " tolower($1) }' | sort -k2 | uniq -D -f 1 | cut -d ' ' -f 1)
+      if [[ -n $conflictingPaths ]]; then
+        echo "Files in nixpkgs must not vary only by case"
+        echo "The offending paths: $conflictingPaths"
+        exit 1
+      fi
+
+      touch "$out"
+    '';
+
+    evalPurity = runCommand "eval-purity-check" { } ''
+      if ! diff -u ${firstEval}/paths ${secondEval}/paths; then
+        echo
+        echo "Error: Nixpkgs evaluation depends on Nixpkgs path"
+        exit 1
+      fi
+
+      touch "$out"
+    '';
+  };
+in
+(linkFarm "release-checks-${evalSystem}" (
+  lib.mapAttrsToList (_: check: {
+    inherit (check) name;
+    path = check;
+  }) checks
+))
+// checks
+// {
+  inherit secondEval;
+}

pkgs/top-level/release-attrpaths-parallel.nix

@@ -9,6 +9,7 @@
   myChunk,
   checkMeta,
   includeBroken,
+  includeUnsupported ? false,
   systems,
 }:
 
@@ -18,7 +19,12 @@ let
 
   unfiltered = import ./release-outpaths.nix {
     inherit path;
-    inherit checkMeta includeBroken systems;
+    inherit
+      checkMeta
+      includeBroken
+      includeUnsupported
+      systems
+      ;
   };
 
   # Turns the unfiltered recursive attribute set into one that is limited to myAttrpaths