Update scripts to gather sources info of nix packages

.github/workflows/build-and-deploy.yml

@@ -0,0 +1,60 @@
+name: Deploy sources-unstable.json to Pages
+
+on:
+  push:
+  pull_request:
+  schedule:
+    # execute workflow every day at midnight
+    - cron: "0 0 * * *"
+
+# required for deploying to GH Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+          install_options: --no-daemon
+      # Call to nix-instantiate eats a LOT of RAM so we need to increase swap space size
+      # to avoid the generate.sh script execution to be killed in flight by the gha runner.
+      # This terribly slow downs the processing though but at least it terminates successfully.
+      - name: Set Swap Space
+        uses: pierotofy/set-swap-space@master
+        with:
+            swap-size-gb: 16
+      - name: Build sources-unstable.json
+        run: ./scripts/generate.sh build/ unstable
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./build
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    needs: build
+    runs-on: ubuntu-latest
+    name: Deploy
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -0,0 +1 @@
+build/

README.md

@@ -9,3 +9,14 @@ by [Hydra](https://hydra.nixos.org/project/nixpkgs).
 
 A basic analysis of this file is also generated and published
 [here](https://nix-community.github.io/nixpkgs-swh).
+
+# Locally generate files
+
+To generate files for the `nixpkgs.hello` nixpkgs subset, run `nix run
+.#nixpkgs-swh-generate -- --testing /tmp/swh unstable`. This is mainly
+used to debugging purposes.
+
+If you have a lot of memory and time, to generate files for the whole nixpkgs:
+```
+nix run .#nixpkgs-swh-generate -- /tmp/swh unstable
+```

flake.nix

@@ -0,0 +1,92 @@
+{
+  description = "nixpkgs-swh";
+
+  outputs = { self, nixpkgs }:
+  let
+    system = "x86_64-linux";
+    pkgs = import nixpkgs {
+      system = "x86_64-linux";
+      overlays = [ self.overlay ];
+    };
+  in {
+    overlay = final: prev: {
+      nixpkgs-swh-generate = let
+        binPath = with final; lib.makeBinPath [ nix git openssh (python3.withPackages(p: [p.aiohttp p.uvloop])) curl jq ];
+      in final.stdenv.mkDerivation {
+        name = "nixpkgs-swh-generate";
+        dontUnpack = true;
+        nativeBuildInputs = [ final.makeWrapper ];
+        installPhase = ''
+          mkdir -p $out/bin
+          cp ${./scripts/generate.sh} $out/bin/nixpkgs-swh-generate
+          substituteInPlace $out/bin/nixpkgs-swh-generate \
+            --replace-fail './scripts/swh-urls.nix' '${./scripts/swh-urls.nix}' \
+            --replace-fail './scripts/post-process.py' '${./scripts/post-process.py}' \
+            --replace-fail './scripts/analyze.py' '${./scripts/analyze.py}' \
+            --replace-fail '$PWD/scripts/find-tarballs.nix' '${./scripts/find-tarballs.nix}'
+          wrapProgram $out/bin/nixpkgs-swh-generate \
+            --prefix PATH : ${binPath}
+        '';
+      };
+    };
+
+    packages.x86_64-linux.nixpkgs-swh-generate = pkgs.nixpkgs-swh-generate;
+    defaultPackage.x86_64-linux = pkgs.nixpkgs-swh-generate;
+
+    nixosModules.nixpkgs-swh = { config, pkgs, lib, ... }: let
+      cfg = config.services.nixpkgs-swh;
+      dir = "/var/lib/nixpkgs-swh";
+    in {
+      # Add an option to specify release or let the script finding
+      # releases.
+      options = {
+        services.nixpkgs-swh = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+            description = ''
+              Whether to run the nixpkgs-swh service.
+            '';
+          };
+          testing = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+            description = ''
+              Whether to only evaluate the hello attribute for testing purpose.
+            '';
+          };
+          fqdn = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The Nginx vhost FQDN used to serve built files.
+            '';
+          };
+        };
+      };
+      config = lib.mkIf cfg.enable {
+        nixpkgs.overlays = [ self.overlay ];
+        systemd.services.nixpkgs-swh = {
+          description = "nixpkgs-swh";
+          wantedBy = [ "multi-user.target" ];
+          startAt = "daily";
+          script = ''
+            ${pkgs.nixpkgs-swh-generate}/bin/nixpkgs-swh-generate ${lib.strings.optionalString cfg.testing "--testing"} ${dir} unstable
+          '';
+        };
+        systemd.timers.nixpkgs-swh.timerConfig = {
+          Persistent = true;
+        };
+        services.nginx.virtualHosts = {
+          "${cfg.fqdn}" = {
+            locations."/" = {
+              root = "${dir}";
+              extraConfig = ''
+                autoindex on;
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}