diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml
new file mode 100644
index 0000000..f4136f3
--- /dev/null
+++ b/.github/workflows/build-and-deploy.yml
@@ -0,0 +1,29 @@
+name: Build sources-unstable.json for nixpkgs.hello
+
+on:
+  push:
+  pull_request:
+  schedule:
+    # execute workflow every day at midnight
+    - cron: "0 0 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+          install_options: --no-daemon
+      - name: Build sources-unstable.json for nixpkgs.hello
+        run: nix run .#nixpkgs-swh-generate -- --testing build unstable
+      - name: Display sources-unstable.json for nixpkgs.hello
+        run: cat build/sources-unstable.json | jq
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..d163863
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+build/
\ No newline at end of file
diff --git a/README.md b/README.md
index b3a8832..c44c73b 100644
--- a/README.md
+++ b/README.md
@@ -9,3 +9,14 @@ by [Hydra](https://hydra.nixos.org/project/nixpkgs).
 
 A basic analysis of this file is also generated and published
 [here](https://nix-community.github.io/nixpkgs-swh).
+
+# Locally generate files
+
+To generate files for the `nixpkgs.hello` nixpkgs subset, run `nix run
+.#nixpkgs-swh-generate -- --testing /tmp/swh unstable`. This is mainly
+used to debugging purposes.
+
+If you have a lot of memory and time, to generate files for the whole nixpkgs:
+```
+nix run .#nixpkgs-swh-generate -- /tmp/swh unstable
+```
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..19f395c
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,25 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1752841793,
+        "narHash": "sha256-pGvcN/yiJ4e3/hgvTwkyZvl6c3HuLLCY/hjEzDuQB54=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "8131c0ea9df6293a247be743a387ff725e464db7",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..f78c215
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,92 @@
+{
+  description = "nixpkgs-swh";
+
+  outputs = { self, nixpkgs }:
+  let
+    system = "x86_64-linux";
+    pkgs = import nixpkgs {
+      system = "x86_64-linux";
+      overlays = [ self.overlay ];
+    };
+  in {
+    overlay = final: prev: {
+      nixpkgs-swh-generate = let
+        binPath = with final; lib.makeBinPath [ nix git openssh (python3.withPackages(p: [p.aiohttp p.uvloop])) curl jq ];
+      in final.stdenv.mkDerivation {
+        name = "nixpkgs-swh-generate";
+        dontUnpack = true;
+        nativeBuildInputs = [ final.makeWrapper ];
+        installPhase = ''
+          mkdir -p $out/bin
+          cp ${./scripts/generate.sh} $out/bin/nixpkgs-swh-generate
+          substituteInPlace $out/bin/nixpkgs-swh-generate \
+            --replace-fail './scripts/swh-urls.nix' '${./scripts/swh-urls.nix}' \
+            --replace-fail './scripts/post-process.py' '${./scripts/post-process.py}' \
+            --replace-fail './scripts/analyze.py' '${./scripts/analyze.py}' \
+            --replace-fail '$PWD/scripts/find-tarballs.nix' '${./scripts/find-tarballs.nix}'
+          wrapProgram $out/bin/nixpkgs-swh-generate \
+            --prefix PATH : ${binPath}
+        '';
+      };
+    };
+
+    packages.x86_64-linux.nixpkgs-swh-generate = pkgs.nixpkgs-swh-generate;
+    defaultPackage.x86_64-linux = pkgs.nixpkgs-swh-generate;
+
+    nixosModules.nixpkgs-swh = { config, pkgs, lib, ... }: let
+      cfg = config.services.nixpkgs-swh;
+      dir = "/var/lib/nixpkgs-swh";
+    in {
+      # Add an option to specify release or let the script finding
+      # releases.
+      options = {
+        services.nixpkgs-swh = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+            description = ''
+              Whether to run the nixpkgs-swh service.
+            '';
+          };
+          testing = lib.mkOption {
+            type = lib.types.bool;
+            default = false;
+            description = ''
+              Whether to only evaluate the hello attribute for testing purpose.
+            '';
+          };
+          fqdn = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              The Nginx vhost FQDN used to serve built files.
+            '';
+          };
+        };
+      };
+      config = lib.mkIf cfg.enable {
+        nixpkgs.overlays = [ self.overlay ];
+        systemd.services.nixpkgs-swh = {
+          description = "nixpkgs-swh";
+          wantedBy = [ "multi-user.target" ];
+          startAt = "daily";
+          script = ''
+            ${pkgs.nixpkgs-swh-generate}/bin/nixpkgs-swh-generate ${lib.strings.optionalString cfg.testing "--testing"} ${dir} unstable
+          '';
+        };
+        systemd.timers.nixpkgs-swh.timerConfig = {
+          Persistent = true;
+        };
+        services.nginx.virtualHosts = {
+          "${cfg.fqdn}" = {
+            locations."/" = {
+              root = "${dir}";
+              extraConfig = ''
+                autoindex on;
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}
diff --git a/nix/default.nix b/nix/default.nix
deleted file mode 100644
index 041b3d5..0000000
--- a/nix/default.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-{ sources ? import ./sources.nix, inNixShell ? false }:
-
-import sources.nixpkgs { inherit inNixShell; }
diff --git a/nix/sources.json b/nix/sources.json
deleted file mode 100644
index 2e41138..0000000
--- a/nix/sources.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-    "nixpkgs": {
-        "branch": "nixos-22.11",
-        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
-        "homepage": "https://github.com/NixOS/nixpkgs",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "13fdd3945d8a2da5e4afe35d8a629193a9680911",
-        "sha256": "14pic9rpn9wwk1bn0gvkv1v5q1jgnj7avm2f2vlfh2xik1ifhkyh",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/13fdd3945d8a2da5e4afe35d8a629193a9680911.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    }
-}
diff --git a/nix/sources.nix b/nix/sources.nix
deleted file mode 100644
index 9a01c8a..0000000
--- a/nix/sources.nix
+++ /dev/null
@@ -1,194 +0,0 @@
-# This file has been generated by Niv.
-
-let
-
-  #
-  # The fetchers. fetch_<type> fetches specs of type <type>.
-  #
-
-  fetch_file = pkgs: name: spec:
-    let
-      name' = sanitizeName name + "-src";
-    in
-      if spec.builtin or true then
-        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
-      else
-        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
-
-  fetch_tarball = pkgs: name: spec:
-    let
-      name' = sanitizeName name + "-src";
-    in
-      if spec.builtin or true then
-        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
-      else
-        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
-
-  fetch_git = name: spec:
-    let
-      ref =
-        if spec ? ref then spec.ref else
-          if spec ? branch then "refs/heads/${spec.branch}" else
-            if spec ? tag then "refs/tags/${spec.tag}" else
-              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
-      submodules = if spec ? submodules then spec.submodules else false;
-      submoduleArg =
-        let
-          nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0;
-          emptyArgWithWarning =
-            if submodules == true
-            then
-              builtins.trace
-                (
-                  "The niv input \"${name}\" uses submodules "
-                  + "but your nix's (${builtins.nixVersion}) builtins.fetchGit "
-                  + "does not support them"
-                )
-                {}
-            else {};
-        in
-          if nixSupportsSubmodules
-          then { inherit submodules; }
-          else emptyArgWithWarning;
-    in
-      builtins.fetchGit
-        ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg);
-
-  fetch_local = spec: spec.path;
-
-  fetch_builtin-tarball = name: throw
-    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
-        $ niv modify ${name} -a type=tarball -a builtin=true'';
-
-  fetch_builtin-url = name: throw
-    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
-        $ niv modify ${name} -a type=file -a builtin=true'';
-
-  #
-  # Various helpers
-  #
-
-  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
-  sanitizeName = name:
-    (
-      concatMapStrings (s: if builtins.isList s then "-" else s)
-        (
-          builtins.split "[^[:alnum:]+._?=-]+"
-            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
-        )
-    );
-
-  # The set of packages used when specs are fetched using non-builtins.
-  mkPkgs = sources: system:
-    let
-      sourcesNixpkgs =
-        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
-      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
-      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
-    in
-      if builtins.hasAttr "nixpkgs" sources
-      then sourcesNixpkgs
-      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
-        import <nixpkgs> {}
-      else
-        abort
-          ''
-            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
-            add a package called "nixpkgs" to your sources.json.
-          '';
-
-  # The actual fetching function.
-  fetch = pkgs: name: spec:
-
-    if ! builtins.hasAttr "type" spec then
-      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
-    else if spec.type == "file" then fetch_file pkgs name spec
-    else if spec.type == "tarball" then fetch_tarball pkgs name spec
-    else if spec.type == "git" then fetch_git name spec
-    else if spec.type == "local" then fetch_local spec
-    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
-    else if spec.type == "builtin-url" then fetch_builtin-url name
-    else
-      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
-
-  # If the environment variable NIV_OVERRIDE_${name} is set, then use
-  # the path directly as opposed to the fetched source.
-  replace = name: drv:
-    let
-      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
-      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
-    in
-      if ersatz == "" then drv else
-        # this turns the string into an actual Nix path (for both absolute and
-        # relative paths)
-        if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
-
-  # Ports of functions for older nix versions
-
-  # a Nix version of mapAttrs if the built-in doesn't exist
-  mapAttrs = builtins.mapAttrs or (
-    f: set: with builtins;
-    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
-  );
-
-  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
-  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
-
-  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
-  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
-
-  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
-  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
-  concatMapStrings = f: list: concatStrings (map f list);
-  concatStrings = builtins.concatStringsSep "";
-
-  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
-  optionalAttrs = cond: as: if cond then as else {};
-
-  # fetchTarball version that is compatible between all the versions of Nix
-  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
-    let
-      inherit (builtins) lessThan nixVersion fetchTarball;
-    in
-      if lessThan nixVersion "1.12" then
-        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
-      else
-        fetchTarball attrs;
-
-  # fetchurl version that is compatible between all the versions of Nix
-  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
-    let
-      inherit (builtins) lessThan nixVersion fetchurl;
-    in
-      if lessThan nixVersion "1.12" then
-        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
-      else
-        fetchurl attrs;
-
-  # Create the final "sources" from the config
-  mkSources = config:
-    mapAttrs (
-      name: spec:
-        if builtins.hasAttr "outPath" spec
-        then abort
-          "The values in sources.json should not have an 'outPath' attribute"
-        else
-          spec // { outPath = replace name (fetch config.pkgs name spec); }
-    ) config.sources;
-
-  # The "config" used by the fetchers
-  mkConfig =
-    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
-    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
-    , system ? builtins.currentSystem
-    , pkgs ? mkPkgs sources system
-    }: rec {
-      # The sources, i.e. the attribute set of spec name to spec
-      inherit sources;
-
-      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
-      inherit pkgs;
-    };
-
-in
-mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }
diff --git a/scripts/add-sri.py b/scripts/add-sri.py
deleted file mode 100644
index 7d1b0ff..0000000
--- a/scripts/add-sri.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import json
-import sys
-import subprocess
-import traceback
-
-# These are heuristics to detect the fetcher, based on the postFetch value!
-fetchZipPattern = "Pass stripRoot=false; to fetchzip to assume flat list of files"
-fetchpatchPattern = "Did you maybe fetch a HTML representation of a patch instead of a raw patch"
-
-sources = None
-with open(sys.argv[1], 'r') as f:
-    sources = json.load(f)
-    new = []
-    for s in sources['sources']:
-        try:
-            hashArray = s['outputHash'].split(":")
-            if len(hashArray) == 2:
-                hashAlgo = hashArray[0]
-                hashStr = hashArray[1]
-            else:
-                hashAlgo = s['outputHashAlgo']
-                hashStr = s['outputHash']
-
-            if (hashAlgo is None or hashAlgo == "") and hashStr != "":
-                s['integrity'] = hashStr
-                continue
-
-            result = subprocess.run(
-                ['nix', 'hash', 'to-sri', '--type', hashAlgo, hashStr],
-                stdout=subprocess.PIPE)
-            s['integrity'] = str(result.stdout.rstrip(), 'utf8')
-        except TypeError as e:
-            print(f'TypeError on %s' % s)
-            print('-'*60)
-            traceback.print_exc(file=sys.stdout)
-            print('-'*60)
-
-        # We try to infer the fetcher
-        s['inferredFetcher'] = 'unclassified'
-        if fetchZipPattern in s['postFetch']:
-            s['inferredFetcher'] = 'fetchzip'
-        elif fetchpatchPattern in s['postFetch']:
-            s['inferredFetcher'] = 'fetchpatch'
-        del s['postFetch']
-            
-if sources is not None:
-    with open(sys.argv[1], 'w') as f:
-        json.dump(sources, f)
diff --git a/scripts/analyze.py b/scripts/analyze.py
index 7525e64..63ae45e 100644
--- a/scripts/analyze.py
+++ b/scripts/analyze.py
@@ -15,73 +15,72 @@
 
 # To provide high level classification
 origin_patterns = {
-    'https?://hackage.haskell.org/.*': 'hackage',
-    'https?://github.com/(.*)/(.*)/archive/(.*)(?:.tar.gz|.zip)': 'github_archive', # noqa
-    'https?://github.com/(.*)/(.*)/releases/download/(.*)/(.*)': 'github_release', # noqa
-    'https://crates.io/api/v1/crates/(.*)/download': 'crates',
-    'https://bitbucket.org/(.*)/(.*)/(get|downloads)/(.*)(.tar.gz|.zip|tar.bz2)': 'bitbucket', # noqa
-    'https://rubygems.org/(.*)': 'rubygems',
-    'svn://(.*)': 'svn',
-    'https://(.*)/api/v4/projects/(.*)/repository/archive.tar.gz\?sha=(.*)': 'gitlab', # noqa
-    '.*': 'unknown',
+    "https?://hackage.haskell.org/.*": "hackage",
+    "https?://github.com/(.*)/(.*)/archive/(.*)(?:.tar.gz|.zip)": "github_archive",  # noqa
+    "https?://github.com/(.*)/(.*)/releases/download/(.*)/(.*)": "github_release",  # noqa
+    "https://crates.io/api/v1/crates/(.*)/download": "crates",
+    "https://bitbucket.org/(.*)/(.*)/(get|downloads)/(.*)(.tar.gz|.zip|tar.bz2)": "bitbucket",  # noqa
+    "https://rubygems.org/(.*)": "rubygems",
+    "svn://(.*)": "svn",
+    "https://(.*)/api/v4/projects/(.*)/repository/archive.tar.gz\\?sha=(.*)": "gitlab",  # noqa
+    ".*": "unknown",
 }
 
 extension_patterns = {
-    '.*(.tar.gz$|.zip$|tar.bz2$|.tbz$|.tar.xz$|.tgz|.tar)': 'archive',
-    '.*(.gem$)': 'gem',
-    '.*(.pom$)': 'pom',
-    '.*(.jar$)': 'jar',
-    '.*(.deb$)': 'deb',
-    '.*(.patch$)': 'patch',
-    '.*(.diff$)': 'diff',
-    '.*(.rpm$)': 'rpm',
-    '.*(.png$)': 'png',
-    '.*(.msi$)': 'msi',
-    '.*(.iso$)': 'iso',
-    '.*(.c$|.h$)': 'c',
-    '.*(.ttf$)': 'ttf',
-    '.*(.rock$)': 'rock',
-    '.*(.whl$)': 'whl',
-    '.*': 'unknown',
+    ".*(.tar.gz$|.zip$|tar.bz2$|.tbz$|.tar.xz$|.tgz|.tar)": "archive",
+    ".*(.gem$)": "gem",
+    ".*(.pom$)": "pom",
+    ".*(.jar$)": "jar",
+    ".*(.deb$)": "deb",
+    ".*(.patch$)": "patch",
+    ".*(.diff$)": "diff",
+    ".*(.rpm$)": "rpm",
+    ".*(.png$)": "png",
+    ".*(.msi$)": "msi",
+    ".*(.iso$)": "iso",
+    ".*(.c$|.h$)": "c",
+    ".*(.ttf$)": "ttf",
+    ".*(.rock$)": "rock",
+    ".*(.whl$)": "whl",
+    ".*": "unknown",
 }
 
-sources = j['sources']
+sources = j["sources"]
 for e in sources:
-
-    u = urlparse(e['urls'][0])
+    if e["type"] != "url" or not e["urls"]:
+        continue
+    u = urlparse(e["urls"][0])
     schemes[u.scheme] = schemes.get(u.scheme, 0) + 1
     hosts[u.netloc] = hosts.get(u.netloc, 0) + 1
 
     for k, v in origin_patterns.items():
-        if re.search(k, e['urls'][0]) is not None:
-            e['type'] = v
+        if re.search(k, e["urls"][0]) is not None:
+            e["type"] = v
             break
 
     for k, v in extension_patterns.items():
-        if re.search(k, e['urls'][0]) is not None:
-            e['file-type'] = v
+        if re.search(k, e["urls"][0]) is not None:
+            e["file-type"] = v
             break
 
 
 readme = """
-
-The file [`sources-{release}-full.json`](https://nix-community.github.io/nixpkgs-swh/sources-{release}-full.json)
-has been built from the [nixpkgs revision
-`{revision}`](https://github.com/NixOS/nixpkgs/tree/{revision}).
-This file contains `{sourceNumber}` sources, coming from
-`{hostNumber}` different hosts.
-
-The file [`sources-{release}.json`](https://nix-community.github.io/nixpkgs-swh/sources-{release}.json) is a filtered version which only contains archives. This file is consumed by SWH.
-
-"""
+The file [`sources-{release}.json`](https://nix-community.github.io/nixpkgs-swh/sources-{release}.json)
+has been built from the [nixpkgs revision `{revision}`](https://github.com/NixOS/nixpkgs/tree/{revision}).
+This file contains `{sourceNumber}` sources, coming from`{hostNumber}` different hosts.
+This file is consumed by SWH.
+"""  # noqa
 
 sortedHosts = sorted(hosts.items(), key=lambda h: h[1], reverse=True)
 
-print(readme.format(
-    revision=j['revision'],
-    release=j['release'],
-    sourceNumber=len(sources),
-    hostNumber=len(sortedHosts)))
+print(
+    readme.format(
+        revision=j["revision"],
+        release=j["release"],
+        sourceNumber=len(sources),
+        hostNumber=len(sortedHosts),
+    )
+)
 
 
 print("\n#### By host\n")
@@ -95,12 +94,10 @@
 
 types = {}
 file_types = {}
-fetchers = {}
 for s in sources:
-    types[s['type']] = types.get(s['type'], 0) + 1
-    file_types[s['file-type']] = file_types.get(s['file-type'], 0) + 1
-    if 'inferredFetcher' in s:
-        fetchers[s['inferredFetcher']] = fetchers.get(s['inferredFetcher'], 0) + 1
+    types[s["type"]] = types.get(s["type"], 0) + 1
+    if "file-type" in s:
+        file_types[s["file-type"]] = file_types.get(s["file-type"], 0) + 1
 
 print("\n#### By types\n")
 for k, v in types.items():
@@ -109,8 +106,3 @@
 print("\n#### By file types\n")
 for k, v in file_types.items():
     print("     %16s %s" % (k, v))
-
-print("\n#### By fetchers\n")
-print("Be careful, fetchers are inferred and could be wrong\n")
-for k, v in fetchers.items():
-    print("     %16s %s" % (k, v))
diff --git a/scripts/find-tarballs.nix b/scripts/find-tarballs.nix
index d449fcf..730a85f 100644
--- a/scripts/find-tarballs.nix
+++ b/scripts/find-tarballs.nix
@@ -10,32 +10,59 @@ let
   root = expr;
 
   uniqueUrls = map (x: x.file) (genericClosure {
-     startSet = map (file: { key = file.outputHash; inherit file; }) urls;
-     operator = const [ ];
-   });
+    startSet = map (file: {
+      key = file.outputHash;
+      inherit file;
+    }) urls;
+    operator = const [ ];
+  });
 
   urls = map (drv: {
     url = head (drv.urls or [ drv.url ]);
-    outputHash = drv.outputHash;
-    outputHashAlgo = drv.outputHashAlgo;
+    outputHash = if builtins.hasAttr "outputHash" drv
+    && (!lib.strings.hasSuffix "=" drv.outputHash
+      || !lib.strings.hasInfix "-" drv.outputHash) then
+      builtins.convertHash {
+        hash = drv.outputHash;
+        toHashFormat = "sri";
+        hashAlgo =
+          if drv.outputHashAlgo != null then
+            drv.outputHashAlgo
+          else
+            "sha256";
+      }
+    else
+      drv.outputHash or "";
+    outputHashAlgo = drv.outputHashAlgo or "";
     name = drv.name;
-    outputHashMode = drv.outputHashMode;
+    outputHashMode = drv.outputHashMode or "";
     postFetch = drv.postFetch or "";
+    rev = drv.rev or "";
+    submodule = builtins.hasAttr "fetchSubmodules" drv && drv.fetchSubmodules;
+    sparseCheckout =
+      if builtins.hasAttr "sparseCheckout" drv then drv.sparseCheckout else [ ];
+    type = if builtins.hasAttr "SVN_SSH" drv then
+      "svn"
+    else if builtins.hasAttr "fetchSubmodules" drv then
+      "git"
+    else if builtins.hasAttr "subrepoClause" drv then
+      "hg"
+    else
+      "url";
+    nixStorePath = drv.out;
   }) fetchurlDependencies;
 
   fetchurlDependencies =
-    filter
-    (drv: drv.outputHash or "" != ""
-          && (drv ? url || drv ? urls))
-          dependencies;
+    filter (drv: drv.outputHash or "" != "" && (drv ? url || drv ? urls))
+    dependencies;
 
-   # If a dichotomy is needed on nixpkgs:/
-   # subset = let
-   #   start = 14379;
-   #   len = 2;
-   #   sub = (pkgs.lib.sublist start len dependencies);
-   # in
-   # builtins.trace [ start len] sub;
+  # If a dichotomy is needed on nixpkgs:/
+  # subset = let
+  #   start = 14379;
+  #   len = 2;
+  #   sub = (pkgs.lib.sublist start len dependencies);
+  # in
+  # builtins.trace [ start len] sub;
 
   dependencies = map (x: x.value) (genericClosure {
     startSet = map keyDrv (derivationsIn' root);
@@ -43,22 +70,38 @@ let
   });
 
   derivationsIn' = x:
-    if !canEval x then []
-    else if isDerivation x then optional (canEval x.drvPath) x
-    else if isList x then concatLists (map derivationsIn' x)
-    else if isAttrs x then concatLists (mapAttrsToList (n: v: derivationsIn' v) x)
-    else [ ];
+    if !canEval x then
+      [ ]
+    else if isDerivation x then
+      optional (canEval x.drvPath) x
+    else if isList x then
+      concatLists (map derivationsIn' x)
+    else if isAttrs x then
+      concatLists (mapAttrsToList (n: v: derivationsIn' v) x)
+    else
+      [ ];
 
-  keyDrv = drv: if canEval drv.drvPath then { key = drv.drvPath; value = drv; } else { };
+  keyDrv = drv:
+    if canEval drv.drvPath then {
+      key = drv.drvPath;
+      value = drv;
+    } else
+      { };
 
   immediateDependenciesOf = drv:
-    concatLists (mapAttrsToList (n: v: derivationsIn v) (removeAttrs drv (["meta" "passthru"] ++ optionals (drv?passthru) (attrNames drv.passthru))));
+    concatLists (mapAttrsToList (n: v: derivationsIn v) (removeAttrs drv
+      ([ "meta" "passthru" ]
+        ++ optionals (drv ? passthru) (attrNames drv.passthru))));
 
   derivationsIn = x:
-    if !canEval x then []
-    else if isDerivation x then optional (canEval x.drvPath) x
-    else if isList x then concatLists (map derivationsIn x)
-    else [ ];
+    if !canEval x then
+      [ ]
+    else if isDerivation x then
+      optional (canEval x.drvPath) x
+    else if isList x then
+      concatLists (map derivationsIn x)
+    else
+      [ ];
 
   canEval = val: (builtins.tryEval val).success;
 
diff --git a/scripts/generate.sh b/scripts/generate.sh
index 3f8c687..919761c 100755
--- a/scripts/generate.sh
+++ b/scripts/generate.sh
@@ -1,7 +1,4 @@
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash
-#!nix-shell -I nixpkgs=./nix
-#!nix-shell -p nix git openssh python3 curl jq
+#!/usr/bin/env bash
 set -euo pipefail
 
 
@@ -31,11 +28,10 @@ generate-release() {
         exit 1
     fi
 
-    export SOURCES_FILE_FULL=${DEST_DIR}/sources-${RELEASE}-full.json
     export SOURCES_FILE=${DEST_DIR}/sources-${RELEASE}.json
 
     echo "*** Generate sources-${RELEASE}.json for commit $COMMIT_ID ..."
-    # This is to make nix-instantiate failing if the commit id can not be downloader
+    # This is to make nix-instantiate failing if the commit id can not be downloaded
     unset NIX_PATH
     export GC_INITIAL_HEAP_SIZE=4g
     # TODO: get the timestamp of the evaluation with the Hydra API. I
@@ -49,18 +45,16 @@ generate-release() {
         --argstr release ${RELEASE} \
         --argstr evaluation ${EVAL_ID} \
         --argstr timestamp $(date +%s) \
-        > ${SOURCES_FILE_FULL}
+        --arg testing $TESTING \
+        --argstr find-tarballs $PWD/scripts/find-tarballs.nix \
+        --show-trace \
+        > ${SOURCES_FILE}
 
-    echo "*** Add integrity attribute"
-    time python ./scripts/add-sri.py ${SOURCES_FILE_FULL}
-
-    # This is to reduce the SWH loader load time since it currently
-    # only support archives.
-    echo "*** Generate a filtered source file"
-    cat ${SOURCES_FILE_FULL} | jq '.sources = (.sources | map(select(.urls[0] | test(".tar.gz$|.zip$|tar.bz2$|.tbz$|.tar.xz$|.tgz$|.tar$"))))' > ${SOURCES_FILE}
+    echo "*** Post process extracted sources data"
+    time python ./scripts/post-process.py ${SOURCES_FILE}
 
     echo "*** Analyze the sources.json file and generating the README in sources-${RELEASE}.md ..."
-    time python ./scripts/analyze.py ${SOURCES_FILE_FULL} > ${DEST_DIR}/readme-${RELEASE}.md
+    time python ./scripts/analyze.py ${SOURCES_FILE} > ${DEST_DIR}/readme-${RELEASE}.md
 }
 
 
@@ -70,17 +64,34 @@ Fill the Software Heritage archive
 
 EOF
 
-for i in $@; do
-    generate-release ${i}
-    echo "### NixOS \`${i}\`" >> ${DEST_DIR}/README.md
-    cat ${DEST_DIR}/readme-${i}.md >> ${DEST_DIR}/README.md
-    echo >> ${DEST_DIR}/README.md
-    echo >> ${DEST_DIR}/README.md
-    shift
-done
+    for i in $@; do
+        generate-release ${i}
+        echo "### NixOS \`${i}\`" >> ${DEST_DIR}/README.md
+        cat ${DEST_DIR}/readme-${i}.md >> ${DEST_DIR}/README.md
+        echo >> ${DEST_DIR}/README.md
+        echo >> ${DEST_DIR}/README.md
+        shift
+    done
 }
 
 
+TESTING=false
+if [ $# -ge 1 ] && [ $1 = "--testing" ]
+then
+    TESTING=true
+    shift
+fi
+
+if [ $# -le 1 ]
+then
+    echo "Usage: nixpkgs-swh-generate [--testing] OUTPUT-DIR RELEASE [RELEASE] ..."
+    echo " --testing only evaluates nixpkgs.hello"
+    echo " OUTPUT-DIR is the diretory where generated files are outputed"
+    echo " RELEASE can be repeated multiple times. There are the release unstable, release-25.05, ..."
+    echo "   Release names correspond to the Hydra jobset names: https://hydra.nixos.org/project/nixos"
+    exit 1
+fi
+
 DEST_DIR=$1
 mkdir -p ${DEST_DIR}
 shift
diff --git a/scripts/post-process.py b/scripts/post-process.py
new file mode 100644
index 0000000..219b7f9
--- /dev/null
+++ b/scripts/post-process.py
@@ -0,0 +1,97 @@
+# This script processes the JSON output of the nix-instantiate call gathering
+# info about nix package sources. It notably removes duplicates, normalizes
+# integrity hashes and computes new fields expected by the Software Heritage
+# nixguix lister. The nixguix lister source code can be browsed at this URL:
+# https://gitlab.softwareheritage.org/swh/devel/swh-lister/-/tree/master/swh/lister/nixguix
+
+import asyncio
+import json
+import sys
+
+import aiohttp
+import uvloop
+
+# to remove duplicates
+seenStorePaths = set()
+
+# to contain unique sources
+filteredSources = []
+
+with open(sys.argv[1], "r") as f:
+    sources = json.load(f)
+    for source in sources["sources"]:
+        storePath = source["nixStorePath"]
+        if storePath in seenStorePaths:
+            # source already processed, skip it
+            continue
+        seenStorePaths.add(storePath)
+
+        # extract hash algorithm and hash value
+        hashArray = source["outputHash"].split(":")
+        if len(hashArray) == 2:
+            # nix sri format
+            hashAlgo = source["outputHashAlgo"] = hashArray[0]
+            hashStr = hashArray[1]
+        else:
+            hashAlgo = source["outputHashAlgo"]
+            hashStr = source["outputHash"]
+        del source["outputHash"]
+        source["integrity"] = hashStr
+        if (hashAlgo is None or hashAlgo == "") and hashStr.find("-") != -1:
+            source["outputHashAlgo"] = hashAlgo = hashStr.split("-", 1)[0]
+        if not hashAlgo:
+            # assume sha256
+            hashAlgo = source["outputHashAlgo"] = "sha256"
+
+        # add fields related to VCS sources
+        if source["type"] == "hg":
+            source["hg_url"] = source["urls"][0]
+            source["hg_changeset"] = source["rev"]
+        elif source["type"] == "git":
+            source["git_url"] = source["urls"][0]
+            source["git_ref"] = source["rev"]
+        elif source["type"] == "svn":
+            source["svn_url"] = source["urls"][0]
+            try:
+                source["svn_revision"] = int(source["rev"])
+            except ValueError:
+                source["svn_revision"] = source["rev"]
+
+        # remove empty/falsy fields
+        del source["rev"]
+        if source["type"] != "url":
+            del source["urls"]
+        for attr in ("submodule", "sparseCheckout", "postFetch"):
+            if not source[attr]:
+                del source[attr]
+        filteredSources.append(source)
+
+
+async def narinfo_get(source, session):
+    try:
+        hash_store = source["nixStorePath"].split("/")[-1].split("-", 1)[0]
+        url = f"https://cache.nixos.org/{hash_store}.narinfo"
+        async with session.get(url) as response:
+            narinfo = await response.read()
+            # print(f"Successfully got URL {url} with resp of length {len(narinfo)}.")
+            source["narinfo"] = narinfo.decode()
+            if source["narinfo"] != "404":
+                source["last_modified"] = response.headers["last-modified"]
+    except Exception as e:
+        print(f"Unable to get URL {url} due to {str(e)}.")
+
+
+async def fetch_narinfos(filteredSources):
+    async with aiohttp.ClientSession() as session:
+        await asyncio.gather(
+            *(narinfo_get(source, session) for source in filteredSources)
+        )
+
+
+# fetch narinfo data from the nix remote cache
+uvloop.run(fetch_narinfos(filteredSources))
+
+# dump post processed sources to file
+sources["sources"] = filteredSources
+with open(sys.argv[1], "w") as f:
+    json.dump(sources, f)
diff --git a/scripts/swh-urls.nix b/scripts/swh-urls.nix
index 0226bf1..4a69ad1 100644
--- a/scripts/swh-urls.nix
+++ b/scripts/swh-urls.nix
@@ -1,36 +1,40 @@
-{ revision ? null, release ? null, evaluation ? null, timestamp ? null }:
+{ revision ? null, release ? null, evaluation ? null, timestamp ? null, testing ? false, find-tarballs}:
 
 with builtins;
 let
-  pkgs = import <nixpkgs> {};
+  pkgs = import <nixpkgs> { };
   mirrors = import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>;
-  expr =  import <nixpkgs/maintainers/scripts/all-tarballs.nix>;
-  urls = import ./find-tarballs.nix {expr = expr;};
-  
+  expr = import <nixpkgs/maintainers/scripts/all-tarballs.nix>;
+  urls = import find-tarballs {expr = if testing then expr.hello else expr;};
+
   # This is avoid double slashes in urls that make url non valid
-  concatUrls = a: b: (pkgs.lib.removeSuffix "/" a) + "/" + (pkgs.lib.removePrefix "/" b);
+  concatUrls = a: b:
+    (pkgs.lib.removeSuffix "/" a) + "/" + (pkgs.lib.removePrefix "/" b);
 
   # If the url scheme is `mirror`, this translates this mirror to a real URL by looking in nixpkgs mirrors
-  resolveMirrorUrl = url: with pkgs.lib; let
-    splited = splitString "/" url;
-    isMirrorUrl = elemAt splited 0 != "mirror:";
-    mirror = elemAt splited 2;
-    path = concatStringsSep "/" (drop 3 splited);
-    resolvedUrls = getAttr mirror mirrors;
-  in if isMirrorUrl
-  then [ url ]
-  else map (r: concatUrls r path) resolvedUrls;
-  
+  resolveMirrorUrl = url:
+    with pkgs.lib;
+    let
+      splited = splitString "/" url;
+      isMirrorUrl = elemAt splited 0 != "mirror:";
+      mirror = elemAt splited 2;
+      path = concatStringsSep "/" (drop 3 splited);
+      resolvedUrls = if builtins.hasAttr mirror mirrors then
+        getAttr mirror mirrors
+      else
+        [ url ];
+    in if isMirrorUrl then [ url ] else map (r: concatUrls r path) resolvedUrls;
+
   # Transform the url list to swh format
   toSwh = s: {
-    inherit (s) postFetch outputHashMode outputHashAlgo outputHash;
-    type="url";
+    inherit (s)
+      postFetch outputHashMode outputHashAlgo outputHash rev submodule type
+      sparseCheckout nixStorePath;
     # There are expressions where the url is a list. See paratype-pt-mono
     # derivation: the url attribute is a list :/
     urls = if isList s.url then s.url else resolveMirrorUrl s.url;
   };
-in
-{
+in {
   inherit revision release evaluation timestamp;
   version = 1;
   sources = map toSwh urls;