diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..7b54cb6 --- /dev/null +++ b/flake.lock @@ -0,0 +1,24 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1738574474, + "narHash": "sha256-rvyfF49e/k6vkrRTV4ILrWd92W+nmBDfRYZgctOyolQ=", + "path": "/nix/store/096c6m1vzz4zr46lhqc8c1ynwr4zpc0a-source", + "rev": "fecfeb86328381268e29e998ddd3ebc70bbd7f7c", + "type": "path" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..cdd8e4e --- /dev/null +++ b/flake.nix @@ -0,0 +1,91 @@ +{ + description = "nixpkgs-swh"; + + outputs = { self, nixpkgs }: + let + system = "x86_64-linux"; + pkgs = import nixpkgs { + system = "x86_64-linux"; + overlays = [ self.overlay ]; + }; + in { + overlay = final: prev: { + nixpkgs-swh-generate = let + binPath = with final; lib.makeBinPath [ curl nix python3 jq pandoc ]; + in final.stdenv.mkDerivation { + name = "nixpkgs-swh-generate"; + dontUnpack = true; + nativeBuildInputs = [ final.makeWrapper ]; + installPhase = '' + mkdir -p $out/bin + cp ${./scripts/generate.sh} $out/bin/nixpkgs-swh-generate + substituteInPlace $out/bin/nixpkgs-swh-generate \ + --replace-fail './scripts/swh-urls.nix' '${./scripts/swh-urls.nix}' \ + --replace-fail './scripts/add-sri.py' '${./scripts/add-sri.py}' \ + --replace-fail './scripts/analyze.py' '${./scripts/analyze.py}' \ + --replace-fail '$PWD/scripts/find-tarballs.nix' '${./scripts/find-tarballs.nix}' + wrapProgram $out/bin/nixpkgs-swh-generate \ + --prefix PATH : ${binPath} + ''; + }; + }; + + packages.x86_64-linux.nixpkgs-swh-generate = pkgs.nixpkgs-swh-generate; + defaultPackage.x86_64-linux = pkgs.nixpkgs-swh-generate; + + nixosModules.nixpkgs-swh = { config, pkgs, lib, ... }: let + cfg = config.services.nixpkgs-swh; + dir = "/var/lib/nixpkgs-swh"; + in { + options = { + services.nixpkgs-swh = { + enable = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to run the nixpkgs-swh service. + ''; + }; + testing = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Whether to only evaluate the hello attribute for testing purpose. + ''; + }; + fqdn = lib.mkOption { + type = lib.types.str; + description = '' + The Nginx vhost FQDN used to serve built files. + ''; + }; + }; + }; + config = lib.mkIf cfg.enable { + nixpkgs.overlays = [ self.overlay ]; + systemd.services.nixpkgs-swh = { + description = "nixpkgs-swh"; + wantedBy = [ "multi-user.target" ]; + # Do it every day + startAt = "daily"; + script = '' + ${pkgs.nixpkgs-swh-generate}/bin/nixpkgs-swh-generate ${dir} ${if cfg.testing then "true" else "false"} unstable + ''; + }; + systemd.timers.nixpkgs-swh.timerConfig = { + Persistent = true; + }; + services.nginx.virtualHosts = { + "${cfg.fqdn}" = { + locations."/" = { + root = "${dir}"; + extraConfig = '' + autoindex on; + ''; + }; + }; + }; + }; + }; + }; +} diff --git a/scripts/analyze.py b/scripts/analyze.py index 7525e64..906fb14 100644 --- a/scripts/analyze.py +++ b/scripts/analyze.py @@ -5,6 +5,7 @@ import json import sys import re +import datetime from urllib.parse import urlparse with open(sys.argv[1], "r") as read_file: @@ -62,17 +63,19 @@ e['file-type'] = v break +today = datetime.date.today() readme = """ -The file [`sources-{release}-full.json`](https://nix-community.github.io/nixpkgs-swh/sources-{release}-full.json) +The file [`sources-{release}-full.json`](./sources-{release}-full.json) has been built from the [nixpkgs revision `{revision}`](https://github.com/NixOS/nixpkgs/tree/{revision}). This file contains `{sourceNumber}` sources, coming from `{hostNumber}` different hosts. -The file [`sources-{release}.json`](https://nix-community.github.io/nixpkgs-swh/sources-{release}.json) is a filtered version which only contains archives. This file is consumed by SWH. +The file [`sources-{release}.json`](./sources-{release}.json) is a filtered version which only contains archives. This file is consumed by Software Heritage to fill its archive. +Generated the {today}. """ sortedHosts = sorted(hosts.items(), key=lambda h: h[1], reverse=True) @@ -81,7 +84,8 @@ revision=j['revision'], release=j['release'], sourceNumber=len(sources), - hostNumber=len(sortedHosts))) + hostNumber=len(sortedHosts), + today=today)) print("\n#### By host\n") diff --git a/scripts/generate.sh b/scripts/generate.sh index 3f8c687..69a16c1 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -1,7 +1,4 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -#!nix-shell -I nixpkgs=./nix -#!nix-shell -p nix git openssh python3 curl jq +#!/usr/bin/env bash set -euo pipefail @@ -49,6 +46,8 @@ generate-release() { --argstr release ${RELEASE} \ --argstr evaluation ${EVAL_ID} \ --argstr timestamp $(date +%s) \ + --argstr find-tarballs $PWD/scripts/find-tarballs.nix \ + --arg testing $TESTING \ > ${SOURCES_FILE_FULL} echo "*** Add integrity attribute" @@ -66,23 +65,28 @@ generate-release() { generate-readme() { cat < ${DEST_DIR}/README.md -Fill the Software Heritage archive +# Fill the Software Heritage archive EOF -for i in $@; do - generate-release ${i} - echo "### NixOS \`${i}\`" >> ${DEST_DIR}/README.md - cat ${DEST_DIR}/readme-${i}.md >> ${DEST_DIR}/README.md - echo >> ${DEST_DIR}/README.md - echo >> ${DEST_DIR}/README.md - shift -done + for i in $@; do + generate-release ${i} + echo "### NixOS \`${i}\` release" >> ${DEST_DIR}/README.md + cat ${DEST_DIR}/readme-${i}.md >> ${DEST_DIR}/README.md + echo >> ${DEST_DIR}/README.md + echo >> ${DEST_DIR}/README.md + shift + done } - DEST_DIR=$1 mkdir -p ${DEST_DIR} shift +TESTING=$1 +shift + generate-readme $@ + +echo "** Converting README.md to index.html" +pandoc ${DEST_DIR}/README.md > ${DEST_DIR}/index.html diff --git a/scripts/swh-urls.nix b/scripts/swh-urls.nix index 0226bf1..1714045 100644 --- a/scripts/swh-urls.nix +++ b/scripts/swh-urls.nix @@ -1,12 +1,12 @@ -{ revision ? null, release ? null, evaluation ? null, timestamp ? null }: +{ revision ? null, release ? null, evaluation ? null, timestamp ? null, testing ? false, find-tarballs }: with builtins; let pkgs = import {}; mirrors = import ; expr = import ; - urls = import ./find-tarballs.nix {expr = expr;}; - + urls = import find-tarballs {expr = if testing then expr.hello else expr;}; + # This is avoid double slashes in urls that make url non valid concatUrls = a: b: (pkgs.lib.removeSuffix "/" a) + "/" + (pkgs.lib.removePrefix "/" b); @@ -20,7 +20,7 @@ let in if isMirrorUrl then [ url ] else map (r: concatUrls r path) resolvedUrls; - + # Transform the url list to swh format toSwh = s: { inherit (s) postFetch outputHashMode outputHashAlgo outputHash;