RFC: More common updater script implementations

pkgs/applications/networking/browsers/firefox/default.nix

@@ -5,7 +5,7 @@
 , hunspell, libevent, libstartup_notification, libvpx
 , cairo, gstreamer, gst_plugins_base, icu, libpng, jemalloc, libpulseaudio
 , autoconf213, which
-, writeScript, xidel, coreutils, gnused, gnugrep, curl, ed
+, writeScript, xidel, common-updater-scripts, coreutils, gnused, gnugrep, curl
 , enableGTK3 ? false
 , debugBuild ? false
 , # If you want the resulting program to call itself "Firefox" instead
@@ -151,8 +151,8 @@ in {
     version = "50.1.0";
     sha512 = "370d2e9b8c4b1b59c3394659c3a7f0f79e6a911ccd9f32095b50b3a22d087132b1f7cb87b734f7497c4381b1df6df80d120b4b87c13eecc425cc66f56acccba5";
     updateScript = import ./update.nix {
-        name = "firefox";
-        inherit writeScript xidel coreutils gnused gnugrep curl ed;
+      attrPath = "firefox-unwrapped";
+      inherit writeScript lib common-updater-scripts xidel coreutils gnused gnugrep curl;
     };
   };
 
@@ -161,9 +161,9 @@ in {
     version = "45.6.0esr";
     sha512 = "b96c71aeed8a1185a085512f33d454a1735237cd9ddf37c8caa9cc91892eafab0615fc0ca6035f282ca8101489fa84c0de1087d1963c05b64df32b0c86446610";
     updateScript = import ./update.nix {
-        name = "firefox-esr";
-        versionSuffix = "esr";
-        inherit writeScript xidel coreutils gnused gnugrep curl ed;
+      attrPath = "firefox-esr-unwrapped";
+      versionSuffix = "esr";
+      inherit writeScript lib common-updater-scripts xidel coreutils gnused gnugrep curl;
     };
   };
 

pkgs/applications/networking/browsers/firefox/update.nix

@@ -1,23 +1,18 @@
-{ name
-, writeScript
+{ writeScript
+, lib
 , xidel
+, common-updater-scripts
 , coreutils
 , gnused
 , gnugrep
 , curl
-, ed
-, sourceSectionRegex ? "${name}-unwrapped = common"
-, basePath ? "pkgs/applications/networking/browsers/firefox"
+, attrPath
 , baseUrl ? "http://archive.mozilla.org/pub/firefox/releases/"
 , versionSuffix ? ""
 }:
 
-let
-  version = (builtins.parseDrvName name).version;
-in writeScript "update-${name}" ''
-  PATH=${coreutils}/bin:${gnused}/bin:${gnugrep}/bin:${xidel}/bin:${curl}/bin:${ed}/bin
-
-  pushd ${basePath}
+writeScript "update-${attrPath}" ''
+  PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep gnused xidel ]}
 
   url=${baseUrl}
 
@@ -35,20 +30,5 @@ in writeScript "update-${name}" ''
 
   shasum=`curl --silent $url$version/SHA512SUMS | grep 'source\.tar\.xz' | cut -d ' ' -f 1`
 
-  ed default.nix <<COMMANDS
-  # search line
-    /${sourceSectionRegex}/
-  # search version number line
-    /version/
-  # update the version
-    s/".*"/"$version"/
-  # search hash line
-    /sha512/
-  # update the hash
-    s/".*"/"$shasum"/
-  # write then quit
-    wq
-  COMMANDS
-
-  popd
+  update-source-version ${attrPath} "$version" "$shasum"
 ''

pkgs/applications/networking/mailreaders/thunderbird/default.nix

@@ -1,10 +1,10 @@
-{ stdenv, fetchurl, pkgconfig, which, m4, gtk2, pango, perl, python2, zip, libIDL
+{ stdenv, lib, fetchurl, pkgconfig, which, m4, gtk2, pango, perl, python2, zip, libIDL
 , libjpeg, libpng, zlib, dbus, dbus_glib, bzip2, xorg
 , freetype, fontconfig, file, alsaLib, nspr, nss, libnotify
 , yasm, mesa, sqlite, unzip, makeWrapper
 , hunspell, libevent, libstartup_notification, libvpx
 , cairo, gstreamer, gst_plugins_base, icu
-, writeScript, xidel, coreutils, gnused, gnugrep, curl, ed
+, writeScript, xidel, common-updater-scripts, coreutils, gnused, gnugrep, curl
 , debugBuild ? false
 , # If you want the resulting program to call itself "Thunderbird"
   # instead of "Earlybird", enable this option.  However, those
@@ -138,10 +138,8 @@ stdenv.mkDerivation rec {
   };
 
   passthru.updateScript = import ./../../browsers/firefox/update.nix {
-    name = "thunderbird";
-    sourceSectionRegex = ".";
-    basePath = "pkgs/applications/networking/mailreaders/thunderbird";
+    attrPath = "thunderbird";
     baseUrl = "http://archive.mozilla.org/pub/thunderbird/releases/";
-    inherit writeScript xidel coreutils gnused gnugrep curl ed;
+    inherit writeScript lib common-updater-scripts xidel coreutils gnused gnugrep curl;
   };
 }

pkgs/common-updater/scripts.nix

@@ -0,0 +1,18 @@
+{ stdenv, makeWrapper, coreutils, gawk, gnused, nix }:
+
+stdenv.mkDerivation {
+  name = "common-updater-scripts";
+
+  buildInputs = [ makeWrapper ];
+
+  unpackPhase = "true";
+
+  installPhase = ''
+    mkdir -p $out/bin
+    cp ${./scripts}/* $out/bin
+
+    for f in $out/bin/*; do
+      wrapProgram $f --prefix PATH : ${stdenv.lib.makeBinPath [ coreutils gawk gnused nix ]}
+    done
+  '';
+}

pkgs/common-updater/scripts/update-source-version

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+set -e
+
+die() {
+    echo "$0: error: $1" >&2
+    exit 1
+}
+
+# Usage: update-source-hash <attr> <version> [<new-source-hash>]
+attr=$1
+newVersion=$2
+newHash=$3
+
+nixFile=$(nix-instantiate --eval --strict -A "$attr.meta.position" | sed -re 's/^"(.*):[0-9]+"$/\1/')
+if [ ! -f "$nixFile" ]; then
+    die "Couldn't evaluate '$attr.meta.position' to locate the .nix file!"
+fi
+
+oldHashAlgo=$(nix-instantiate --eval --strict -A "$attr.src.drvAttrs.outputHashAlgo" | tr -d '"')
+oldHash=$(nix-instantiate --eval --strict -A "$attr.src.drvAttrs.outputHash" | tr -d '"')
+
+if [ -z "$oldHashAlgo" -o -z "$oldHash" ]; then
+    die "Couldn't evaluate old source hash from '$attr.src'!"
+fi
+
+if [ $(grep -c "$oldHash" "$nixFile") != 1 ]; then
+    die "Couldn't locate old source hash '$oldHash' (or it appeared more than once) in '$nixFile'!"
+fi
+
+drvName=$(nix-instantiate --eval -E "with import ./. {}; (builtins.parseDrvName $attr.name).name" | tr -d '"')
+oldVersion=$(nix-instantiate --eval -E "with import ./. {}; $attr.version or (builtins.parseDrvName $attr.name).version" | tr -d '"')
+
+if [ -z "$drvName" -o -z "$oldVersion" ]; then
+    die "Couldn't evaluate name and version from '$attr.name'!"
+fi
+
+if [ "$oldVersion" = "$newVersion" ]; then
+    echo "$0: New version same as old version, nothing to do." >&2
+    exit 0
+fi
+
+# Escape dots, there should not be any other regex characters allowed in store path names
+oldVersion=$(echo "$oldVersion" | sed -re 's|\.|\\.|g')
+
+if [ $(grep -c -E "^\s*(let\b)?\s*version\s+=\s+\"$oldVersion\"" "$nixFile") = 1 ]; then
+    pattern="/\bversion\b\s*=/ s|\"$oldVersion\"|\"$newVersion\"|"
+elif [ $(grep -c -E "^\s*(let\b)?\s*name\s+=\s+\"$drvName-$oldVersion\"" "$nixFile") = 1 ]; then
+    pattern="/\bname\b\s*=/ s|\"$drvName-$oldVersion\"|\"$drvName-$newVersion\"|"
+else
+    die "Couldn't figure out where out where to patch in new version in '$attr'!"
+fi
+
+# Replace new version
+sed -i.bak "$nixFile" -re "$pattern"
+if cmp -s "$nixFile" "$nixFile.bak"; then
+    die "Failed to replace version '$oldVersion' to '$newVersion' in '$attr'!"
+fi
+
+case "$oldHashAlgo" in
+    sha256) hashLength=64 ;;
+    sha512) hashLength=128 ;;
+    *) die "Unhandled hash algorithm '$oldHashAlgo' in '$attr'!" ;;
+esac
+
+# Make a temporary all-zeroes hash of $hashLength characters
+tempHash=$(printf '%0*d' "$hashLength" 0)
+
+sed -i "$nixFile" -re "s|\"$oldHash\"|\"$tempHash\"|"
+if cmp -s "$nixFile" "$nixFile.bak"; then
+    die "Failed to replace source hash of '$attr' to a temporary hash!"
+fi
+
+# If new hash not given on the command line, recalculate it ourselves.
+if [ -z "$newHash" ]; then
+    nix-build --no-out-link -A "$attr.src" 2>"$attr.fetchlog" >/dev/null || true
+    # FIXME: use nix-build --hash here once https://github.com/NixOS/nix/issues/1172 is fixed
+    newHash=$(tail -n2 "$attr.fetchlog" | grep "output path .* has .* hash .* when .* was expected" | head -n1 | tr -dc '\040-\177' | awk '{ print $(NF-4) }')
+fi
+
+if [ -z "$newHash" ]; then
+    cat "$attr.fetchlog" >&2
+    die "Couldn't figure out new hash of '$attr.src'!"
+fi
+
+sed -i "$nixFile" -re "s|\"$tempHash\"|\"$newHash\"|"
+if cmp -s "$nixFile" "$nixFile.bak"; then
+    die "Failed to replace temporary source hash of '$attr' to the final source hash!"
+fi
+
+rm -f "$nixFile.bak"
+rm -f "$attr.fetchlog"

pkgs/os-specific/linux/tp_smapi/default.nix

@@ -1,12 +1,17 @@
-{ stdenv, fetchurl, kernel, writeScript, coreutils, gnugrep, jq, curl
+{ stdenv, lib, fetchFromGitHub, kernel, writeScript, coreutils, gnugrep, jq, curl, common-updater-scripts
 }:
 
-let
-  data = stdenv.lib.importJSON ./update.json;
-in stdenv.mkDerivation rec {
-  name = "tp_smapi-${data.version}-${kernel.version}";
+stdenv.mkDerivation rec {
+  name = "tp_smapi-${version}-${kernel.version}";
+  version = "0.42";
 
-  src = fetchurl  { inherit (data) url sha256; };
+  src = fetchFromGitHub {
+    owner = "evgeni";
+    repo = "tp_smapi";
+    rev = "tp-smapi/${version}";
+    sha256 = "12lnig90lrmkmqwl386q7ssqs9p0jikqhwl2wsmcmii1gn92hzfy";
+    name = "tp-smapi-${version}";
+  };
 
   hardeningDisable = [ "pic" ];
 
@@ -25,7 +30,7 @@ in stdenv.mkDerivation rec {
   enableParallelBuilding = true;
 
   passthru.updateScript = import ./update.nix {
-    inherit writeScript coreutils gnugrep jq curl;
+    inherit lib writeScript coreutils gnugrep jq curl common-updater-scripts;
   };
 
   meta = {

pkgs/os-specific/linux/tp_smapi/update.nix

@@ -1,23 +1,10 @@
-{ writeScript, coreutils, gnugrep, jq, curl
-}:
+{ lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }:
 
 writeScript "update-tp_smapi" ''
-PATH=${coreutils}/bin:${gnugrep}/bin:${jq}/bin:${curl}/bin
+PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq ]}
 
-pushd pkgs/os-specific/linux/tp_smapi
-
-tmpfile=`mktemp`
 tags=`curl -s https://api.github.com/repos/evgeni/tp_smapi/tags`
 latest_tag=`echo $tags | jq -r '.[] | .name' | grep -oP "^tp-smapi/\K.*" | sort --version-sort | tail -1`
-sha256=`curl -sL "https://github.com/evgeni/tp_smapi/archive/tp-smapi/$latest_tag.tar.gz" | sha256sum | cut -d" " -f1`
-
-cat > update.json <<EOF
-{
-  "version": "$latest_tag",
-  "url": "https://github.com/evgeni/tp_smapi/archive/tp-smapi/$latest_tag.tar.gz",
-  "sha256": "$sha256"
-}
-EOF
 
-popd
+update-source-version linuxPackages.tp_smapi "$latest_tag"
 ''

pkgs/top-level/all-packages.nix

@@ -88,6 +88,7 @@ in
 
   nixpkgs-lint = callPackage ../../maintainers/scripts/nixpkgs-lint.nix { };
 
+  common-updater-scripts = callPackage ../common-updater/scripts.nix { };
 
   ### BUILD SUPPORT