maintainers/scripts/update.nix: Add support for auto-commiting changes

doc/stdenv/stdenv.xml

@@ -475,12 +475,54 @@ passthru.updateScript = writeScript "update-zoom-us" ''
 <programlisting>
 passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ];
 </programlisting>
+      Finally, the attribute can be an attribute set, listing the extra supported features among other things.
+<programlisting>
+passthru.updateScript = {
+  command = [ ../../update.sh pname ];
+  supportedFeatures = [ "commit" ];
+};
+</programlisting>
+      <note>
+       <para>
+        The script will be usually run from the root of the Nixpkgs repository
+        but you should not rely on that. Also note that the update scripts will
+        be run in parallel by default; you should avoid running <command>git
+        commit</command> or any other commands that cannot handle that.
+       </para>
+      </note>
      </para>
+     <variablelist>
+      <title>Supported features</title>
+      <varlistentry>
+       <term>
+        <varname>commit</varname>
+       </term>
+       <listitem>
+        <para>
+         Whenever the update script exits with <literal>0</literal> return
+         status, it is expected to print a JSON list containing an object for
+         each updated attribute. Empty list can be returned when the script did
+         not update any files: for example, when the attribute is already the
+         latest version. The required keys can be seen below:
+<programlisting>
+[
+  {
+    "attrName": "volume_key",
+    "oldVersion": "0.3.11",
+    "newVersion": "0.3.12",
+    "files": [
+      "/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix"
+    ]
+  }
+]
+</programlisting>
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
      <para>
-      The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
-     </para>
-     <para>
-      For information about how to run the updates, execute <command>nix-shell maintainers/scripts/update.nix</command>.
+      For information about how to run the updates, execute <command>nix-shell
+      maintainers/scripts/update.nix</command>.
      </para>
     </listitem>
    </varlistentry>

maintainers/scripts/update.nix

@@ -4,6 +4,7 @@
 , max-workers ? null
 , include-overlays ? false
 , keep-going ? null
+, commit ? null
 }:
 
 # TODO: add assert statements
@@ -132,19 +133,26 @@ let
         --argstr keep-going true
 
     to continue running when a single update fails.
+
+    You can also make the updater automatically commit on your behalf from updateScripts
+    that support it by adding
+
+        --argstr commit true
   '';
 
   packageData = package: {
     name = package.name;
     pname = lib.getName package;
-    updateScript = map builtins.toString (lib.toList package.updateScript);
+    updateScript = map builtins.toString (lib.toList (package.updateScript.command or package.updateScript));
+    supportedFeatures = package.updateScript.supportedFeatures or [];
   };
 
   packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages));
 
   optionalArgs =
     lib.optional (max-workers != null) "--max-workers=${max-workers}"
-    ++ lib.optional (keep-going == "true") "--keep-going";
+    ++ lib.optional (keep-going == "true") "--keep-going"
+    ++ lib.optional (commit == "true") "--commit";
 
   args = [ packagesJson ] ++ optionalArgs;
 

maintainers/scripts/update.py

@@ -1,22 +1,81 @@
+from typing import Dict, Generator, Tuple, Union
 import argparse
+import contextlib
 import concurrent.futures
 import json
 import os
 import subprocess
 import sys
+import tempfile
+import threading
 
-updates = {}
+updates: Dict[concurrent.futures.Future, Dict] = {}
+
+TempDirs = Dict[str, Tuple[str, str, threading.Lock]]
+
+thread_name_prefix = 'UpdateScriptThread'
 
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
 
-def run_update_script(package):
+def run_update_script(package: Dict, commit: bool, temp_dirs: TempDirs) -> subprocess.CompletedProcess:
+    worktree: Union[None, str] = None
+
+    if commit and 'commit' in package['supportedFeatures']:
+        thread_name = threading.current_thread().name
+        worktree, _branch, lock = temp_dirs[thread_name]
+        lock.acquire()
+        package['thread'] = thread_name
+
     eprint(f" - {package['name']}: UPDATING ...")
 
-    subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
+    return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree)
 
+@contextlib.contextmanager
+def make_worktree() -> Generator[Tuple[str, str], None, None]:
+    with tempfile.TemporaryDirectory() as wt:
+        branch_name = f'update-{os.path.basename(wt)}'
+        target_directory = f'{wt}/nixpkgs'
 
-def main(max_workers, keep_going, packages):
+        subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True)
+        yield (target_directory, branch_name)
+        subprocess.run(['git', 'worktree', 'remove', target_directory], check=True)
+        subprocess.run(['git', 'branch', '-D', branch_name], check=True)
+
+def commit_changes(worktree: str, branch: str, execution: subprocess.CompletedProcess) -> None:
+    changes = json.loads(execution.stdout)
+    for change in changes:
+        subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree)
+        commit_message = '{attrName}: {oldVersion} → {newVersion}'.format(**change)
+        subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree)
+        subprocess.run(['git', 'cherry-pick', branch], check=True)
+
+def merge_changes(package: Dict, future: concurrent.futures.Future, commit: bool, keep_going: bool, temp_dirs: TempDirs) -> None:
+    try:
+        execution = future.result()
+        if commit and 'commit' in package['supportedFeatures']:
+            thread_name = package['thread']
+            worktree, branch, lock = temp_dirs[thread_name]
+            commit_changes(worktree, branch, execution)
+        eprint(f" - {package['name']}: DONE.")
+    except subprocess.CalledProcessError as e:
+        eprint(f" - {package['name']}: ERROR")
+        eprint()
+        eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
+        eprint()
+        eprint(e.stdout.decode('utf-8'))
+        with open(f"{package['pname']}.log", 'wb') as logfile:
+            logfile.write(e.stdout)
+        eprint()
+        eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
+
+        if not keep_going:
+            sys.exit(1)
+    finally:
+        if commit and 'commit' in package['supportedFeatures']:
+            lock.release()
+
+def main(max_workers: int, keep_going: bool, commit: bool, packages: Dict) -> None:
     with open(sys.argv[1]) as f:
         packages = json.load(f)
 
@@ -31,29 +90,19 @@ def main(max_workers, keep_going, packages):
         eprint()
         eprint('Running update for:')
 
-        with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
+        with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor:
+            if commit:
+                temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)}
+            else:
+                temp_dirs = {}
+
             for package in packages:
-                updates[executor.submit(run_update_script, package)] = package
+                updates[executor.submit(run_update_script, package, commit, temp_dirs)] = package
 
             for future in concurrent.futures.as_completed(updates):
                 package = updates[future]
 
-                try:
-                    future.result()
-                    eprint(f" - {package['name']}: DONE.")
-                except subprocess.CalledProcessError as e:
-                    eprint(f" - {package['name']}: ERROR")
-                    eprint()
-                    eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
-                    eprint()
-                    eprint(e.stdout.decode('utf-8'))
-                    with open(f"{package['pname']}.log", 'wb') as f:
-                        f.write(e.stdout)
-                    eprint()
-                    eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
-
-                    if not keep_going:
-                        sys.exit(1)
+                merge_changes(package, future, commit, keep_going, temp_dirs)
 
         eprint()
         eprint('Packages updated!')
@@ -65,13 +114,14 @@ def main(max_workers, keep_going, packages):
 parser = argparse.ArgumentParser(description='Update packages')
 parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
 parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
+parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
 parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
 
 if __name__ == '__main__':
     args = parser.parse_args()
 
     try:
-        main(args.max_workers, args.keep_going, args.packages)
+        main(args.max_workers, args.keep_going, args.commit, args.packages)
     except (KeyboardInterrupt, SystemExit) as e:
         for update in updates:
             update.cancel()

pkgs/common-updater/scripts/update-source-version

@@ -11,7 +11,7 @@ die() {
 usage() {
     echo "Usage: $scriptName <attr> <version> [<new-source-hash>] [<new-source-url>]"
     echo "                              [--version-key=<version-key>] [--system=<system>] [--file=<file-to-update>]"
-    echo "                              [--ignore-same-hash]"
+    echo "                              [--ignore-same-hash] [--print-changes]"
 }
 
 args=()
@@ -33,6 +33,9 @@ for arg in "$@"; do
         --ignore-same-hash)
             ignoreSameHash="true"
         ;;
+        --print-changes)
+            printChanges="true"
+        ;;
         --help)
             usage
             exit 0
@@ -102,6 +105,9 @@ fi
 
 if [[ "$oldVersion" = "$newVersion" ]]; then
     echo "$scriptName: New version same as old version, nothing to do." >&2
+    if [ -n "$printChanges" ]; then
+        printf '[]\n'
+    fi
     exit 0
 fi
 
@@ -197,3 +203,7 @@ fi
 
 rm -f "$nixFile.bak"
 rm -f "$attr.fetchlog"
+
+if [ -n "$printChanges" ]; then
+    printf '[{"attrName":"%s","oldVersion":"%s","newVersion":"%s","files":["%s"]}]\n' "$attr" "$oldVersion" "$newVersion" "$nixFile"
+fi

pkgs/desktops/gnome-3/update.nix

@@ -21,6 +21,9 @@ let
     version_policy="$3"
     PATH=${lib.makeBinPath [ common-updater-scripts python ]}
     latest_tag=$(python "${./find-latest-version.py}" "$package_name" "$version_policy" "stable" ${upperBoundFlag})
-    update-source-version "$attr_path" "$latest_tag"
+    update-source-version "$attr_path" "$latest_tag" --print-changes
   '';
-in [ updateScript packageName attrPath versionPolicy ]
+in {
+  command = [ updateScript packageName attrPath versionPolicy ];
+  supportedFeatures = [ "commit" ];
+}

pkgs/development/python-modules/asyncio-pool/default.nix

@@ -0,0 +1,41 @@
+{ stdenv
+, buildPythonPackage
+, fetchPypi
+, pythonOlder
+, pytest
+, pytest-asyncio
+, async-timeout
+}:
+
+buildPythonPackage rec {
+  pname = "asyncio-pool";
+  version = "0.5.1";
+
+  src = fetchPypi {
+    pname = "asyncio_pool";
+    inherit version;
+    sha256 = "06ygc7g8hpzgwv91ic7v5vsaqr725dnwh4ysbrj91yppaq1djz5c";
+  };
+
+  checkInputs = [
+    pytest
+    pytest-asyncio
+    async-timeout
+  ];
+
+  checkPhase = ''
+    pytest tests
+  '';
+
+  # tarball missing tests
+  # https://github.com/gistart/asyncio-pool/issues/2
+  doCheck = false;
+
+  disabled = pythonOlder "3.5";
+
+  meta = with stdenv.lib; {
+    description = "Pool for asyncio with multiprocessing, threading and gevent-like interface";
+    license = licenses.mit;
+    maintainers = with maintainers; [ jtojnar ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -414,6 +414,8 @@ in {
 
   async_generator = callPackage ../development/python-modules/async_generator { };
 
+  asyncio-pool = callPackage ../development/python-modules/asyncio-pool { };
+
   asyncpg = callPackage ../development/python-modules/asyncpg { };
 
   asyncssh = callPackage ../development/python-modules/asyncssh { };