diff --git a/doc/stdenv/stdenv.xml b/doc/stdenv/stdenv.xml index f97c2a145af52..b0b08955164b6 100644 --- a/doc/stdenv/stdenv.xml +++ b/doc/stdenv/stdenv.xml @@ -475,12 +475,54 @@ passthru.updateScript = writeScript "update-zoom-us" '' passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ]; + Finally, the attribute can be an attribute set, listing the extra supported features among other things. + +passthru.updateScript = { + command = [ ../../update.sh pname ]; + supportedFeatures = [ "commit" ]; +}; + + + + The script will be usually run from the root of the Nixpkgs repository + but you should not rely on that. Also note that the update scripts will + be run in parallel by default; you should avoid running git + commit or any other commands that cannot handle that. + + + + Supported features + + + commit + + + + Whenever the update script exits with 0 return + status, it is expected to print a JSON list containing an object for + each updated attribute. Empty list can be returned when the script did + not update any files: for example, when the attribute is already the + latest version. The required keys can be seen below: + +[ + { + "attrName": "volume_key", + "oldVersion": "0.3.11", + "newVersion": "0.3.12", + "files": [ + "/path/to/nixpkgs/pkgs/development/libraries/volume-key/default.nix" + ] + } +] + + + + + - The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running git commit or any other commands that cannot handle that. - - - For information about how to run the updates, execute nix-shell maintainers/scripts/update.nix. + For information about how to run the updates, execute nix-shell + maintainers/scripts/update.nix. diff --git a/maintainers/scripts/update.nix b/maintainers/scripts/update.nix index 9568c6cbbccd9..e11e2450bd0ad 100755 --- a/maintainers/scripts/update.nix +++ b/maintainers/scripts/update.nix @@ -4,6 +4,7 @@ , max-workers ? null , include-overlays ? false , keep-going ? null +, commit ? null }: # TODO: add assert statements @@ -132,19 +133,26 @@ let --argstr keep-going true to continue running when a single update fails. + + You can also make the updater automatically commit on your behalf from updateScripts + that support it by adding + + --argstr commit true ''; packageData = package: { name = package.name; pname = lib.getName package; - updateScript = map builtins.toString (lib.toList package.updateScript); + updateScript = map builtins.toString (lib.toList (package.updateScript.command or package.updateScript)); + supportedFeatures = package.updateScript.supportedFeatures or []; }; packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages)); optionalArgs = lib.optional (max-workers != null) "--max-workers=${max-workers}" - ++ lib.optional (keep-going == "true") "--keep-going"; + ++ lib.optional (keep-going == "true") "--keep-going" + ++ lib.optional (commit == "true") "--commit"; args = [ packagesJson ] ++ optionalArgs; diff --git a/maintainers/scripts/update.py b/maintainers/scripts/update.py index eb7d0ef2647bf..6188bffa18ca8 100644 --- a/maintainers/scripts/update.py +++ b/maintainers/scripts/update.py @@ -1,22 +1,81 @@ +from typing import Dict, Generator, Tuple, Union import argparse +import contextlib import concurrent.futures import json import os import subprocess import sys +import tempfile +import threading -updates = {} +updates: Dict[concurrent.futures.Future, Dict] = {} + +TempDirs = Dict[str, Tuple[str, str, threading.Lock]] + +thread_name_prefix = 'UpdateScriptThread' def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) -def run_update_script(package): +def run_update_script(package: Dict, commit: bool, temp_dirs: TempDirs) -> subprocess.CompletedProcess: + worktree: Union[None, str] = None + + if commit and 'commit' in package['supportedFeatures']: + thread_name = threading.current_thread().name + worktree, _branch, lock = temp_dirs[thread_name] + lock.acquire() + package['thread'] = thread_name + eprint(f" - {package['name']}: UPDATING ...") - subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + return subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True, cwd=worktree) +@contextlib.contextmanager +def make_worktree() -> Generator[Tuple[str, str], None, None]: + with tempfile.TemporaryDirectory() as wt: + branch_name = f'update-{os.path.basename(wt)}' + target_directory = f'{wt}/nixpkgs' -def main(max_workers, keep_going, packages): + subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory], check=True) + yield (target_directory, branch_name) + subprocess.run(['git', 'worktree', 'remove', target_directory], check=True) + subprocess.run(['git', 'branch', '-D', branch_name], check=True) + +def commit_changes(worktree: str, branch: str, execution: subprocess.CompletedProcess) -> None: + changes = json.loads(execution.stdout) + for change in changes: + subprocess.run(['git', 'add'] + change['files'], check=True, cwd=worktree) + commit_message = '{attrName}: {oldVersion} → {newVersion}'.format(**change) + subprocess.run(['git', 'commit', '-m', commit_message], check=True, cwd=worktree) + subprocess.run(['git', 'cherry-pick', branch], check=True) + +def merge_changes(package: Dict, future: concurrent.futures.Future, commit: bool, keep_going: bool, temp_dirs: TempDirs) -> None: + try: + execution = future.result() + if commit and 'commit' in package['supportedFeatures']: + thread_name = package['thread'] + worktree, branch, lock = temp_dirs[thread_name] + commit_changes(worktree, branch, execution) + eprint(f" - {package['name']}: DONE.") + except subprocess.CalledProcessError as e: + eprint(f" - {package['name']}: ERROR") + eprint() + eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------") + eprint() + eprint(e.stdout.decode('utf-8')) + with open(f"{package['pname']}.log", 'wb') as logfile: + logfile.write(e.stdout) + eprint() + eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------") + + if not keep_going: + sys.exit(1) + finally: + if commit and 'commit' in package['supportedFeatures']: + lock.release() + +def main(max_workers: int, keep_going: bool, commit: bool, packages: Dict) -> None: with open(sys.argv[1]) as f: packages = json.load(f) @@ -31,29 +90,19 @@ def main(max_workers, keep_going, packages): eprint() eprint('Running update for:') - with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + with contextlib.ExitStack() as stack, concurrent.futures.ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix=thread_name_prefix) as executor: + if commit: + temp_dirs = {f'{thread_name_prefix}_{str(i)}': (*stack.enter_context(make_worktree()), threading.Lock()) for i in range(max_workers)} + else: + temp_dirs = {} + for package in packages: - updates[executor.submit(run_update_script, package)] = package + updates[executor.submit(run_update_script, package, commit, temp_dirs)] = package for future in concurrent.futures.as_completed(updates): package = updates[future] - try: - future.result() - eprint(f" - {package['name']}: DONE.") - except subprocess.CalledProcessError as e: - eprint(f" - {package['name']}: ERROR") - eprint() - eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------") - eprint() - eprint(e.stdout.decode('utf-8')) - with open(f"{package['pname']}.log", 'wb') as f: - f.write(e.stdout) - eprint() - eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------") - - if not keep_going: - sys.exit(1) + merge_changes(package, future, commit, keep_going, temp_dirs) eprint() eprint('Packages updated!') @@ -65,13 +114,14 @@ def main(max_workers, keep_going, packages): parser = argparse.ArgumentParser(description='Update packages') parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4) parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure') +parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes') parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts') if __name__ == '__main__': args = parser.parse_args() try: - main(args.max_workers, args.keep_going, args.packages) + main(args.max_workers, args.keep_going, args.commit, args.packages) except (KeyboardInterrupt, SystemExit) as e: for update in updates: update.cancel() diff --git a/pkgs/common-updater/scripts/update-source-version b/pkgs/common-updater/scripts/update-source-version index 6a66f94597f48..0bfa2b71242ce 100755 --- a/pkgs/common-updater/scripts/update-source-version +++ b/pkgs/common-updater/scripts/update-source-version @@ -11,7 +11,7 @@ die() { usage() { echo "Usage: $scriptName [] []" echo " [--version-key=] [--system=] [--file=]" - echo " [--ignore-same-hash]" + echo " [--ignore-same-hash] [--print-changes]" } args=() @@ -33,6 +33,9 @@ for arg in "$@"; do --ignore-same-hash) ignoreSameHash="true" ;; + --print-changes) + printChanges="true" + ;; --help) usage exit 0 @@ -102,6 +105,9 @@ fi if [[ "$oldVersion" = "$newVersion" ]]; then echo "$scriptName: New version same as old version, nothing to do." >&2 + if [ -n "$printChanges" ]; then + printf '[]\n' + fi exit 0 fi @@ -197,3 +203,7 @@ fi rm -f "$nixFile.bak" rm -f "$attr.fetchlog" + +if [ -n "$printChanges" ]; then + printf '[{"attrName":"%s","oldVersion":"%s","newVersion":"%s","files":["%s"]}]\n' "$attr" "$oldVersion" "$newVersion" "$nixFile" +fi diff --git a/pkgs/desktops/gnome-3/update.nix b/pkgs/desktops/gnome-3/update.nix index 1bceddf77eb5b..78cefdd0b0a7c 100644 --- a/pkgs/desktops/gnome-3/update.nix +++ b/pkgs/desktops/gnome-3/update.nix @@ -21,6 +21,9 @@ let version_policy="$3" PATH=${lib.makeBinPath [ common-updater-scripts python ]} latest_tag=$(python "${./find-latest-version.py}" "$package_name" "$version_policy" "stable" ${upperBoundFlag}) - update-source-version "$attr_path" "$latest_tag" + update-source-version "$attr_path" "$latest_tag" --print-changes ''; -in [ updateScript packageName attrPath versionPolicy ] +in { + command = [ updateScript packageName attrPath versionPolicy ]; + supportedFeatures = [ "commit" ]; +} diff --git a/pkgs/development/python-modules/asyncio-pool/default.nix b/pkgs/development/python-modules/asyncio-pool/default.nix new file mode 100644 index 0000000000000..0554287812383 --- /dev/null +++ b/pkgs/development/python-modules/asyncio-pool/default.nix @@ -0,0 +1,41 @@ +{ stdenv +, buildPythonPackage +, fetchPypi +, pythonOlder +, pytest +, pytest-asyncio +, async-timeout +}: + +buildPythonPackage rec { + pname = "asyncio-pool"; + version = "0.5.1"; + + src = fetchPypi { + pname = "asyncio_pool"; + inherit version; + sha256 = "06ygc7g8hpzgwv91ic7v5vsaqr725dnwh4ysbrj91yppaq1djz5c"; + }; + + checkInputs = [ + pytest + pytest-asyncio + async-timeout + ]; + + checkPhase = '' + pytest tests + ''; + + # tarball missing tests + # https://github.com/gistart/asyncio-pool/issues/2 + doCheck = false; + + disabled = pythonOlder "3.5"; + + meta = with stdenv.lib; { + description = "Pool for asyncio with multiprocessing, threading and gevent-like interface"; + license = licenses.mit; + maintainers = with maintainers; [ jtojnar ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index b837acff87847..525741c89653a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -414,6 +414,8 @@ in { async_generator = callPackage ../development/python-modules/async_generator { }; + asyncio-pool = callPackage ../development/python-modules/asyncio-pool { }; + asyncpg = callPackage ../development/python-modules/asyncpg { }; asyncssh = callPackage ../development/python-modules/asyncssh { };