From 54b0e5c2ff4fe95d95b6f4da95b6b4f12637a549 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Mon, 22 Aug 2022 15:17:36 +0300 Subject: [PATCH] scripts/publish.py: replace toposort dependency with python std graphlib module Python standard library already includes a topological sorter: https://docs.python.org/3/library/graphlib.html We don't need a third party dependency in scripts/publish.py. Closes #2965 --- scripts/publish.py | 441 +++++++++++++++++++++++++++++---------------- 1 file changed, 284 insertions(+), 157 deletions(-) diff --git a/scripts/publish.py b/scripts/publish.py index 9afcafc07e0..3076bf413ec 100644 --- a/scripts/publish.py +++ b/scripts/publish.py @@ -1,199 +1,326 @@ #! /usr/bin/env python3 -# This is a script for publishing the wasmer crates to crates.io. -# It should be run in the root of wasmer like `python3 scripts/publish.py --no-dry-run`. -# By default the script executes a test run and does not publish the crates to crates.io. - -# install dependencies: -# pip3 install toposort +"""This is a script for publishing the wasmer crates to crates.io. +It should be run in the root of wasmer like `python3 scripts/publish.py --help`.""" import argparse +import itertools import os import re import subprocess import time +import sys +import typing +from pprint import pprint -from typing import Optional +try: + # try import tomllib from standard Python library: New in version 3.11. + import tomllib +except ModuleNotFoundError: + try: + # if tomllib is missing, this is an older python3 version. Try + # importing the equivalent third-party library tomli: + import tomli as tomllib + except ModuleNotFoundError: + print("Please install tomli, `pip3 install tomli`") + sys.exit(1) try: - from toposort import toposort_flatten + from graphlib import TopologicalSorter except ImportError: - print("Please install toposort, `pip3 install toposort`") - - -# TODO: find this automatically -target_version = "3.0.0-beta" - -# TODO: generate this by parsing toml files -dep_graph = { - "wasmer-types": set([]), - "wasmer-derive": set([]), - "wasmer-vm": set(["wasmer-types"]), - "wasmer-compiler": set(["wasmer-types"]), - "wasmer-compiler-singlepass": set(["wasmer-types", "wasmer-compiler"]), - "wasmer-compiler-cranelift": set(["wasmer-types", "wasmer-compiler"]), - "wasmer-compiler-cli": set( - [ - "wasmer-compiler", - "wasmer-types", - "wasmer-compiler-singlepass", - "wasmer-compiler-cranelift", - ] - ), - "wasmer-object": set(["wasmer-types", "wasmer-compiler"]), - "wasmer-compiler-llvm": set(["wasmer-compiler", "wasmer-vm", "wasmer-types"]), - "wasmer": set( - [ - "wasmer-vm", - "wasmer-compiler", - "wasmer-derive", - "wasmer-types", - "wasmer-compiler-singlepass", - "wasmer-compiler-cranelift", - "wasmer-compiler-llvm", - ] - ), - "wasmer-vfs": set([]), - "wasmer-vbus": set([]), - "wasmer-vnet": set([]), - "wasmer-wasi-local-networking": set([]), - "wasmer-cache": set(["wasmer"]), - "wasmer-wasi": set(["wasmer", "wasmer-wasi-types", "wasmer-vfs", "wasmer-vbus", "wasmer-vnet"]), - "wasmer-wasi-types": set(["wasmer-types"]), - "wasmer-wasi-experimental-io-devices": set(["wasmer-wasi"]), - "wasmer-emscripten": set(["wasmer"]), - "wasmer-c-api": set( - [ - "wasmer", - "wasmer-compiler", - "wasmer-compiler-cranelift", - "wasmer-compiler-singlepass", - "wasmer-compiler-llvm", - "wasmer-emscripten", - "wasmer-middlewares", - "wasmer-wasi", - "wasmer-types", - ] - ), - "wasmer-middlewares": set(["wasmer", "wasmer-types", "wasmer-vm"]), - "wasmer-wast": set(["wasmer", "wasmer-wasi", "wasmer-vfs"]), - "wasmer-cli": set( - [ - "wasmer", - "wasmer-compiler", - "wasmer-compiler-cranelift", - "wasmer-compiler-singlepass", - "wasmer-compiler-llvm", - "wasmer-emscripten", - "wasmer-vm", - "wasmer-wasi", - "wasmer-wasi-experimental-io-devices", - "wasmer-wast", - "wasmer-cache", - "wasmer-types", - "wasmer-vfs", - ] - ), -} + print("Incompatible python3 version: lacks graphlib standard library module.") + sys.exit(1) -# where each crate is located in the `lib` directory -# TODO: this could also be generated from the toml files -location = { - "wasmer-compiler-cli": "cli-compiler", - "wasmer-types": "types", - "wasmer-derive": "derive", - "wasmer-vm": "vm", - "wasmer-compiler": "compiler", - "wasmer-object": "object", - "wasmer-compiler-singlepass": "compiler-singlepass", - "wasmer-compiler-cranelift": "compiler-cranelift", - "wasmer-compiler-llvm": "compiler-llvm", - "wasmer-cache": "cache", - "wasmer": "api", - "wasmer-wasi": "wasi", - "wasmer-wasi-types": "wasi-types", - "wasmer-emscripten": "emscripten", - "wasmer-wasi-experimental-io-devices": "wasi-experimental-io-devices", - "wasmer-c-api": "c-api", - "wasmer-middlewares": "middlewares", - "wasmer-vfs": "vfs", - "wasmer-vbus": "vbus", - "wasmer-vnet": "vnet", - "wasmer-cli": "cli", - "wasmer-wasi-local-networking": "wasi-local-networking", - "wasmer-wast": "../tests/lib/wast", +SETTINGS = { + # extra features to put when publishing, for example wasmer-cli needs a + # compiler by default otherwise it won't work standalone + "publish_features": { + "wasmer-cli": "default,cranelift", + }, + # workspace members we want to publish but whose path doesn't start by + # "./lib/" + "non-lib-workspace-members": set(["tests/lib/wast"]), } -no_dry_run = False - -def get_latest_version_for_crate(crate_name: str) -> Optional[str]: - output = subprocess.run(["cargo", "search", crate_name], capture_output=True) - rexp_src = '^{} = "([^"]+)"'.format(crate_name) +def get_latest_version_for_crate(crate_name: str) -> typing.Optional[str]: + """Fetches the latest version of a given crate name in local cargo registry.""" + output = subprocess.run( + ["cargo", "search", crate_name], capture_output=True, check=True + ) + rexp_src = f'^{crate_name} = "([^"]+)"' prog = re.compile(rexp_src) haystack = output.stdout.decode("utf-8") for line in haystack.splitlines(): result = prog.match(line) if result: return result.group(1) + return None + + +class Crate: + """Represents a workspace crate that is to be published to crates.io.""" + + def __init__( + self, + dependencies: typing.List[str], + cargo_manifest: dict, + cargo_file_path="Cargo.toml", + ): + self.name = cargo_manifest["package"]["name"] + self.dependencies = dependencies + self.cargo_manifest = cargo_manifest + if not os.path.isabs(cargo_file_path): + cargo_file_path = os.path.join(os.getcwd(), cargo_file_path) + self.cargo_file_path = cargo_file_path + + def __str__(self): + return f"{self.name}: {self.dependencies} {self.cargo_file_path} {self.path()}" + + def path(self) -> str: + """Return the absolute filesystem path containing this crate.""" + return os.path.dirname(self.cargo_file_path) + + +class Publisher: + """A class responsible for figuring out dependencies, + creating a topological sorting in order to publish them + to crates.io in a valid order.""" + + def __init__(self, version=None, dry_run=True, verbose=True): + self.dry_run = dry_run + self.verbose = verbose + + # open workspace Cargo.toml + with open("Cargo.toml", "rb") as file: + data = tomllib.load(file) + + if version is None: + version = data["package"]["version"] + self.version = version + + if self.verbose and not self.dry_run: + print(f"Publishing version {self.version}") + elif self.verbose and self.dry_run: + print(f"Publishing version {self.version} dry run!") + + # define helper function + check_local_dep_fn = lambda t: isinstance(t[1], dict) and "path" in t[1] + members = set( + map( + lambda p: p + "/Cargo.toml", + filter( + lambda path: ( + path.startswith("lib/") and os.path.exists(path + "/Cargo.toml") + ) + or path in SETTINGS["non-lib-workspace-members"], + itertools.chain( + data["workspace"]["members"], + map( + lambda p: p[1]["path"], + filter(check_local_dep_fn, data["dependencies"].items()), + ), + ), + ), + ) + ) + crates = [] + for member in members: + with open(member, "rb") as file: + member_data = tomllib.load(file) + + def return_dependencies(toml) -> typing.List[str]: + acc = set() + stack = [toml] + while len(stack) > 0: + toml = stack.pop() + if "dependencies" in toml: + acc.update( + list( + map( + lambda dep: dep[1]["package"] + if "package" in dep[1] + else dep[0], + filter( + check_local_dep_fn, toml["dependencies"].items() + ), + ) + ) + ) + if "target" in toml: + stack.append(toml["target"]) + for key, value in toml.items(): + if key.startswith("cfg"): + stack.append(value) + return list(acc) + + dependencies = return_dependencies(member_data) + crates.append(Crate(dependencies, member_data, cargo_file_path=member)) + self.crates = crates + self.crate_index: typing.Dict[str, Crate] = {c.name: c for c in crates} -def is_crate_already_published(crate_name: str) -> bool: - found_string = get_latest_version_for_crate(crate_name) - if found_string is None: - return False + self.create_publish_order() - return target_version == found_string + self.starting_dir = os.getcwd() + def create_publish_order(self): + """Creates a valid publish order by topologically + sorting crates using the dependency graph.""" + topological_sorter = TopologicalSorter() -def publish_crate(crate: str): - starting_dir = os.getcwd() - os.chdir("lib/{}".format(location[crate])) + for crate in self.crates: + topological_sorter.add(crate.name, *crate.dependencies) - global no_dry_run - if no_dry_run: - output = subprocess.run(["cargo", "publish"]) - else: - print("In dry-run: not publishing crate `{}`".format(crate)) - output = subprocess.run(["cargo", "publish", "--dry-run"]) + self.publish_order: typing.List[str] = [*topological_sorter.static_order()] - os.chdir(starting_dir) + def is_crate_already_published(self, crate_name: str) -> bool: + """Checks if a given crate name is already published + with the version string we intend to publish with.""" + found_string: str = get_latest_version_for_crate(crate_name) + if found_string is None: + return False + + return self.version == found_string + + def publish_crate(self, crate_name: str): + # pylint: disable=broad-except + """Publish a given crate by name.""" + status = None + try: + crate = self.crate_index[crate_name] + os.chdir(crate.cargo_file_path) + if self.dry_run: + print(f"In dry-run: not publishing crate `{crate_name}`") + output = subprocess.run(["cargo", "publish", "--dry-run"], check=True) + if self.verbose: + print(output) + else: + output = subprocess.run(["cargo", "publish"], check=True) + if self.verbose: + print(output) + if self.verbose: + print("Success.") + except Exception as exc: + if self.verbose: + print(f"Failed to publish {crate_name}.") + print(exc) + status = exc + finally: + os.chdir(self.starting_dir) + return status + + def publish(self): + # pylint: disable=too-many-branches + """Publish all packages in workspace.""" + if self.verbose and self.dry_run: + print("(Dry run, not actually publishing anything)") + if self.verbose: + print("Publishing order:") + pprint(self.publish_order) + status = {} + failures = 0 + + for crate_name in self.publish_order: + print(f"Publishing `{crate_name}`...") + if not self.is_crate_already_published(crate_name): + status[crate_name] = self.publish_crate(crate_name) + if status[crate_name]: + failures = +1 + else: + print(f"`{crate_name}` was already published!") + continue + + # sleep for 16 seconds between crates to ensure the crates.io + # index has time to update + if not self.dry_run: + print( + "Sleeping for 16 seconds to allow the `crates.io` index to update..." + ) + time.sleep(16) + else: + print("In dry-run: not sleeping for crates.io to update.") + if failures > 0 and self.verbose: + print(f"encountered {failures} failures.") + for key, value in status.items(): + if value is None: + result = "ok" + else: + result = str(value) + print(f"{key}\t{result}") + if self.verbose: + print(f"Published {len(status) - failures} crates.") + return failures def main(): + """Main executable function.""" os.environ["WASMER_PUBLISH_SCRIPT_IS_RUNNING"] = "1" parser = argparse.ArgumentParser( description="Publish the Wasmer crates to crates.io" ) - parser.add_argument( - "--no-dry-run", + subparsers = parser.add_subparsers(dest="subcommand") + health_cmd = subparsers.add_parser( + "health-check", + help="""Check the dependency graph is a tree, meaning a non-cyclic + planar graph. Combine with verbosity to print a topological sorting + of the graph.""", + ) + health_cmd.add_argument( + "-v", + "--verbose", default=False, action="store_true", - help="Run the script without actually publishing anything to crates.io", + help="Be verbose.", + ) + health_cmd.add_argument( + "--print-dependencies", + default=False, + action="store_true", + help="For each crate, print its dependencies.", + ) + publish_cmd = subparsers.add_parser( + "publish", help="Publish Wasmer crates to crates.io." ) - args = vars(parser.parse_args()) - - global no_dry_run - no_dry_run = args["no_dry_run"] - - # get the order to publish the crates in - order = list(toposort_flatten(dep_graph, sort=True)) - - for crate in order: - print("Publishing `{}`...".format(crate)) - if not is_crate_already_published(crate): - publish_crate(crate) - else: - print("`{}` was already published!".format(crate)) - continue - # sleep for 16 seconds between crates to ensure the crates.io index has time to update - # this can be optimized with knowledge of our dep graph via toposort; we can even publish - # crates in parallel; however this is out of scope for the first version of this script - if no_dry_run: - print("Sleeping for 16 seconds to allow the `crates.io` index to update...") - time.sleep(16) - else: - print("In dry-run: not sleeping for crates.io to update.") + publish_cmd.add_argument( + "--version", + default=None, + type=str, + help="""Define the semver target triple (Default is automatically + read from workspace Cargo.toml.""", + ) + publish_cmd.add_argument( + "--dry-run", + default=False, + action="store_true", + help="""Run the script without actually publishing anything to + crates.io""", + ) + publish_cmd.add_argument( + "-v", + "--verbose", + default=False, + action="store_true", + help="Be verbose.", + ) + + args = parser.parse_args() + if args.subcommand == "health-check": + verbose = args.verbose + publisher = Publisher(verbose=verbose) + if verbose: + print(f"Version is {publisher.version}") + pprint(publisher.publish_order) + if args.print_dependencies: + for crate in publisher.crates: + print(f"{crate.name}: {crate.dependencies}") + return 0 + + if args.subcommand == "publish": + verbose = args.verbose + publisher = Publisher(dry_run=args.dry_run, verbose=verbose) + return publisher.publish() + return 0 if __name__ == "__main__":