diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml index 7dd75b5bfe4..761e0459543 100644 --- a/.github/workflows/archery.yml +++ b/.github/workflows/archery.yml @@ -53,8 +53,7 @@ jobs: with: python-version: '3.6' - name: Install Archery, Crossbow- and Test Dependencies - working-directory: dev/archery - run: pip install pytest responses toolz jinja2 -e .[all] + run: pip install pytest responses -e dev/archery[all] - name: Archery Unittests working-directory: dev/archery run: pytest -v archery @@ -62,4 +61,4 @@ jobs: run: archery docker - name: Crossbow Check Config working-directory: dev/tasks - run: python crossbow.py check-config + run: archery crossbow check-config diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py index 54a57f28585..35b69a68cca 100644 --- a/dev/archery/archery/bot.py +++ b/dev/archery/archery/bot.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -import operator +import os import shlex from pathlib import Path from functools import partial @@ -24,8 +24,9 @@ import click import github -from .utils.crossbow import Crossbow -from .utils.git import Git +from .utils.git import git +from .utils.logger import logger +from .crossbow import Repo, Queue, Config, Target, Job, CommentReport class EventError(Exception): @@ -81,86 +82,6 @@ def parse_args(self, ctx, args): group = partial(click.group, cls=Group) -class CrossbowCommentFormatter: - - _markdown_badge = '[![{title}]({badge})]({url})' - - badges = { - 'github': _markdown_badge.format( - title='Github Actions', - url='https://github.com/{repo}/actions?query=branch:{branch}', - badge=( - 'https://github.com/{repo}/workflows/Crossbow/' - 'badge.svg?branch={branch}' - ), - ), - 'azure': _markdown_badge.format( - title='Azure', - url=( - 'https://dev.azure.com/{repo}/_build/latest' - '?definitionId=1&branchName={branch}' - ), - badge=( - 'https://dev.azure.com/{repo}/_apis/build/status/' - '{repo_dotted}?branchName={branch}' - ) - ), - 'travis': _markdown_badge.format( - title='TravisCI', - url='https://travis-ci.com/{repo}/branches', - badge='https://img.shields.io/travis/{repo}/{branch}.svg' - ), - 'circle': _markdown_badge.format( - title='CircleCI', - url='https://circleci.com/gh/{repo}/tree/{branch}', - badge=( - 'https://img.shields.io/circleci/build/github' - '/{repo}/{branch}.svg' - ) - ), - 'appveyor': _markdown_badge.format( - title='Appveyor', - url='https://ci.appveyor.com/project/{repo}/history', - badge='https://img.shields.io/appveyor/ci/{repo}/{branch}.svg' - ), - 'drone': _markdown_badge.format( - title='Drone', - url='https://cloud.drone.io/{repo}', - badge='https://img.shields.io/drone/build/{repo}/{branch}.svg' - ), - } - - def __init__(self, crossbow_repo): - self.crossbow_repo = crossbow_repo - - def render(self, job): - url = 'https://github.com/{repo}/branches/all?query={branch}' - sha = job['target']['head'] - - msg = 'Revision: {}\n\n'.format(sha) - msg += 'Submitted crossbow builds: [{repo} @ {branch}]' - msg += '({})\n'.format(url) - msg += '\n|Task|Status|\n|----|------|' - - tasks = sorted(job['tasks'].items(), key=operator.itemgetter(0)) - for key, task in tasks: - branch = task['branch'] - - try: - template = self.badges[task['ci']] - badge = template.format( - repo=self.crossbow_repo, - repo_dotted=self.crossbow_repo.replace('/', '.'), - branch=branch - ) - except KeyError: - badge = 'unsupported CI service `{}`'.format(task['ci']) - - msg += '\n|{}|{}|'.format(key, badge) - - return msg.format(repo=self.crossbow_repo, branch=job['branch']) - - class CommentBot: def __init__(self, name, handler, token=None): @@ -195,8 +116,7 @@ def handle(self, event, payload): try: command = self.parse_command(payload) except EventError as e: - print(e) - # TODO(kszucs): log + logger.error(e) # see the possible reasons in the validate method return @@ -220,14 +140,13 @@ def handle_issue_comment(self, command, payload): comment = pull.get_issue_comment(payload['comment']['id']) try: - self.handler(command, issue=issue, pull=pull, comment=comment) + self.handler(command, issue=issue, pull_request=pull, + comment=comment) except CommandError as e: - # TODO(kszucs): log - print(e) + logger.error(e) pull.create_issue_comment("```\n{}\n```".format(e.message)) except Exception as e: - # TODO(kszucs): log - print(e) + logger.error(e) comment.create_reaction('-1') else: comment.create_reaction('+1') @@ -248,81 +167,94 @@ def actions(ctx): help='Crossbow repository on github to use') @click.pass_obj def crossbow(obj, crossbow): - """Trigger crossbow builds for this pull request""" + """ + Trigger crossbow builds for this pull request + """ obj['crossbow_repo'] = crossbow +def _clone_arrow_and_crossbow(dest, crossbow_repo, pull_request): + """ + Clone the repositories and initialize crossbow objects. + + Parameters + ---------- + dest : Path + Filesystem path to clone the repositories to. + crossbow_repo : str + Github repository name, like kszucs/crossbow. + pull_request : pygithub.PullRequest + Object containing information about the pull request the comment bot + was triggered from. + """ + arrow_path = dest / 'arrow' + queue_path = dest / 'crossbow' + + # clone arrow and checkout the pull request's branch + pull_request_ref = 'pull/{}/head:{}'.format( + pull_request.number, pull_request.head.ref + ) + git.clone(pull_request.base.repo.clone_url, str(arrow_path)) + git.fetch('origin', pull_request_ref, git_dir=arrow_path) + git.checkout(pull_request.head.ref, git_dir=arrow_path) + + # clone crossbow repository + crossbow_url = 'https://github.com/{}'.format(crossbow_repo) + git.clone(crossbow_url, str(queue_path)) + + # initialize crossbow objects + github_token = os.environ['CROSSBOW_GITHUB_TOKEN'] + arrow = Repo(arrow_path) + queue = Queue(queue_path, github_token=github_token, require_https=True) + + return (arrow, queue) + + @crossbow.command() @click.argument('tasks', nargs=-1, required=False) @click.option('--group', '-g', 'groups', multiple=True, help='Submit task groups as defined in tests.yml') @click.option('--param', '-p', 'params', multiple=True, help='Additional task parameters for rendering the CI templates') -@click.option('--dry-run/--push', default=False, - help='Just display the new changelog, don\'t write it') +@click.option('--arrow-version', '-v', default=None, + help='Set target version explicitly.') @click.pass_obj -def submit(obj, tasks, groups, params, dry_run): - """Submit crossbow testing tasks. +def submit(obj, tasks, groups, params, arrow_version): + """ + Submit crossbow testing tasks. See groups defined in arrow/dev/tasks/tests.yml """ - from ruamel.yaml import YAML - - git = Git() - - # construct crossbow arguments - args = [] - if dry_run: - args.append('--dry-run') - - for p in params: - args.extend(['-p', p]) - for g in groups: - args.extend(['-g', g]) - for t in tasks: - args.append(t) - - # pygithub pull request object - pr = obj['pull'] - crossbow_url = 'https://github.com/{}'.format(obj['crossbow_repo']) - + crossbow_repo = obj['crossbow_repo'] + pull_request = obj['pull_request'] with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) - arrow = tmpdir / 'arrow' - queue = tmpdir / 'crossbow' - - # clone arrow and checkout the pull request's branch - git.clone(pr.base.repo.clone_url, str(arrow)) - git.fetch('origin', 'pull/{}/head:{}'.format(pr.number, pr.head.ref), - git_dir=str(arrow)) - git.checkout(pr.head.ref, git_dir=str(arrow)) - - # clone crossbow - git.clone(crossbow_url, str(queue)) - - # submit the crossbow tasks - result = Path('result.yml').resolve() - xbow = Crossbow(str(arrow / 'dev' / 'tasks' / 'crossbow.py')) - xbow.run( - '--queue-path', str(queue), - '--output-file', str(result), - 'submit', - '--job-prefix', 'actions', - # don't rely on crossbow's remote and branch detection, because - # it doesn't work without a tracking upstream branch - '--arrow-remote', pr.head.repo.clone_url, - '--arrow-branch', pr.head.ref, - *args + arrow, queue = _clone_arrow_and_crossbow( + dest=Path(tmpdir), + crossbow_repo=crossbow_repo, + pull_request=pull_request, ) + # load available tasks configuration and groups from yaml + config = Config.load_yaml(arrow.path / "dev" / "tasks" / "tasks.yml") + config.validate() + + # initialize the crossbow build's target repository + target = Target.from_repo(arrow, version=arrow_version, + remote=pull_request.base.repo.clone_url) + + # parse additional job parameters + params = dict([p.split("=") for p in params]) + + # instantiate the job object + job = Job.from_config(config=config, target=target, tasks=tasks, + groups=groups, params=params) - # parse the result yml describing the submitted job - yaml = YAML() - with result.open() as fp: - job = yaml.load(fp) + # add the job to the crossbow queue and push to the remote repository + queue.put(job, prefix="actions") + queue.push() - # render the response comment's content - formatter = CrossbowCommentFormatter(obj['crossbow_repo']) - response = formatter.render(job) + # render the response comment's content + report = CommentReport(job, crossbow_repo=crossbow_repo) - # send the response - pr.create_issue_comment(response) + # send the response + pull_request.create_issue_comment(report.show()) diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py index 74e2373821c..bcaddf1c795 100644 --- a/dev/archery/archery/cli.py +++ b/dev/archery/archery/cli.py @@ -1069,5 +1069,23 @@ def release_cherry_pick(obj, version, dry_run, recreate): click.echo('git cherry-pick {}'.format(commit.hexsha)) +try: + from .crossbow.cli import crossbow # noqa +except ImportError as exc: + missing_package = exc.name + + @archery.command( + 'crossbow', + context_settings={"ignore_unknown_options": True} + ) + def crossbow(): + raise click.ClickException( + "Couldn't import crossbow because of missing dependency: {}" + .format(missing_package) + ) +else: + archery.add_command(crossbow) + + if __name__ == "__main__": archery(obj={}) diff --git a/dev/archery/archery/utils/crossbow.py b/dev/archery/archery/crossbow/__init__.py similarity index 80% rename from dev/archery/archery/utils/crossbow.py rename to dev/archery/archery/crossbow/__init__.py index f28b9762469..bc72e81f050 100644 --- a/dev/archery/archery/utils/crossbow.py +++ b/dev/archery/archery/crossbow/__init__.py @@ -15,9 +15,5 @@ # specific language governing permissions and limitations # under the License. -from .command import Command, default_bin - - -class Crossbow(Command): - def __init__(self, crossbow_bin=None): - self.bin = default_bin(crossbow_bin, "arrow/dev/tasks/crossbow.py") +from .core import Config, Repo, Queue, Target, Job # noqa +from .reports import CommentReport, ConsoleReport, EmailReport # noqa diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py new file mode 100644 index 00000000000..71c25e0460f --- /dev/null +++ b/dev/archery/archery/crossbow/cli.py @@ -0,0 +1,352 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pathlib import Path + +import click + +from .core import Config, Repo, Queue, Target, Job, CrossbowError +from .reports import EmailReport, ConsoleReport +from ..utils.source import ArrowSources + + +_default_arrow_path = ArrowSources.find().path +_default_queue_path = _default_arrow_path.parent / "crossbow" +_default_config_path = _default_arrow_path / "dev" / "tasks" / "tasks.yml" + + +@click.group() +@click.option('--github-token', '-t', default=None, + envvar="CROSSBOW_GITHUB_TOKEN", + help='OAuth token for GitHub authentication') +@click.option('--arrow-path', '-a', + type=click.Path(), default=_default_arrow_path, + help='Arrow\'s repository path. Defaults to the repository of ' + 'this script') +@click.option('--queue-path', '-q', + type=click.Path(), default=_default_queue_path, + help='The repository path used for scheduling the tasks. ' + 'Defaults to crossbow directory placed next to arrow') +@click.option('--queue-remote', '-qr', default=None, + help='Force to use this remote URL for the Queue repository') +@click.option('--output-file', metavar='', + type=click.File('w', encoding='utf8'), default='-', + help='Capture output result into file.') +@click.pass_context +def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote, + output_file): + """ + Schedule packaging tasks or nightly builds on CI services. + """ + ctx.ensure_object(dict) + ctx.obj['output'] = output_file + ctx.obj['arrow'] = Repo(arrow_path) + ctx.obj['queue'] = Queue(queue_path, remote_url=queue_remote, + github_token=github_token, require_https=True) + + +@crossbow.command() +@click.option('--config-path', '-c', + type=click.Path(exists=True), default=_default_config_path, + help='Task configuration yml. Defaults to tasks.yml') +@click.pass_obj +def check_config(obj, config_path): + # load available tasks configuration and groups from yaml + config = Config.load_yaml(config_path) + config.validate() + + output = obj['output'] + config.show(output) + + +@crossbow.command() +@click.argument('tasks', nargs=-1, required=False) +@click.option('--group', '-g', 'groups', multiple=True, + help='Submit task groups as defined in task.yml') +@click.option('--param', '-p', 'params', multiple=True, + help='Additional task parameters for rendering the CI templates') +@click.option('--job-prefix', default='build', + help='Arbitrary prefix for branch names, e.g. nightly') +@click.option('--config-path', '-c', + type=click.Path(exists=True), default=_default_config_path, + help='Task configuration yml. Defaults to tasks.yml') +@click.option('--arrow-version', '-v', default=None, + help='Set target version explicitly.') +@click.option('--arrow-remote', '-r', default=None, + help='Set GitHub remote explicitly, which is going to be cloned ' + 'on the CI services. Note, that no validation happens ' + 'locally. Examples: https://github.com/apache/arrow or ' + 'https://github.com/kszucs/arrow.') +@click.option('--arrow-branch', '-b', default=None, + help='Give the branch name explicitly, e.g. master, ARROW-1949.') +@click.option('--arrow-sha', '-t', default=None, + help='Set commit SHA or Tag name explicitly, e.g. f67a515, ' + 'apache-arrow-0.11.1.') +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.option('--dry-run/--commit', default=False, + help='Just display the rendered CI configurations without ' + 'committing them') +@click.option('--no-push/--push', default=False, + help='Don\'t push the changes') +@click.pass_obj +def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version, + arrow_remote, arrow_branch, arrow_sha, fetch, dry_run, no_push): + output = obj['output'] + queue, arrow = obj['queue'], obj['arrow'] + + # load available tasks configuration and groups from yaml + config = Config.load_yaml(config_path) + try: + config.validate() + except CrossbowError as e: + raise click.ClickException(str(e)) + + # Override the detected repo url / remote, branch and sha - this aims to + # make release procedure a bit simpler. + # Note, that the target resivion's crossbow templates must be + # compatible with the locally checked out version of crossbow (which is + # in case of the release procedure), because the templates still + # contain some business logic (dependency installation, deployments) + # which will be reduced to a single command in the future. + target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch, + head=arrow_sha, version=arrow_version) + + # parse additional job parameters + params = dict([p.split("=") for p in params]) + + # instantiate the job object + try: + job = Job.from_config(config=config, target=target, tasks=tasks, + groups=groups, params=params) + except CrossbowError as e: + raise click.ClickException(str(e)) + + job.show(output) + if dry_run: + return + + if fetch: + queue.fetch() + queue.put(job, prefix=job_prefix) + + if no_push: + click.echo('Branches and commits created but not pushed: `{}`' + .format(job.branch)) + else: + queue.push() + click.echo('Pushed job identifier is: `{}`'.format(job.branch)) + + +@crossbow.command() +@click.argument('task', required=True) +@click.option('--config-path', '-c', + type=click.Path(exists=True), default=_default_config_path, + help='Task configuration yml. Defaults to tasks.yml') +@click.option('--arrow-version', '-v', default=None, + help='Set target version explicitly.') +@click.option('--arrow-remote', '-r', default=None, + help='Set GitHub remote explicitly, which is going to be cloned ' + 'on the CI services. Note, that no validation happens ' + 'locally. Examples: https://github.com/apache/arrow or ' + 'https://github.com/kszucs/arrow.') +@click.option('--arrow-branch', '-b', default=None, + help='Give the branch name explicitly, e.g. master, ARROW-1949.') +@click.option('--arrow-sha', '-t', default=None, + help='Set commit SHA or Tag name explicitly, e.g. f67a515, ' + 'apache-arrow-0.11.1.') +@click.option('--param', '-p', 'params', multiple=True, + help='Additional task parameters for rendering the CI templates') +@click.pass_obj +def render(obj, task, config_path, arrow_version, arrow_remote, arrow_branch, + arrow_sha, params): + """ + Utility command to check the rendered CI templates. + """ + from .core import _flatten + + def highlight(code): + try: + from pygments import highlight + from pygments.lexers import YamlLexer + from pygments.formatters import TerminalFormatter + return highlight(code, YamlLexer(), TerminalFormatter()) + except ImportError: + return code + + arrow = obj['arrow'] + + target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch, + head=arrow_sha, version=arrow_version) + config = Config.load_yaml(config_path) + params = dict([p.split("=") for p in params]) + job = Job.from_config(config=config, target=target, tasks=[task], + params=params) + + for task_name, rendered_files in job.render_tasks().items(): + for path, content in _flatten(rendered_files).items(): + click.echo('#' * 80) + click.echo('### {:^72} ###'.format("/".join(path))) + click.echo('#' * 80) + click.echo(highlight(content)) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.pass_obj +def status(obj, job_name, fetch): + output = obj['output'] + queue = obj['queue'] + if fetch: + queue.fetch() + job = queue.get(job_name) + ConsoleReport(job).show(output) + + +@crossbow.command() +@click.argument('prefix', required=True) +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.pass_obj +def latest_prefix(obj, prefix, fetch): + queue = obj['queue'] + if fetch: + queue.fetch() + latest = queue.latest_for_prefix(prefix) + click.echo(latest.branch) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--sender-name', '-n', + help='Name to use for report e-mail.') +@click.option('--sender-email', '-e', + help='E-mail to use for report e-mail.') +@click.option('--recipient-email', '-r', + help='Where to send the e-mail report') +@click.option('--smtp-user', '-u', + help='E-mail address to use for SMTP login') +@click.option('--smtp-password', '-P', + help='SMTP password to use for report e-mail.') +@click.option('--smtp-server', '-s', default='smtp.gmail.com', + help='SMTP server to use for report e-mail.') +@click.option('--smtp-port', '-p', default=465, + help='SMTP port to use for report e-mail.') +@click.option('--poll/--no-poll', default=False, + help='Wait for completion if there are tasks pending') +@click.option('--poll-max-minutes', default=180, + help='Maximum amount of time waiting for job completion') +@click.option('--poll-interval-minutes', default=10, + help='Number of minutes to wait to check job status again') +@click.option('--send/--dry-run', default=False, + help='Just display the report, don\'t send it') +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.pass_obj +def report(obj, job_name, sender_name, sender_email, recipient_email, + smtp_user, smtp_password, smtp_server, smtp_port, poll, + poll_max_minutes, poll_interval_minutes, send, fetch): + """ + Send an e-mail report showing success/failure of tasks in a Crossbow run + """ + output = obj['output'] + queue = obj['queue'] + if fetch: + queue.fetch() + + job = queue.get(job_name) + report = EmailReport( + job=job, + sender_name=sender_name, + sender_email=sender_email, + recipient_email=recipient_email + ) + + if poll: + job.wait_until_finished( + poll_max_minutes=poll_max_minutes, + poll_interval_minutes=poll_interval_minutes + ) + + if send: + report.send( + smtp_user=smtp_user, + smtp_password=smtp_password, + smtp_server=smtp_server, + smtp_port=smtp_port + ) + else: + report.show(output) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('-t', '--target-dir', + default=_default_arrow_path / 'packages', + type=click.Path(file_okay=False, dir_okay=True), + help='Directory to download the build artifacts') +@click.option('--dry-run/--execute', default=False, + help='Just display process, don\'t download anything') +@click.option('--fetch/--no-fetch', default=True, + help='Fetch references (branches and tags) from the remote') +@click.pass_obj +def download_artifacts(obj, job_name, target_dir, dry_run, fetch): + """Download build artifacts from GitHub releases""" + output = obj['output'] + + # fetch the queue repository + queue = obj['queue'] + if fetch: + queue.fetch() + + # query the job's artifacts + job = queue.get(job_name) + + # create directory to download the assets to + target_dir = Path(target_dir).absolute() / job_name + target_dir.mkdir(parents=True, exist_ok=True) + + # download the assets while showing the job status + def asset_callback(task_name, task, asset): + if asset is not None: + path = target_dir / task_name / asset.name + path.parent.mkdir(exist_ok=True) + if not dry_run: + asset.download(path) + + click.echo('Downloading {}\'s artifacts.'.format(job_name)) + click.echo('Destination directory is {}'.format(target_dir)) + click.echo() + + report = ConsoleReport(job) + report.show(output, asset_callback=asset_callback) + + +@crossbow.command() +@click.option('--sha', required=True, help='Target committish') +@click.option('--tag', required=True, help='Target tag') +@click.option('--method', default='curl', help='Use cURL to upload') +@click.option('--pattern', '-p', 'patterns', required=True, multiple=True, + help='File pattern to upload as assets') +@click.pass_obj +def upload_artifacts(obj, tag, sha, patterns, method): + queue = obj['queue'] + queue.github_overwrite_release_assets( + tag_name=tag, target_commitish=sha, method=method, patterns=patterns + ) diff --git a/dev/tasks/crossbow.py b/dev/archery/archery/crossbow/core.py old mode 100755 new mode 100644 similarity index 57% rename from dev/tasks/crossbow.py rename to dev/archery/archery/crossbow/core.py index 2874472bdb2..d4d3d5183b5 --- a/dev/tasks/crossbow.py +++ b/dev/archery/archery/crossbow/core.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -28,13 +26,9 @@ import textwrap from io import StringIO from pathlib import Path -from textwrap import dedent from datetime import date -from functools import partial - -import click -import toolz +import jinja2 from ruamel.yaml import YAML try: @@ -51,22 +45,37 @@ else: PygitRemoteCallbacks = pygit2.RemoteCallbacks +from ..utils.source import ArrowSources + + +for pkg in ["requests", "urllib3", "github3"]: + logging.getLogger(pkg).setLevel(logging.WARNING) -# initialize logging -logging.basicConfig() -logging.getLogger().setLevel(logging.ERROR) +logger = logging.getLogger("crossbow") -# enable verbose logging for requests -# http_client.HTTPConnection.debuglevel = 1 -requests_log = logging.getLogger("requests.packages.urllib3") -requests_log.setLevel(logging.ERROR) -requests_log.propagate = True +class CrossbowError(Exception): + pass -CWD = Path(__file__).parent.absolute() +def _flatten(mapping): + """Converts a hierarchical mapping to a flat dictionary""" + result = {} + for k, v in mapping.items(): + if isinstance(v, dict): + for ik, iv in _flatten(v).items(): + ik = ik if isinstance(ik, tuple) else (ik,) + result[(k,) + ik] = iv + elif isinstance(v, list): + for ik, iv in enumerate(_flatten(v)): + ik = ik if isinstance(ik, tuple) else (ik,) + result[(k,) + ik] = iv + else: + result[(k,)] = v + return result -def unflatten(mapping): + +def _unflatten(mapping): """Converts a flat tuple => object mapping to hierarchical one""" result = {} for path, value in mapping.items(): @@ -82,7 +91,7 @@ def unflatten(mapping): return result -def unflatten_tree(files): +def _unflatten_tree(files): """Converts a flat path => object mapping to a hierarchical directories Input: @@ -102,8 +111,20 @@ def unflatten_tree(files): } } """ - files = toolz.keymap(lambda path: tuple(path.split('/')), files) - return unflatten(files) + files = {tuple(k.split('/')): v for k, v in files.items()} + return _unflatten(files) + + +def _render_jinja_template(searchpath, template, params): + def format_all(items, pattern): + return [pattern.format(item) for item in items] + + loader = jinja2.FileSystemLoader(searchpath) + env = jinja2.Environment(loader=loader, trim_blocks=True, + lstrip_blocks=True) + env.filters['format_all'] = format_all + template = env.get_template(template) + return template.render(**params) # configurations for setting up branch skipping @@ -170,7 +191,7 @@ def credentials(self, url, username_from_url, allowed_types): # pygit2 doesn't propagate the exception properly msg = 'Wrong oauth personal access token' print(msg) - raise ValueError(msg) + raise CrossbowError(msg) if allowed_types & pygit2.credentials.GIT_CREDTYPE_USERPASS_PLAINTEXT: return pygit2.UserPass(self.token, 'x-oauth-basic') @@ -183,7 +204,8 @@ def _git_ssh_to_https(url): class Repo: - """Base class for interaction with local git repositories + """ + Base class for interaction with local git repositories A high level wrapper used for both reading revision information from arrow's repository and pushing continuous integration tasks to the queue @@ -194,6 +216,7 @@ class Repo: require_https : boolean, default False Raise exception for SSH origin URLs """ + def __init__(self, path, github_token=None, remote_url=None, require_https=False): self.path = Path(path) @@ -205,7 +228,7 @@ def __init__(self, path, github_token=None, remote_url=None, self._updated_refs = [] def __str__(self): - tpl = dedent(''' + tpl = textwrap.dedent(''' Repo: {remote}@{branch} Commit: {head} ''') @@ -225,8 +248,8 @@ def repo(self): def origin(self): remote = self.repo.remotes['origin'] if self.require_https and remote.url.startswith('git@github.com'): - raise ValueError("Change SSH origin URL to HTTPS to use " - "Crossbow: {}".format(remote.url)) + raise CrossbowError("Change SSH origin URL to HTTPS to use " + "Crossbow: {}".format(remote.url)) return remote def fetch(self): @@ -236,7 +259,7 @@ def fetch(self): def push(self, refs=None, github_token=None): github_token = github_token or self.github_token if github_token is None: - raise click.ClickException( + raise RuntimeError( 'Could not determine GitHub token. Please set the ' 'CROSSBOW_GITHUB_TOKEN environment variable to a ' 'valid GitHub access token or pass one to --github-token.' @@ -366,9 +389,10 @@ def file_contents(self, commit_id, file): def _parse_github_user_repo(self): m = re.match(r'.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote_url) if m is None: - raise ValueError("Unable to parse the github owner and repository " - "from the repository's remote url '{}'" - .format(self.remote_url)) + raise CrossbowError( + "Unable to parse the github owner and repository from the " + "repository's remote url '{}'".format(self.remote_url) + ) user, repo = m.group(1), m.group(2) return user, repo @@ -412,12 +436,12 @@ def github_upload_asset_requests(self, release, path, name, mime, result = release.upload_asset(name=name, asset=fp, content_type=mime) except github3.exceptions.ResponseError as e: - click.echo('Attempt {} has failed with message: {}.' - .format(i + 1, str(e))) - click.echo('Error message {}'.format(e.msg)) - click.echo('List of errors provided by Github:') + logger.error('Attempt {} has failed with message: {}.' + .format(i + 1, str(e))) + logger.error('Error message {}'.format(e.msg)) + logger.error('List of errors provided by Github:') for err in e.errors: - click.echo(' - {}'.format(err)) + logger.error(' - {}'.format(err)) if e.code == 422: # 422 Validation Failed, probably raised because @@ -425,16 +449,16 @@ def github_upload_asset_requests(self, release, path, name, mime, # reattempting the asset upload for asset in release.assets(): if asset.name == name: - click.echo('Release asset {} already exists, ' - 'removing it...'.format(name)) + logger.info('Release asset {} already exists, ' + 'removing it...'.format(name)) asset.delete() - click.echo('Asset {} removed.'.format(name)) + logger.info('Asset {} removed.'.format(name)) break except github3.exceptions.ConnectionError as e: - click.echo('Attempt {} has failed with message: {}.' - .format(i + 1, str(e))) + logger.error('Attempt {} has failed with message: {}.' + .format(i + 1, str(e))) else: - click.echo('Attempt {} has finished.'.format(i + 1)) + logger.info('Attempt {} has finished.'.format(i + 1)) return result time.sleep(retry_backoff) @@ -463,9 +487,9 @@ def github_overwrite_release_assets(self, tag_name, target_commitish, # sigmavirus24/github3.py/issues/779#issuecomment-379470626 repo = self.as_github_repo() if not tag_name: - raise ValueError('Empty tag name') + raise CrossbowError('Empty tag name') if not target_commitish: - raise ValueError('Empty target commit for the release tag') + raise CrossbowError('Empty target commit for the release tag') # remove the whole release if it already exists try: @@ -482,7 +506,7 @@ def github_overwrite_release_assets(self, tag_name, target_commitish, size = os.path.getsize(path) mime = mimetypes.guess_type(name)[0] or 'application/zip' - click.echo( + logger.info( 'Uploading asset `{}` with mimetype {} and size {}...' .format(name, mime, size) ) @@ -494,7 +518,7 @@ def github_overwrite_release_assets(self, tag_name, target_commitish, self.github_upload_asset_curl(release, path, name=name, mime=mime) else: - raise ValueError( + raise CrossbowError( 'Unsupported upload method {}'.format(method) ) @@ -550,7 +574,9 @@ def get(self, job_name): try: content = self.file_contents(branch.target, 'job.yml') except KeyError: - raise ValueError('No job is found with name: {}'.format(job_name)) + raise CrossbowError( + 'No job is found with name: {}'.format(job_name) + ) buffer = StringIO(content.decode('utf-8')) job = yaml.load(buffer) @@ -559,19 +585,19 @@ def get(self, job_name): def put(self, job, prefix='build'): if not isinstance(job, Job): - raise ValueError('`job` must be an instance of Job') + raise CrossbowError('`job` must be an instance of Job') if job.branch is not None: - raise ValueError('`job.branch` is automatically generated, thus ' - 'it must be blank') + raise CrossbowError('`job.branch` is automatically generated, ' + 'thus it must be blank') if job.target.remote is None: - raise RuntimeError( + raise CrossbowError( 'Cannot determine git remote for the Arrow repository to ' 'clone or push to, try to push the `{}` branch first to have ' 'a remote tracking counterpart.'.format(job.target.branch) ) if job.target.branch is None: - raise RuntimeError( + raise CrossbowError( 'Cannot determine the current branch of the Arrow repository ' 'to clone or push to, perhaps it is in detached HEAD state. ' 'Please checkout a branch.' @@ -586,9 +612,12 @@ def put(self, job, prefix='build'): # adding CI's name to the end of the branch in order to use skip # patterns on travis and circleci task.branch = '{}-{}-{}'.format(job.branch, task.ci, task_name) - files = task.render_files(**job.params, - arrow=job.target, - queue_remote_url=self.remote_url) + params = { + **job.params, + "arrow": job.target, + "queue_remote_url": self.remote_url + } + files = task.render_files(job.template_searchpath, params=params) branch = self.create_branch(task.branch, files=files) self.create_tag(task.tag, branch.target) task.commit = str(branch.target) @@ -632,7 +661,8 @@ def to_yaml(cls, representer, data): class Target(Serializable): - """Describes target repository and revision the builds run against + """ + Describes target repository and revision the builds run against This serializable data container holding information about arrow's git remote, branch, sha and version number as well as some metadata @@ -685,7 +715,8 @@ def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, class Task(Serializable): - """Describes a build task and metadata required to render CI templates + """ + Describes a build task and metadata required to render CI templates A task is represented as a single git commit and branch containing jinja2 rendered files (currently appveyor.yml or .travis.yml configurations). @@ -714,24 +745,20 @@ def __init__(self, ci, template, artifacts=None, params=None): self._status = None # status cache self._assets = None # assets cache - def render_files(self, **params): - from jinja2 import Template, StrictUndefined - from jinja2.exceptions import TemplateError - - path = CWD / self.template - params = toolz.merge(self.params, params) - template = Template(path.read_text(), undefined=StrictUndefined) + def render_files(self, searchpath, params=None): + params = {**self.params, **(params or {}), "task": self} try: - rendered = template.render(task=self, **params) - except TemplateError as e: + rendered = _render_jinja_template(searchpath, self.template, + params=params) + except jinja2.TemplateError as e: raise RuntimeError( 'Failed to render template `{}` with {}: {}'.format( - path, e.__class__.__name__, str(e) + self.template, e.__class__.__name__, str(e) ) ) - tree = toolz.merge(_default_tree, {self.filename: rendered}) - return unflatten_tree(tree) + tree = {**_default_tree, self.filename: rendered} + return _unflatten_tree(tree) @property def tag(self): @@ -766,7 +793,8 @@ def assets(self, force_query=False): class TaskStatus: - """Combine the results from status and checks API to a single state. + """ + Combine the results from status and checks API to a single state. Azure pipelines uses checks API which doesn't provide a combined interface like status API does, so we need to manually combine @@ -818,19 +846,25 @@ def __init__(self, commit): states.append('pending') # it could be more effective, but the following is more descriptive - if any(state in {'error', 'failure'} for state in states): - combined_state = 'failure' - elif any(state == 'pending' for state in states): - combined_state = 'pending' - elif all(state == 'success' for state in states): - combined_state = 'success' - else: - combined_state = 'error' + combined_state = 'error' + if len(states): + if any(state in {'error', 'failure'} for state in states): + combined_state = 'failure' + elif any(state == 'pending' for state in states): + combined_state = 'pending' + elif all(state == 'success' for state in states): + combined_state = 'success' + + # show link to the actual build, some of the CI providers implement + # the statuses API others implement the checks API, so display both + build_links = [s.target_url for s in status.statuses] + build_links += [c.html_url for c in check_runs] self.combined_state = combined_state self.github_status = status self.github_check_runs = check_runs self.total_count = len(states) + self.build_links = build_links class TaskAssets(dict): @@ -860,7 +894,7 @@ def __init__(self, github_release, artifact_patterns): elif num_matches == 1: self[pattern] = github_assets[matches[0].group(0)] else: - raise ValueError( + raise CrossbowError( 'Only a single asset should match pattern `{}`, there are ' 'multiple ones: {}'.format(pattern, ', '.join(matches)) ) @@ -875,7 +909,7 @@ def uploaded_assets(self): class Job(Serializable): """Describes multiple tasks against a single target repository""" - def __init__(self, target, tasks, params=None): + def __init__(self, target, tasks, params=None, template_searchpath=None): if not tasks: raise ValueError('no tasks were provided for the job') if not all(isinstance(task, Task) for task in tasks.values()): @@ -892,13 +926,33 @@ def __init__(self, target, tasks, params=None): self.params = params or {} # additional parameters for the tasks self.branch = None # filled after adding to a queue self._queue = None # set by the queue object after put or get + if template_searchpath is None: + self._template_searchpath = ArrowSources.find().path + else: + self._template_searchpath = template_searchpath def render_files(self): with StringIO() as buf: yaml.dump(self, buf) content = buf.getvalue() - tree = toolz.merge(_default_tree, {'job.yml': content}) - return unflatten_tree(tree) + tree = {**_default_tree, "job.yml": content} + return _unflatten_tree(tree) + + def render_tasks(self, params=None): + result = {} + params = { + **self.params, + "arrow": self.target, + **(params or {}) + } + for task_name, task in self.tasks.items(): + files = task.render_files(self._template_searchpath, params) + result[task_name] = files + return result + + @property + def template_searchpath(self): + return self._template_searchpath @property def queue(self): @@ -920,6 +974,9 @@ def email(self): def date(self): return self.queue.date_of(self) + def show(self, stream=None): + return yaml.dump(self, stream=stream) + @classmethod def from_config(cls, config, target, tasks=None, groups=None, params=None): """ @@ -945,7 +1002,7 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): Raises ------ - click.ClickException + Exception: If invalid groups or tasks has been passed. """ task_definitions = config.select(tasks, groups=groups) @@ -960,7 +1017,8 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): artifacts = [fn.format(**versions) for fn in artifacts] tasks[task_name] = Task(artifacts=artifacts, **task) - return cls(target=target, tasks=tasks, params=params) + return cls(target=target, tasks=tasks, params=params, + template_searchpath=config.template_searchpath) def is_finished(self): for task in self.tasks.values(): @@ -982,18 +1040,28 @@ def wait_until_finished(self, poll_max_minutes=120, 'to finish, waited for {} minutes.') raise RuntimeError(msg.format(waited_for_minutes)) - # TODO(kszucs): use logging - click.echo('Waiting {} minutes and then checking again' - .format(poll_interval_minutes)) + logger.info('Waiting {} minutes and then checking again' + .format(poll_interval_minutes)) time.sleep(poll_interval_minutes * 60) class Config(dict): + def __init__(self, tasks, template_searchpath): + super().__init__(tasks) + self.template_searchpath = template_searchpath + @classmethod def load_yaml(cls, path): - with Path(path).open() as fp: - return cls(yaml.load(fp)) + path = Path(path) + searchpath = path.parent + rendered = _render_jinja_template(searchpath, template=path.name, + params={}) + config = yaml.load(rendered) + return cls(config, template_searchpath=searchpath) + + def show(self, stream=None): + return yaml.dump(dict(self), stream=stream) def select(self, tasks=None, groups=None): config_groups = dict(self['groups']) @@ -1010,18 +1078,22 @@ def select(self, tasks=None, groups=None): msg = 'Invalid group(s) {!r}. Must be one of {!r}'.format( invalid_groups, valid_groups ) - raise ValueError(msg) + raise CrossbowError(msg) # merge the tasks defined in the selected groups task_patterns = [list(config_groups[name]) for name in group_whitelist] task_patterns = set(sum(task_patterns, task_whitelist)) # treat the task names as glob patterns to select tasks more easily - requested_tasks = set( - toolz.concat( - fnmatch.filter(valid_tasks, p) for p in task_patterns - ) - ) + requested_tasks = set() + for pattern in task_patterns: + matches = fnmatch.filter(valid_tasks, pattern) + if len(matches): + requested_tasks.update(matches) + else: + raise CrossbowError( + "Unable to match any tasks for `{}`".format(pattern) + ) # validate that the passed and matched tasks are defined in the config invalid_tasks = requested_tasks - valid_tasks @@ -1029,7 +1101,7 @@ def select(self, tasks=None, groups=None): msg = 'Invalid task(s) {!r}. Must be one of {!r}'.format( invalid_tasks, valid_tasks ) - raise ValueError(msg) + raise CrossbowError(msg) return { task_name: config_tasks[task_name] for task_name in requested_tasks @@ -1041,7 +1113,7 @@ def validate(self): for pattern in group: tasks = self.select(tasks=[pattern]) if not tasks: - raise ValueError( + raise CrossbowError( "The pattern `{}` defined for task group `{}` is not " "matching any of the tasks defined in the " "configuration file.".format(pattern, group_name) @@ -1052,7 +1124,7 @@ def validate(self): try: Task(**task) except Exception as e: - raise ValueError( + raise CrossbowError( 'Unable to construct a task object from the ' 'definition of task `{}`. The original error message ' 'is: `{}`'.format(task_name, str(e)) @@ -1071,308 +1143,15 @@ def validate(self): for task_name, task in self['tasks'].items(): task = Task(**task) files = task.render_files( - arrow=target, - queue_remote_url='https://github.com/org/crossbow' + self.template_searchpath, + params=dict( + arrow=target, + queue_remote_url='https://github.com/org/crossbow' + ) ) if not files: - raise ValueError('No files have been rendered for task `{}`' - .format(task_name)) - - -class Report: - - def __init__(self, job): - self.job = job - - def show(self): - raise NotImplementedError() - - -class ConsoleReport(Report): - """Report the status of a Job to the console using click""" - - # output table's header template - HEADER = '[{state:>7}] {branch:<52} {content:>16}' - DETAILS = ' └ {url}' - - # output table's row template for assets - ARTIFACT_NAME = '{artifact:>69} ' - ARTIFACT_STATE = '[{state:>7}]' - - # state color mapping to highlight console output - COLORS = { - # from CombinedStatus - 'error': 'red', - 'failure': 'red', - 'pending': 'yellow', - 'success': 'green', - # custom state messages - 'ok': 'green', - 'missing': 'red' - } - - def lead(self, state, branch, n_uploaded, n_expected): - line = self.HEADER.format( - state=state.upper(), - branch=branch, - content='uploaded {} / {}'.format(n_uploaded, n_expected) - ) - return click.style(line, fg=self.COLORS[state.lower()]) - - def header(self): - header = self.HEADER.format( - state='state', - branch='Task / Branch', - content='Artifacts' - ) - delimiter = '-' * len(header) - return '{}\n{}'.format(header, delimiter) - - def artifact(self, state, pattern, asset): - if asset is None: - artifact = pattern - state = 'pending' if state == 'pending' else 'missing' - else: - artifact = asset.name - state = 'ok' - - name_ = self.ARTIFACT_NAME.format(artifact=artifact) - state_ = click.style( - self.ARTIFACT_STATE.format(state=state.upper()), - self.COLORS[state] - ) - return name_ + state_ - - def show(self, outstream, asset_callback=None): - echo = partial(click.echo, file=outstream) - - # write table's header - echo(self.header()) - - # write table's body - for task_name, task in sorted(self.job.tasks.items()): - # write summary of the uploaded vs total assets - status = task.status() - assets = task.assets() - - # mapping of artifact pattern to asset or None of not uploaded - n_expected = len(task.artifacts) - n_uploaded = len(assets.uploaded_assets()) - echo(self.lead(status.combined_state, task_name, n_uploaded, - n_expected)) - - # show link to the actual build, some of the CI providers implement - # the statuses API others implement the checks API, so display both - for s in status.github_status.statuses: - echo(self.DETAILS.format(url=s.target_url)) - for c in status.github_check_runs: - echo(self.DETAILS.format(url=c.html_url)) - - # write per asset status - for artifact_pattern, asset in assets.items(): - if asset_callback is not None: - asset_callback(task_name, task, asset) - echo(self.artifact(status.combined_state, artifact_pattern, - asset)) - - -class EmailReport(Report): - - HEADER = textwrap.dedent(""" - Arrow Build Report for Job {job_name} - - All tasks: {all_tasks_url} - """) - - TASK = textwrap.dedent(""" - - {name}: - URL: {url} - """).strip() - - EMAIL = textwrap.dedent(""" - From: {sender_name} <{sender_email}> - To: {recipient_email} - Subject: {subject} - - {body} - """).strip() - - STATUS_HEADERS = { - # from CombinedStatus - 'error': 'Errored Tasks:', - 'failure': 'Failed Tasks:', - 'pending': 'Pending Tasks:', - 'success': 'Succeeded Tasks:', - } - - def __init__(self, job, sender_name, sender_email, recipient_email): - self.sender_name = sender_name - self.sender_email = sender_email - self.recipient_email = recipient_email - super().__init__(job) - - def url(self, query): - repo_url = self.job.queue.remote_url.strip('.git') - return '{}/branches/all?query={}'.format(repo_url, query) - - def listing(self, tasks): - return '\n'.join( - sorted( - self.TASK.format(name=task_name, url=self.url(task.branch)) - for task_name, task in tasks.items() - ) - ) - - def header(self): - url = self.url(self.job.branch) - return self.HEADER.format(job_name=self.job.branch, all_tasks_url=url) - - def subject(self): - return ( - "[NIGHTLY] Arrow Build Report for Job {}".format(self.job.branch) - ) - - def body(self): - buffer = StringIO() - buffer.write(self.header()) - - tasks_by_state = toolz.groupby( - lambda name_task_pair: name_task_pair[1].status().combined_state, - self.job.tasks.items() - ) - - for state in ('failure', 'error', 'pending', 'success'): - if state in tasks_by_state: - tasks = dict(tasks_by_state[state]) - buffer.write('\n') - buffer.write(self.STATUS_HEADERS[state]) - buffer.write('\n') - buffer.write(self.listing(tasks)) - buffer.write('\n') - - return buffer.getvalue() - - def email(self): - return self.EMAIL.format( - sender_name=self.sender_name, - sender_email=self.sender_email, - recipient_email=self.recipient_email, - subject=self.subject(), - body=self.body() - ) - - def show(self, outstream): - outstream.write(self.email()) - - def send(self, smtp_user, smtp_password, smtp_server, smtp_port): - import smtplib - - email = self.email() - - server = smtplib.SMTP_SSL(smtp_server, smtp_port) - server.ehlo() - server.login(smtp_user, smtp_password) - server.sendmail(smtp_user, self.recipient_email, email) - server.close() - - -class GithubPage: - - def __init__(self, jobs): - self.jobs = list(jobs) - - def _generate_page(self, links): - links = ['
  • {}
    • '.format(url, name) - for name, url in sorted(links.items())] - return ''.format(''.join(links)) - - def _generate_toc(self, files): - result, links = {}, {} - for k, v in files.items(): - if isinstance(v, dict): - result[k] = self._generate_toc(v) - links[k] = '{}/'.format(k) - else: - result[k] = v - - if links: - result['index.html'] = self._generate_page(links) - - return result - - def _is_failed(self, status, task_name): - # for showing task statuses during the rendering procedure - if status.combined_state == 'success': - msg = click.style('[ OK] {}'.format(task_name), fg='green') - failed = False - else: - msg = click.style('[FAIL] {}'.format(task_name), fg='yellow') - failed = True - - click.echo(msg) - return failed - - def render_nightlies(self): - click.echo('\n\nRENDERING NIGHTLIES') - nightly_files = {} - - for job in self.jobs: - click.echo('\nJOB: {}'.format(job.branch)) - job_files = {} - - for task_name, task in sorted(job.tasks.items()): - # TODO: also render check runs? - status = task.status() - - task_files = {'status.json': status.github_status.as_json()} - links = {'status.json': 'status.json'} - - if not self._is_failed(status, task_name): - # accumulate links to uploaded assets - for asset in task.assets().uploaded_assets(): - links[asset.name] = asset.browser_download_url - - if links: - page_content = self._generate_page(links) - task_files['index.html'] = page_content - - job_files[task_name] = task_files - - nightly_files[str(job.date)] = job_files - - # write the most recent wheels under the latest directory - if 'latest' not in nightly_files: - nightly_files['latest'] = job_files - - return nightly_files - - def render_pypi_simple(self): - click.echo('\n\nRENDERING PYPI') - - wheels = {} - for job in self.jobs: - click.echo('\nJOB: {}'.format(job.branch)) - - for task_name, task in sorted(job.tasks.items()): - if not task_name.startswith('wheel'): - continue - status = task.status() - if self._is_failed(status, task_name): - continue - for asset in task.assets().uploaded_assets(): - wheels[asset.name] = asset.browser_download_url - - return {'pyarrow': {'index.html': self._generate_page(wheels)}} - - def render(self): - # directory structure for the github pages, only wheels are supported - # at the moment - files = self._generate_toc({ - 'nightly': self.render_nightlies(), - 'pypi': self.render_pypi_simple(), - }) - files['.nojekyll'] = '' - return files + raise CrossbowError('No files have been rendered for task `{}`' + .format(task_name)) # configure yaml serializer @@ -1380,302 +1159,3 @@ def render(self): yaml.register_class(Job) yaml.register_class(Task) yaml.register_class(Target) - - -# define default paths -DEFAULT_CONFIG_PATH = str(CWD / 'tasks.yml') -DEFAULT_ARROW_PATH = CWD.parents[1] -DEFAULT_QUEUE_PATH = CWD.parents[2] / 'crossbow' - - -@click.group() -@click.option('--github-token', '-t', default=None, - help='OAuth token for GitHub authentication') -@click.option('--arrow-path', '-a', - type=click.Path(), default=str(DEFAULT_ARROW_PATH), - help='Arrow\'s repository path. Defaults to the repository of ' - 'this script') -@click.option('--queue-path', '-q', - type=click.Path(), default=str(DEFAULT_QUEUE_PATH), - help='The repository path used for scheduling the tasks. ' - 'Defaults to crossbow directory placed next to arrow') -@click.option('--queue-remote', '-qr', default=None, - help='Force to use this remote URL for the Queue repository') -@click.option('--output-file', metavar='', - type=click.File('w', encoding='utf8'), default='-', - help='Capture output result into file.') -@click.pass_context -def crossbow(ctx, github_token, arrow_path, queue_path, queue_remote, - output_file): - ctx.ensure_object(dict) - ctx.obj['output'] = output_file - ctx.obj['arrow'] = Repo(arrow_path) - ctx.obj['queue'] = Queue(queue_path, remote_url=queue_remote, - github_token=github_token, require_https=True) - - -@crossbow.command() -@click.option('--config-path', '-c', - type=click.Path(exists=True), default=DEFAULT_CONFIG_PATH, - help='Task configuration yml. Defaults to tasks.yml') -def check_config(config_path): - # load available tasks configuration and groups from yaml - config = Config.load_yaml(config_path) - config.validate() - - -@crossbow.command() -@click.argument('tasks', nargs=-1, required=False) -@click.option('--group', '-g', 'groups', multiple=True, - help='Submit task groups as defined in task.yml') -@click.option('--param', '-p', 'params', multiple=True, - help='Additional task parameters for rendering the CI templates') -@click.option('--job-prefix', default='build', - help='Arbitrary prefix for branch names, e.g. nightly') -@click.option('--config-path', '-c', - type=click.Path(exists=True), default=DEFAULT_CONFIG_PATH, - help='Task configuration yml. Defaults to tasks.yml') -@click.option('--arrow-version', '-v', default=None, - help='Set target version explicitly.') -@click.option('--arrow-remote', '-r', default=None, - help='Set GitHub remote explicitly, which is going to be cloned ' - 'on the CI services. Note, that no validation happens ' - 'locally. Examples: https://github.com/apache/arrow or ' - 'https://github.com/kszucs/arrow.') -@click.option('--arrow-branch', '-b', default=None, - help='Give the branch name explicitly, e.g. master, ARROW-1949.') -@click.option('--arrow-sha', '-t', default=None, - help='Set commit SHA or Tag name explicitly, e.g. f67a515, ' - 'apache-arrow-0.11.1.') -@click.option('--fetch/--no-fetch', default=True, - help='Fetch references (branches and tags) from the remote') -@click.option('--dry-run/--push', default=False, - help='Just display the rendered CI configurations without ' - 'submitting them') -@click.pass_obj -def submit(obj, tasks, groups, params, job_prefix, config_path, arrow_version, - arrow_remote, arrow_branch, arrow_sha, fetch, dry_run): - output = obj['output'] - queue, arrow = obj['queue'], obj['arrow'] - - # load available tasks configuration and groups from yaml - config = Config.load_yaml(config_path) - config.validate() - - # Override the detected repo url / remote, branch and sha - this aims to - # make release procedure a bit simpler. - # Note, that the target resivion's crossbow templates must be - # compatible with the locally checked out version of crossbow (which is - # in case of the release procedure), because the templates still - # contain some business logic (dependency installation, deployments) - # which will be reduced to a single command in the future. - target = Target.from_repo(arrow, remote=arrow_remote, branch=arrow_branch, - head=arrow_sha, version=arrow_version) - - # parse additional job parameters - params = dict([p.split("=") for p in params]) - - # instantiate the job object - job = Job.from_config(config=config, target=target, tasks=tasks, - groups=groups, params=params) - - if dry_run: - yaml.dump(job, output) - else: - if fetch: - queue.fetch() - queue.put(job, prefix=job_prefix) - queue.push() - yaml.dump(job, output) - click.echo('Pushed job identifier is: `{}`'.format(job.branch)) - - -@crossbow.command() -@click.argument('job-name', required=True) -@click.pass_obj -def status(obj, job_name): - output = obj['output'] - queue = obj['queue'] - queue.fetch() - - job = queue.get(job_name) - ConsoleReport(job).show(output) - - -@crossbow.command() -@click.argument('prefix', required=True) -@click.pass_obj -def latest_prefix(obj, prefix): - queue = obj['queue'] - queue.fetch() - - latest = queue.latest_for_prefix(prefix) - click.echo(latest.branch) - - -@crossbow.command() -@click.argument('job-name', required=True) -@click.option('--sender-name', '-n', - help='Name to use for report e-mail.') -@click.option('--sender-email', '-e', - help='E-mail to use for report e-mail.') -@click.option('--recipient-email', '-r', - help='Where to send the e-mail report') -@click.option('--smtp-user', '-u', - help='E-mail address to use for SMTP login') -@click.option('--smtp-password', '-P', - help='SMTP password to use for report e-mail.') -@click.option('--smtp-server', '-s', default='smtp.gmail.com', - help='SMTP server to use for report e-mail.') -@click.option('--smtp-port', '-p', default=465, - help='SMTP port to use for report e-mail.') -@click.option('--poll/--no-poll', default=False, - help='Wait for completion if there are tasks pending') -@click.option('--poll-max-minutes', default=180, - help='Maximum amount of time waiting for job completion') -@click.option('--poll-interval-minutes', default=10, - help='Number of minutes to wait to check job status again') -@click.option('--send/--dry-run', default=False, - help='Just display the report, don\'t send it') -@click.pass_obj -def report(obj, job_name, sender_name, sender_email, recipient_email, - smtp_user, smtp_password, smtp_server, smtp_port, poll, - poll_max_minutes, poll_interval_minutes, send): - """ - Send an e-mail report showing success/failure of tasks in a Crossbow run - """ - output = obj['output'] - queue = obj['queue'] - queue.fetch() - - job = queue.get(job_name) - report = EmailReport( - job=job, - sender_name=sender_name, - sender_email=sender_email, - recipient_email=recipient_email - ) - - if poll: - job.wait_until_finished( - poll_max_minutes=poll_max_minutes, - poll_interval_minutes=poll_interval_minutes - ) - - if send: - report.send( - smtp_user=smtp_user, - smtp_password=smtp_password, - smtp_server=smtp_server, - smtp_port=smtp_port - ) - else: - report.show(output) - - -@crossbow.group() -@click.pass_context -def github_page(ctx): - # currently We only list links to nightly binary wheels - pass - - -@github_page.command('generate') -@click.option('-n', default=10, - help='Number of most recent jobs') -@click.option('--gh-branch', default='gh-pages', - help='Github pages branch') -@click.option('--job-prefix', default='nightly', - help='Job/tag prefix the wheel links should be generated for') -@click.option('--dry-run/--push', default=False, - help='Just render the files without pushing') -@click.option('--github-push-token', '-t', default=None, - help='OAuth token for GitHub authentication only used for ' - 'pushing to the crossbow repository, the API requests ' - 'will consume the token passed to the top level crossbow ' - 'command.') -@click.pass_context -def generate_github_page(ctx, n, gh_branch, job_prefix, dry_run, - github_push_token): - queue = ctx.obj['queue'] - queue.fetch() - - # fail early if the requested branch is not available in the local checkout - remote = 'origin' - branch = queue.repo.branches['{}/{}'.format(remote, gh_branch)] - head = queue.repo[branch.target] - - # $ at the end of the pattern is important because we're only looking for - # branches belonging to jobs not branches belonging to tasks - # the branches we're looking for are like 2020-01-01-0 - jobs = queue.jobs(pattern=r"^nightly-(\d{4})-(\d{2})-(\d{2})-(\d+)$") - page = GithubPage(toolz.take(n, jobs)) - files = page.render() - files.update(unflatten_tree(_default_tree)) - - if dry_run: - click.echo(files) - return - - refname = 'refs/heads/{}'.format(gh_branch) - message = 'Update nightly wheel links {}'.format(date.today()) - commit = queue.create_commit(files, parents=[head.id], message=message, - reference_name=refname) - click.echo('Updated `{}` branch\'s head to `{}`' - .format(gh_branch, commit.id)) - queue.push([refname], github_token=github_push_token) - - -@crossbow.command() -@click.argument('job-name', required=True) -@click.option('-t', '--target-dir', - default=str(DEFAULT_ARROW_PATH / 'packages'), - type=click.Path(file_okay=False, dir_okay=True), - help='Directory to download the build artifacts') -@click.pass_obj -def download_artifacts(obj, job_name, target_dir): - """Download build artifacts from GitHub releases""" - output = obj['output'] - - # fetch the queue repository - queue = obj['queue'] - queue.fetch() - - # query the job's artifacts - job = queue.get(job_name) - - # create directory to download the assets to - target_dir = Path(target_dir).absolute() / job_name - target_dir.mkdir(parents=True, exist_ok=True) - - # download the assets while showing the job status - def asset_callback(task_name, task, asset): - if asset is not None: - path = target_dir / task_name / asset.name - path.parent.mkdir(exist_ok=True) - asset.download(path) - - click.echo('Downloading {}\'s artifacts.'.format(job_name)) - click.echo('Destination directory is {}'.format(target_dir)) - click.echo() - - report = ConsoleReport(job) - report.show(output, asset_callback=asset_callback) - - -@crossbow.command() -@click.option('--sha', required=True, help='Target committish') -@click.option('--tag', required=True, help='Target tag') -@click.option('--method', default='curl', help='Use cURL to upload') -@click.option('--pattern', '-p', 'patterns', required=True, multiple=True, - help='File pattern to upload as assets') -@click.pass_obj -def upload_artifacts(obj, tag, sha, patterns, method): - queue = obj['queue'] - queue.github_overwrite_release_assets( - tag_name=tag, target_commitish=sha, method=method, patterns=patterns - ) - - -if __name__ == '__main__': - crossbow(obj={}, auto_envvar_prefix='CROSSBOW') diff --git a/dev/archery/archery/crossbow/reports.py b/dev/archery/archery/crossbow/reports.py new file mode 100644 index 00000000000..bc82db7f51a --- /dev/null +++ b/dev/archery/archery/crossbow/reports.py @@ -0,0 +1,302 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import click +import collections +import operator +import functools +from io import StringIO +import textwrap + + +# TODO(kszucs): use archery.report.JinjaReport instead +class Report: + + def __init__(self, job): + self.job = job + + def show(self): + raise NotImplementedError() + + +class ConsoleReport(Report): + """Report the status of a Job to the console using click""" + + # output table's header template + HEADER = '[{state:>7}] {branch:<52} {content:>16}' + DETAILS = ' └ {url}' + + # output table's row template for assets + ARTIFACT_NAME = '{artifact:>69} ' + ARTIFACT_STATE = '[{state:>7}]' + + # state color mapping to highlight console output + COLORS = { + # from CombinedStatus + 'error': 'red', + 'failure': 'red', + 'pending': 'yellow', + 'success': 'green', + # custom state messages + 'ok': 'green', + 'missing': 'red' + } + + def lead(self, state, branch, n_uploaded, n_expected): + line = self.HEADER.format( + state=state.upper(), + branch=branch, + content='uploaded {} / {}'.format(n_uploaded, n_expected) + ) + return click.style(line, fg=self.COLORS[state.lower()]) + + def header(self): + header = self.HEADER.format( + state='state', + branch='Task / Branch', + content='Artifacts' + ) + delimiter = '-' * len(header) + return '{}\n{}'.format(header, delimiter) + + def artifact(self, state, pattern, asset): + if asset is None: + artifact = pattern + state = 'pending' if state == 'pending' else 'missing' + else: + artifact = asset.name + state = 'ok' + + name_ = self.ARTIFACT_NAME.format(artifact=artifact) + state_ = click.style( + self.ARTIFACT_STATE.format(state=state.upper()), + self.COLORS[state] + ) + return name_ + state_ + + def show(self, outstream, asset_callback=None): + echo = functools.partial(click.echo, file=outstream) + + # write table's header + echo(self.header()) + + # write table's body + for task_name, task in sorted(self.job.tasks.items()): + # if not task_name.startswith("test-debian-10-python-3"): + # continue + # write summary of the uploaded vs total assets + status = task.status() + assets = task.assets() + + # mapping of artifact pattern to asset or None of not uploaded + n_expected = len(task.artifacts) + n_uploaded = len(assets.uploaded_assets()) + echo(self.lead(status.combined_state, task_name, n_uploaded, + n_expected)) + + # show link to the actual build, some of the CI providers implement + # the statuses API others implement the checks API, so display both + for link in status.build_links: + echo(self.DETAILS.format(url=link)) + + # write per asset status + for artifact_pattern, asset in assets.items(): + if asset_callback is not None: + asset_callback(task_name, task, asset) + echo(self.artifact(status.combined_state, artifact_pattern, + asset)) + + +class EmailReport(Report): + + HEADER = textwrap.dedent(""" + Arrow Build Report for Job {job_name} + + All tasks: {all_tasks_url} + """) + + TASK = textwrap.dedent(""" + - {name}: + URL: {url} + """).strip() + + EMAIL = textwrap.dedent(""" + From: {sender_name} <{sender_email}> + To: {recipient_email} + Subject: {subject} + + {body} + """).strip() + + STATUS_HEADERS = { + # from CombinedStatus + 'error': 'Errored Tasks:', + 'failure': 'Failed Tasks:', + 'pending': 'Pending Tasks:', + 'success': 'Succeeded Tasks:', + } + + def __init__(self, job, sender_name, sender_email, recipient_email): + self.sender_name = sender_name + self.sender_email = sender_email + self.recipient_email = recipient_email + super().__init__(job) + + def url(self, query): + repo_url = self.job.queue.remote_url.strip('.git') + return '{}/branches/all?query={}'.format(repo_url, query) + + def listing(self, tasks): + return '\n'.join( + sorted( + self.TASK.format(name=task_name, url=self.url(task.branch)) + for task_name, task in tasks.items() + ) + ) + + def header(self): + url = self.url(self.job.branch) + return self.HEADER.format(job_name=self.job.branch, all_tasks_url=url) + + def subject(self): + return ( + "[NIGHTLY] Arrow Build Report for Job {}".format(self.job.branch) + ) + + def body(self): + buffer = StringIO() + buffer.write(self.header()) + + tasks_by_state = collections.defaultdict(dict) + for task_name, task in self.job.tasks.items(): + state = task.status().combined_state + tasks_by_state[state][task_name] = task + + for state in ('failure', 'error', 'pending', 'success'): + if state in tasks_by_state: + tasks = tasks_by_state[state] + buffer.write('\n') + buffer.write(self.STATUS_HEADERS[state]) + buffer.write('\n') + buffer.write(self.listing(tasks)) + buffer.write('\n') + + return buffer.getvalue() + + def email(self): + return self.EMAIL.format( + sender_name=self.sender_name, + sender_email=self.sender_email, + recipient_email=self.recipient_email, + subject=self.subject(), + body=self.body() + ) + + def show(self, outstream): + outstream.write(self.email()) + + def send(self, smtp_user, smtp_password, smtp_server, smtp_port): + import smtplib + + email = self.email() + + server = smtplib.SMTP_SSL(smtp_server, smtp_port) + server.ehlo() + server.login(smtp_user, smtp_password) + server.sendmail(smtp_user, self.recipient_email, email) + server.close() + + +class CommentReport(Report): + + _markdown_badge = '[![{title}]({badge})]({url})' + + badges = { + 'github': _markdown_badge.format( + title='Github Actions', + url='https://github.com/{repo}/actions?query=branch:{branch}', + badge=( + 'https://github.com/{repo}/workflows/Crossbow/' + 'badge.svg?branch={branch}' + ), + ), + 'azure': _markdown_badge.format( + title='Azure', + url=( + 'https://dev.azure.com/{repo}/_build/latest' + '?definitionId=1&branchName={branch}' + ), + badge=( + 'https://dev.azure.com/{repo}/_apis/build/status/' + '{repo_dotted}?branchName={branch}' + ) + ), + 'travis': _markdown_badge.format( + title='TravisCI', + url='https://travis-ci.com/{repo}/branches', + badge='https://img.shields.io/travis/{repo}/{branch}.svg' + ), + 'circle': _markdown_badge.format( + title='CircleCI', + url='https://circleci.com/gh/{repo}/tree/{branch}', + badge=( + 'https://img.shields.io/circleci/build/github' + '/{repo}/{branch}.svg' + ) + ), + 'appveyor': _markdown_badge.format( + title='Appveyor', + url='https://ci.appveyor.com/project/{repo}/history', + badge='https://img.shields.io/appveyor/ci/{repo}/{branch}.svg' + ), + 'drone': _markdown_badge.format( + title='Drone', + url='https://cloud.drone.io/{repo}', + badge='https://img.shields.io/drone/build/{repo}/{branch}.svg' + ), + } + + def __init__(self, job, crossbow_repo): + self.crossbow_repo = crossbow_repo + super().__init__(job) + + def show(self): + url = 'https://github.com/{repo}/branches/all?query={branch}' + sha = self.job.target.head + + msg = 'Revision: {}\n\n'.format(sha) + msg += 'Submitted crossbow builds: [{repo} @ {branch}]' + msg += '({})\n'.format(url) + msg += '\n|Task|Status|\n|----|------|' + + tasks = sorted(self.job.tasks.items(), key=operator.itemgetter(0)) + for key, task in tasks: + branch = task.branch + + try: + template = self.badges[task.ci] + badge = template.format( + repo=self.crossbow_repo, + repo_dotted=self.crossbow_repo.replace('/', '.'), + branch=branch + ) + except KeyError: + badge = 'unsupported CI service `{}`'.format(task.ci) + + msg += '\n|{}|{}|'.format(key, badge) + + return msg.format(repo=self.crossbow_repo, branch=self.job.branch) diff --git a/dev/archery/archery/tests/fixtures/crossbow-job.yaml b/dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml similarity index 100% rename from dev/archery/archery/tests/fixtures/crossbow-job.yaml rename to dev/archery/archery/crossbow/tests/fixtures/crossbow-job.yaml diff --git a/dev/archery/archery/tests/fixtures/crossbow-success-message.md b/dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md similarity index 100% rename from dev/archery/archery/tests/fixtures/crossbow-success-message.md rename to dev/archery/archery/crossbow/tests/fixtures/crossbow-success-message.md diff --git a/dev/archery/archery/crossbow/tests/test_core.py b/dev/archery/archery/crossbow/tests/test_core.py new file mode 100644 index 00000000000..518474236ac --- /dev/null +++ b/dev/archery/archery/crossbow/tests/test_core.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from archery.utils.source import ArrowSources +from archery.crossbow import Config + + +def test_config(): + src = ArrowSources.find() + conf = Config.load_yaml(src.dev / "tasks" / "tasks.yml") + conf.validate() diff --git a/dev/archery/archery/crossbow/tests/test_crossbow_cli.py b/dev/archery/archery/crossbow/tests/test_crossbow_cli.py new file mode 100644 index 00000000000..ee9ba1ee2fc --- /dev/null +++ b/dev/archery/archery/crossbow/tests/test_crossbow_cli.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from click.testing import CliRunner +import pytest + +from archery.crossbow.cli import crossbow +from archery.utils.git import git + + +@pytest.mark.integration +def test_crossbow_submit(tmp_path): + runner = CliRunner() + + def invoke(*args): + return runner.invoke(crossbow, ['--queue-path', str(tmp_path), *args]) + + # initialize an empty crossbow repository + git.run_cmd("init", str(tmp_path)) + git.run_cmd("-C", str(tmp_path), "remote", "add", "origin", + "https://github.com/dummy/repo") + git.run_cmd("-C", str(tmp_path), "commit", "-m", "initial", + "--allow-empty") + + result = invoke('check-config') + assert result.exit_code == 0 + + result = invoke('submit', '--no-fetch', '--no-push', '-g', 'wheel') + assert result.exit_code == 0 diff --git a/dev/archery/archery/crossbow/tests/test_reports.py b/dev/archery/archery/crossbow/tests/test_reports.py new file mode 100644 index 00000000000..0df292bb557 --- /dev/null +++ b/dev/archery/archery/crossbow/tests/test_reports.py @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import textwrap + +from archery.crossbow.core import yaml +from archery.crossbow.reports import CommentReport + + +def test_crossbow_comment_formatter(load_fixture): + msg = load_fixture('crossbow-success-message.md') + job = load_fixture('crossbow-job.yaml', decoder=yaml.load) + + report = CommentReport(job, crossbow_repo='ursa-labs/crossbow') + expected = msg.format( + repo='ursa-labs/crossbow', + branch='ursabot-1', + revision='f766a1d615dd1b7ee706d05102e579195951a61c', + status='has been succeeded.' + ) + assert report.show() == textwrap.dedent(expected).strip() diff --git a/dev/archery/archery/tests/test_bot.py b/dev/archery/archery/tests/test_bot.py index 9b2d023bc11..e00853ceb2c 100644 --- a/dev/archery/archery/tests/test_bot.py +++ b/dev/archery/archery/tests/test_bot.py @@ -16,18 +16,13 @@ # under the License. import json -from pathlib import Path from unittest.mock import Mock +import click import pytest -import textwrap import responses as rsps -import click -from ruamel.yaml import YAML -from archery.bot import ( - CommentBot, CommandError, CrossbowCommentFormatter, group -) +from archery.bot import CommentBot, CommandError, group @pytest.fixture @@ -36,18 +31,6 @@ def responses(): yield mock -def load_fixture(name): - path = Path(__file__).parent / 'fixtures' / name - with path.open('r') as fp: - if name.endswith('.json'): - return json.load(fp) - elif name.endswith('.yaml'): - yaml = YAML() - return yaml.load(fp) - else: - return fp.read() - - def github_url(path): return 'https://api.github.com:443/{}'.format(path.strip('/')) @@ -86,21 +69,6 @@ def test_click_based_commands(): assert custom_handler('extra', extra='data') == {'extra': 'data'} -def test_crossbow_comment_formatter(): - job = load_fixture('crossbow-job.yaml') - msg = load_fixture('crossbow-success-message.md') - - formatter = CrossbowCommentFormatter(crossbow_repo='ursa-labs/crossbow') - response = formatter.render(job) - expected = msg.format( - repo='ursa-labs/crossbow', - branch='ursabot-1', - revision='f766a1d615dd1b7ee706d05102e579195951a61c', - status='has been succeeded.' - ) - assert response == textwrap.dedent(expected).strip() - - @pytest.mark.parametrize('fixture_name', [ # the bot is not mentioned, nothing to do 'event-issue-comment-not-mentioning-ursabot.json', @@ -109,7 +77,7 @@ def test_crossbow_comment_formatter(): # non-authorized user sent the comment, do not respond 'event-issue-comment-by-non-authorized-user.json', ]) -def test_noop_events(fixture_name): +def test_noop_events(load_fixture, fixture_name): payload = load_fixture(fixture_name) handler = Mock() @@ -119,7 +87,7 @@ def test_noop_events(fixture_name): handler.assert_not_called() -def test_issue_comment_without_pull_request(responses): +def test_issue_comment_without_pull_request(load_fixture, responses): responses.add( responses.GET, github_url('/repositories/169101701/issues/19'), @@ -151,7 +119,7 @@ def handler(command, **kwargs): } -def test_respond_with_usage(responses): +def test_respond_with_usage(load_fixture, responses): responses.add( responses.GET, github_url('/repositories/169101701/issues/26'), @@ -190,7 +158,8 @@ def handler(command, **kwargs): ('@ursabot build', '+1'), ('@ursabot listen', '-1'), ]) -def test_issue_comment_with_commands(responses, command, reaction): +def test_issue_comment_with_commands(load_fixture, responses, command, + reaction): responses.add( responses.GET, github_url('/repositories/169101701/issues/26'), @@ -230,13 +199,3 @@ def handler(command, **kwargs): post = responses.calls[3] assert json.loads(post.request.body) == {'content': reaction} - - -# TODO(kszucs): properly mock it -# def test_crossbow_submit(): -# from click.testing import CliRunner -# runner = CliRunner() -# result = runner.invoke( -# bot, ['crossbow', 'submit', '-g', 'wheel', '--dry-run'] -# ) -# assert result.exit_code == 0 diff --git a/dev/archery/conftest.py b/dev/archery/conftest.py new file mode 100644 index 00000000000..06a643bea56 --- /dev/null +++ b/dev/archery/conftest.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib + +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--enable-integration", + action="store_true", + default=False, + help="run slow tests" + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + ( + "integration: mark test as integration tests involving more " + "extensive setup (only used for crossbow at the moment)" + ) + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--enable-integration"): + return + marker = pytest.mark.skip(reason="need --enable-integration option to run") + for item in items: + if "integration" in item.keywords: + item.add_marker(marker) + + +@pytest.fixture +def load_fixture(request): + current_test_directory = pathlib.Path(request.node.fspath).parent + + def decoder(path): + with path.open('r') as fp: + if path.suffix == '.json': + import json + return json.load(fp) + elif path.suffix == '.yaml': + import yaml + return yaml.load(fp) + else: + return fp.read() + + def loader(name, decoder=decoder): + path = current_test_directory / 'fixtures' / name + return decoder(path) + + return loader diff --git a/dev/archery/setup.py b/dev/archery/setup.py index 2696c0ae5a6..892e6b2a8bd 100755 --- a/dev/archery/setup.py +++ b/dev/archery/setup.py @@ -26,12 +26,12 @@ extras = { 'benchmark': ['pandas'], - # bot extra includes crossbow's dependencies - 'bot': ['ruamel.yaml', 'pygithub', 'github3.py', 'jinja2', 'jira', - 'pygit2', 'setuptools_scm', 'toolz'], 'docker': ['ruamel.yaml', 'python-dotenv'], - 'release': ['jinja2', 'jira', 'semver', 'gitpython'] + 'release': ['jinja2', 'jira', 'semver', 'gitpython'], + 'crossbow': ['github3.py', 'jinja2', 'pygit2', 'ruamel.yaml', + 'setuptools_scm'], } +extras['bot'] = extras['crossbow'] + ['pygithub', 'jira'] extras['all'] = list(set(functools.reduce(operator.add, extras.values()))) setup( diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index c2e22abd39c..e8d813558c1 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -47,6 +47,7 @@ dev/requirements*.txt dev/archery/MANIFEST.in dev/archery/requirements*.txt dev/archery/archery/tests/fixtures/* +dev/archery/archery/crossbow/tests/fixtures/* dev/release/rat_exclude_files.txt dev/tasks/homebrew-formulae/apache-arrow.rb dev/tasks/linux-packages/apache-arrow-archive-keyring/debian/apache-arrow-archive-keyring.install diff --git a/dev/tasks/conda-recipes/azure.linux.yml b/dev/tasks/conda-recipes/azure.linux.yml index 3ab9504dbb6..161fd14e90c 100755 --- a/dev/tasks/conda-recipes/azure.linux.yml +++ b/dev/tasks/conda-recipes/azure.linux.yml @@ -1,3 +1,5 @@ +{% import 'macros.jinja' as macros with context %} + jobs: - job: linux pool: @@ -19,21 +21,11 @@ jobs: displayName: Configure binfmt_misc condition: not(startsWith(variables['CONFIG'], 'linux_64')) - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow + {{ macros.azure_checkout_arrow() }} - # Using github release tries to find a common ancestor between the - # currently pushed tag and the latest tag of the github repository - # (don't know why). - # The tag upload took 43 minutes because of this scan, so use an - # alternative upload script. - task: CondaEnvironment@1 inputs: - packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz anaconda-client shyaml' + packageSpecs: 'anaconda-client shyaml' installOptions: '-c conda-forge' updateConda: false @@ -42,22 +34,5 @@ jobs: CI=azure arrow/dev/tasks/conda-recipes/run_docker_build.sh $(pwd)/build_artifacts displayName: Run docker build - # Upload to github releases - - script: | - python arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - --pattern "build_artifacts/linux-64/*.tar.bz2" - env: - CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) - displayName: Upload packages as a GitHub release - - {% if arrow.branch == 'master' %} - # Upload to custom anaconda channel - - script: | - anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force build_artifacts/linux-64/*.tar.bz2 - displayName: Upload packages to Anaconda - {% endif %} + {{ macros.azure_upload_releases("build_artifacts/linux-64/*.tar.bz2") }} + {{ macros.azure_upload_anaconda("build_artifacts/linux-64/*.tar.bz2") }} diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 3251754d2dc..58afa8045cb 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -1,3 +1,5 @@ +{% import 'macros.jinja' as macros with context %} + jobs: - job: osx pool: @@ -27,12 +29,7 @@ jobs: conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=3 conda-build displayName: 'Add conda-forge-ci-setup=3' - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow + {{ macros.azure_checkout_arrow() }} - script: | source activate base @@ -44,9 +41,8 @@ jobs: conda update --yes --quiet --override-channels -c conda-forge -c defaults --all displayName: Configure conda and conda-build workingDirectory: arrow/dev/tasks/conda-recipes - env: { + env: OSX_FORCE_SDK_DOWNLOAD: "1" - } - script: | source activate base @@ -75,31 +71,5 @@ jobs: workingDirectory: arrow/dev/tasks/conda-recipes displayName: Build recipes - # Using github release tries to find a common ancestor between the - # currently pushed tag and the latest tag of the github repository - # (don't know why). - # The tag upload took 43 minutes because of this scan, so use an - # alternative upload script. - - script: | - source activate base - conda install -y click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz - python arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - --pattern "arrow/dev/tasks/conda-recipes/build_artifacts/osx-64/*.tar.bz2" - env: - CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) - displayName: Upload packages as a GitHub release - - {% if arrow.branch == 'master' %} - # Upload to custom anaconda channel - - script: | - source activate base - conda install -y anaconda-client - anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force build_artifacts/osx-64/*.tar.bz2 - displayName: Upload packages to Anaconda - workingDirectory: arrow/dev/tasks/conda-recipes - {% endif %} + {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-64/*.tar.bz2") }} + {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-64/*.tar.bz2") }} diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index d4322746eac..a3ec6931caf 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -1,3 +1,5 @@ +{% import 'macros.jinja' as macros with context %} + jobs: - job: win pool: @@ -41,12 +43,7 @@ jobs: displayName: Install conda-build and activate environment - script: set PYTHONUNBUFFERED=1 - - script: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - displayName: Clone arrow + {{ macros.azure_checkout_arrow()|indent(2) }} # Configure the VM - script: setup_conda_rc .\ .\ .\.ci_support\%CONFIG%.yaml @@ -76,34 +73,5 @@ jobs: PYTHONUNBUFFERED: 1 condition: contains(variables['R_CONFIG'], 'win') - # Using github release tries to find a common ancestor between the - # currently pushed tag and the latest tag of the github repository - # (don't know why). - # The tag upload took 43 minutes because of this scan, so use an - # alternative upload script. - - task: CondaEnvironment@1 - inputs: - packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz' - installOptions: '-c conda-forge' - updateConda: false - - script: | - python arrow/dev/tasks/crossbow.py ^ - --queue-path . ^ - --queue-remote {{ queue_remote_url }} ^ - upload-artifacts ^ - --sha {{ task.branch }} ^ - --tag {{ task.tag }} ^ - --pattern "D:\bld\win-64\*.tar.bz2" - env: - CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) - displayName: Upload packages as a GitHub release - - {% if arrow.branch == 'master' %} - # Upload to custom anaconda channel - - script: | - source activate base - conda install -y anaconda-client - anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force D:\bld\win-64\*.tar.bz2 - displayName: Upload packages to Anaconda - workingDirectory: arrow/dev/tasks/conda-recipes - {% endif %} + {{ macros.azure_upload_releases("D:\bld\win-64\*.tar.bz2")|indent(2) }} + {{ macros.azure_upload_anaconda("D:\bld\win-64\*.tar.bz2")|indent(2) }} diff --git a/dev/tasks/conda-recipes/drone-steps.sh b/dev/tasks/conda-recipes/drone-steps.sh index 71564108b2f..dffdb41b088 100755 --- a/dev/tasks/conda-recipes/drone-steps.sh +++ b/dev/tasks/conda-recipes/drone-steps.sh @@ -12,9 +12,11 @@ conda install -y mamba $FEEDSTOCK_ROOT/build_steps.sh ${OUTPUT_DIR} # Upload as Github release -mamba install -y click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz anaconda-client shyaml -c conda-forge +mamba install -y anaconda-client shyaml -c conda-forge + pushd $DRONE_WORKSPACE -python arrow/dev/tasks/crossbow.py \ +pip install -e arrow/dev/archery[crossbow] +archery crossbow \ --queue-path . \ --queue-remote ${QUEUE_REMOTE_URL} \ upload-artifacts \ diff --git a/dev/tasks/docker-tests/azure.linux.yml b/dev/tasks/docker-tests/azure.linux.yml index f136947c524..c3706be4434 100644 --- a/dev/tasks/docker-tests/azure.linux.yml +++ b/dev/tasks/docker-tests/azure.linux.yml @@ -20,12 +20,12 @@ jobs: pool: vmImage: ubuntu-16.04 timeoutInMinutes: 360 - {%- if env is defined %} + {% if env is defined %} variables: - {%- for key, value in env.items() %} + {% for key, value in env.items() %} {{ key }}: {{ value }} - {%- endfor %} - {%- endif %} + {% endfor %} + {% endif %} steps: - task: DockerInstaller@0 diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index c5851ed73ef..255c9ac14c4 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -15,42 +15,28 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: test: name: Docker Test runs-on: ubuntu-latest steps: - - name: Checkout Arrow - shell: bash - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Free Up Disk Space shell: bash run: arrow/ci/scripts/util_cleanup.sh - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.9 - - name: Setup Archery - run: pip install -e arrow/dev/archery[docker] + - name: Execute Docker Build shell: bash - {%- if env is defined %} + {% if env is defined %} env: - {%- for key, value in env.items() %} + {% for key, value in env.items() %} {{ key }}: {{ value }} - {%- endfor %} - {%- endif %} + {% endfor %} + {% endif %} run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ run }} diff --git a/dev/tasks/gandiva-jars/github.linux.yml b/dev/tasks/gandiva-jars/github.linux.yml index 79d2cccb0f7..eb164182868 100644 --- a/dev/tasks/gandiva-jars/github.linux.yml +++ b/dev/tasks/gandiva-jars/github.linux.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: package: @@ -48,21 +43,5 @@ jobs: env: OS_NAME: "linux" CHECK_SHARED_DEPENDENCIES: true - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Set up Crossbow - run: | - pip install --requirement arrow/dev/tasks/requirements-crossbow.txt - - name: Upload artifacts - run: | - python arrow/dev/tasks/crossbow.py \ - --queue-path arrow \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - --pattern "arrow/dist/*.jar" - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} + + {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }} diff --git a/dev/tasks/gandiva-jars/github.osx.yml b/dev/tasks/gandiva-jars/github.osx.yml index 158c22dc869..3dd6fe46bb6 100644 --- a/dev/tasks/gandiva-jars/github.osx.yml +++ b/dev/tasks/gandiva-jars/github.osx.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: package: @@ -47,26 +42,5 @@ jobs: OS_NAME: "osx" CHECK_SHARED_DEPENDENCIES: true MACOSX_DEPLOYMENT_TARGET: "10.11" - - name: Set up Crossbow - run: | - brew install libgit2 - pip3 install \ - click \ - github3.py \ - jinja2 \ - jira \ - pygit2 \ - ruamel.yaml \ - setuptools_scm \ - toolz - - name: Upload artifacts - run: | - python3 arrow/dev/tasks/crossbow.py \ - --queue-path arrow \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - --pattern "arrow/dist/*.jar" - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} \ No newline at end of file + + {{ macros.github_upload_releases("arrow/dist/*.jar")|indent }} diff --git a/dev/tasks/linux-packages/github.linux.amd64.yml b/dev/tasks/linux-packages/github.linux.amd64.yml index 1335a979f47..380f025afaa 100644 --- a/dev/tasks/linux-packages/github.linux.amd64.yml +++ b/dev/tasks/linux-packages/github.linux.amd64.yml @@ -15,28 +15,20 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: package: name: Package runs-on: ubuntu-18.04 steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_login_dockerhub()|indent }} + - name: Set up Ruby uses: actions/setup-ruby@v1 - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - name: Free Up Disk Space shell: bash run: arrow/ci/scripts/util_cleanup.sh @@ -65,8 +57,6 @@ jobs: shell: bash run: | pushd arrow/dev/tasks/linux-packages - docker login -u "{{ '${{ secrets.DOCKERHUB_USER }}' }}" \ - -p "{{ '${{ secrets.DOCKERHUB_TOKEN }}' }}" rake docker:push popd env: @@ -103,28 +93,6 @@ jobs: APT_TARGETS: {{ target }} ARROW_VERSION: {{ arrow.version }} YUM_TARGETS: {{ target }} - # Using GitHub release tries to find a common ancestor between the - # currently pushed tag and the latest tag of the GitHub repository - # (don't know why). - # The tag upload took 43 minutes because of this scan, so use an - # alternative upload script. - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Set up Crossbow - run: | - pip install --requirement arrow/dev/tasks/requirements-crossbow.txt - - name: Upload artifacts - run: | - python arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - {%- for extension in upload_extensions %} - --pattern "arrow/dev/tasks/linux-packages/**/*{{ extension }}" \ - {%- endfor %} - --sha {{ task.branch }} \ - --tag {{ task.tag }} - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} + + {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/**/*{}") %} + {{ macros.github_upload_releases(patterns)|indent }} diff --git a/dev/tasks/linux-packages/travis.linux.arm64.yml b/dev/tasks/linux-packages/travis.linux.arm64.yml index 0b83520ccf0..e9457d6a337 100644 --- a/dev/tasks/linux-packages/travis.linux.arm64.yml +++ b/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +{% import 'macros.jinja' as macros with context %} + arch: arm64-graviton2 virt: vm os: linux @@ -66,9 +68,8 @@ env: - YUM_TARGETS={{ target }} before_script: - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD + {{ macros.travis_checkout_arrow() }} + {{ macros.travis_docker_login() }} # Build createrepo_c from source. # We can remove them when we can install createrepo_c package @@ -143,15 +144,5 @@ script: - popd after_success: - - sudo -H pip3 install --upgrade pip - - sudo -H pip3 install -r arrow/dev/tasks/requirements-crossbow.txt pygit2==1.0 - - | - python3 arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - {%- for extension in upload_extensions %} - --pattern "arrow/dev/tasks/linux-packages/**/*{{ extension }}" \ - {%- endfor %} - --sha {{ task.branch }} \ - --tag {{ task.tag }} + {% set patterns = upload_extensions | format_all("arrow/python/repaired_wheels/*.whl") %} + {{ macros.github_upload_releases(patterns) }} diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja new file mode 100644 index 00000000000..e0552b11bcf --- /dev/null +++ b/dev/tasks/macros.jinja @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{%- macro github_header() -%} +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow +on: + push: + branches: + - "*-github-*" +{% endmacro %} + +{%- macro github_checkout_arrow() -%} + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow config core.symlinks true + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh +{% endmacro %} + +{%- macro github_login_dockerhub() -%} + - name: Login to Dockerhub + uses: docker/login-action@v1 + with: + username: {{ '${{ secrets.DOCKERHUB_USER }}' }} + password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }} +{% endmacro %} + +{%- macro github_login_ghcr() -%} + - name: Login to GitHub Container Registry + shell: bash + run: docker login ghcr.io -u {{ '${{ github.repository_owner }}' }} -p {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }} +{% endmacro %} + +{%- macro github_install_archery() -%} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Archery + shell: bash + run: pip install -e arrow/dev/archery[all] +{% endmacro %} + +{%- macro github_upload_releases(pattern) -%} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Setup Crossbow + shell: bash + run: pip install -e arrow/dev/archery[crossbow] + - name: Upload artifacts + shell: bash + run: | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + --pattern "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + --pattern "{{ p }}" {{ "\\" if not loop.last else "" }} + {% endfor %} + {% endif %} + env: + CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} +{% endmacro %} + +{%- macro github_upload_gemfury(pattern) -%} + {% if arrow.branch == 'master' %} + - name: Upload package to Gemfury + shell: bash + run: | + path=$(ls {{ patter }}) + curl -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/ + env: + CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} + CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} + {% endif %} +{% endmacro %} + +{%- macro azure_checkout_arrow() -%} + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + displayName: Clone arrow +{% endmacro %} + +{%- macro azure_upload_releases(pattern) -%} + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.8' + - script: pip install -e arrow/dev/archery[crossbow] + displayName: Install Crossbow + - bash: | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + --pattern "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + --pattern "{{ p }}" {{ "" if not loop.last else "" }} + {% endfor %} + {% endif %} + env: + CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) + displayName: Upload packages as a GitHub release +{% endmacro %} + +{%- macro azure_upload_anaconda(pattern) -%} + {% if arrow.branch == 'master' %} + - task: CondaEnvironment@1 + inputs: + packageSpecs: 'anaconda-client' + installOptions: '-c conda-forge' + updateConda: no + - script: | + conda install -y anaconda-client + anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force {{ pattern }} + displayName: Upload packages to Anaconda + {% endif %} +{% endmacro %} + +{%- macro travis_checkout_arrow() -%} + - git clone --no-checkout {{ arrow.remote }} arrow + - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + - git -C arrow checkout FETCH_HEAD + - git -C arrow submodule update --init --recursive +{% endmacro %} + +{%- macro travis_install_archery() -%} + - sudo -H pip3 install --upgrade pip + - sudo -H pip3 install docker-compose + - sudo -H pip3 install -e arrow/dev/archery[docker] +{% endmacro %} + +{%- macro travis_docker_login() -%} + - echo "${DOCKERHUB_TOKEN}" | docker login --username "${DOCKERHUB_USER}" --password-stdin +{% endmacro %} + +{%- macro travis_upload_releases(pattern) -%} + - sudo -H pip3 install pygit2==1.0 + - sudo -H pip3 install arrow/dev/archery[crossbow] + - | + archery crossbow + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + --pattern "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + --pattern "{{ p }}" {{ "\\" if not loop.last else "" }} + {% endfor %} + {% endif %} +{% endmacro %} + +{%- macro travis_upload_gemfury(pattern) -%} + {% if arrow.branch == 'master' %} + - | + WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl) + curl \ + -F "package=@${WHEEL_PATH}" \ + "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/" + {% endif %} +{% endmacro %} diff --git a/dev/tasks/nightlies.sample.yml b/dev/tasks/nightlies.sample.yml index 12b7a094285..710f7c0ad37 100644 --- a/dev/tasks/nightlies.sample.yml +++ b/dev/tasks/nightlies.sample.yml @@ -52,32 +52,17 @@ before_install: conda config --set show_channel_urls true install: - - | - conda install -y \ - jinja2 \ - pygit2 \ - click \ - ruamel.yaml \ - setuptools_scm \ - github3.py \ - python-gnupg - -script: + - pushd .. # to build against a specific branch of a fork # git clone -b https://github.com//arrow - - pushd .. - - git clone -b master https://github.com/apache/arrow + - git clone https://github.com/apache/arrow + - pip install dev/archery[crossbow] +script: # submit packaging tasks - | if [ $TRAVIS_EVENT_TYPE = "cron" ]; then - python arrow/dev/tasks/crossbow.py submit \ - -g conda \ - -g wheel \ - -g linux + archery crossbow submit -g conda -g wheel -g linux else - python arrow/dev/tasks/crossbow.py submit --dry-run \ - -g conda \ - -g wheel \ - -g linux + archery crossbow submit --dry-run -g conda -g wheel -g linux fi diff --git a/dev/tasks/nuget-packages/github.linux.yml b/dev/tasks/nuget-packages/github.linux.yml index 6bb5ea0e13f..cd03a7bfeea 100644 --- a/dev/tasks/nuget-packages/github.linux.yml +++ b/dev/tasks/nuget-packages/github.linux.yml @@ -15,36 +15,18 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# GitHub comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" - -env: - DOCKER_BUILDKIT: 0 - COMPOSE_DOCKER_CLI_BUILD: 1 +{{ macros.github_header() }} jobs: package: name: Package runs-on: ubuntu-latest steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Setup Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Setup Archery - run: pip install -e arrow/dev/archery[docker] + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Prepare version run: | sed -i'' -E -e \ @@ -55,31 +37,7 @@ jobs: pushd arrow archery docker run {{ run }} popd - # Using GitHub release tries to find a common ancestor between the - # currently pushed tag and the latest tag of the GitHub repository - # (don't know why). - # The tag upload took 43 minutes because of this scan, so use an - # alternative upload script. - - name: Set up Crossbow - run: | - pip install \ - click \ - github3.py \ - jinja2 \ - jira \ - pygit2 \ - ruamel.yaml \ - setuptools_scm \ - toolz - - name: Upload artifacts - run: | - python arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --pattern "arrow/csharp/artifacts/**/*.nupkg" \ - --pattern "arrow/csharp/artifacts/**/*.snupkg" \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} + + {% set patterns = ["arrow/csharp/artifacts/**/*.nupkg", + "arrow/csharp/artifacts/**/*.snupkg"] %} + {{ macros.github_upload_releases(patterns)|indent }} diff --git a/dev/tasks/python-sdist/github.yml b/dev/tasks/python-sdist/github.yml index b784fbdf8f1..68371876ab8 100644 --- a/dev/tasks/python-sdist/github.yml +++ b/dev/tasks/python-sdist/github.yml @@ -15,35 +15,18 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: build: name: "Build sdist" runs-on: ubuntu-20.04 steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow config core.symlinks true - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Fetch Submodules and Tags - run: cd arrow && ci/scripts/util_checkout.sh - - name: Free Up Disk Space - shell: bash - run: arrow/ci/scripts/util_cleanup.sh - - uses: actions/setup-python@v2 - - name: Setup Archery - run: pip install -e arrow/dev/archery[docker] + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Build sdist run: | archery docker run python-sdist @@ -52,32 +35,11 @@ jobs: {% endif %} env: PYARROW_VERSION: {{ arrow.no_rc_version }} + - name: Test sdist run: archery docker run ubuntu-python-sdist-test env: PYARROW_VERSION: {{ arrow.no_rc_version }} - {% if arrow.branch == 'master' %} - - name: Upload to gemfury - run: | - SDIST_PATH=$(echo arrow/python/dist/*.tar.gz) - curl \ - -F "package=@${SDIST_PATH}" \ - "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/" - env: - CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} - CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} - {% endif %} - - name: Setup Crossbow - run: pip install --requirement arrow/dev/tasks/requirements-crossbow.txt - - name: Upload artifacts - shell: bash - run: | - python arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --pattern "arrow/python/dist/*.tar.gz" \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} + + {{ macros.github_upload_releases("arrow/python/dist/*.tar.gz")|indent }} + {{ macros.github_upload_gemfury("arrow/python/dist/*.tar.gz")|indent }} diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml b/dev/tasks/python-wheels/github.linux.amd64.yml index 779aa541d82..a6264078715 100644 --- a/dev/tasks/python-wheels/github.linux.amd64.yml +++ b/dev/tasks/python-wheels/github.linux.amd64.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: build: @@ -34,38 +29,14 @@ jobs: PYTHON: {{ python_version }} steps: - ############################ Checkout ################################### - - name: Checkout Arrow - shell: bash - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow config core.symlinks true - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh - - ############################ Docker Registry ############################ - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - username: {{ '${{ secrets.DOCKERHUB_USER }}' }} - password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }} + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + {{ macros.github_login_dockerhub()|indent }} - ############################ Archery Installation ####################### - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Archery and Crossbow dependencies - run: pip install -e arrow/dev/archery[all] - - ############################ Build & Test ############################### - name: Build wheel shell: bash run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} + # TODO(kszucs): auditwheel show - name: Test wheel shell: bash @@ -73,35 +44,5 @@ jobs: archery docker run python-wheel-manylinux-test-imports archery docker run python-wheel-manylinux-test-unittests - ############################ Artifact Uploading ######################### - - name: Upload artifacts - shell: bash - run: | - python arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --pattern "arrow/python/repaired_wheels/*.whl" \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} - - {% if arrow.branch == 'master' %} - - name: Push the docker image - shell: bash - run: | - archery docker push python-wheel-manylinux-{{ manylinux_version }} - archery docker push python-wheel-manylinux-test-unittests - - - name: Upload to gemfury - shell: bash - run: | - WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl) - curl \ - -F "package=@${WHEEL_PATH}" \ - "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/" - env: - CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} - CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} - {% endif %} + {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index db67d99342d..af0cc44ef09 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -14,14 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} env: ARROW_S3: {{ arrow_s3 }} @@ -41,13 +36,7 @@ jobs: name: Build wheel for OS X runs-on: macos-latest steps: - - name: Checkout Arrow - run: | - set -ex - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive + {{ macros.github_checkout_arrow()|indent }} - name: Install System Dependencies run: brew install bison ninja @@ -140,28 +129,5 @@ jobs: unset MACOSX_DEPLOYMENT_TARGET arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow - - name: Setup Crossbow - run: pip install --requirement arrow/dev/tasks/requirements-crossbow.txt - - - name: Upload artifacts - run: | - python arrow/dev/tasks/crossbow.py \ - --queue-path $(pwd) \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - --pattern "arrow/python/dist/*.whl" - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} - - {% if arrow.branch == 'master' %} - - name: Upload package to Gemfury - run: | - # upload to gemfury pypi repository, there should be a single wheel - path=$(ls arrow/python/dist/*.whl) - curl -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/ - env: - CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} - CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} - {% endif %} + {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.windows.yml b/dev/tasks/python-wheels/github.windows.yml index 2e24f36c701..922533b0f82 100644 --- a/dev/tasks/python-wheels/github.windows.yml +++ b/dev/tasks/python-wheels/github.windows.yml @@ -15,14 +15,9 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: build: @@ -42,64 +37,17 @@ jobs: ARCHERY_USE_DOCKER_CLI: 1 steps: - ############################ Checkout ################################### - - name: Checkout Arrow - shell: bash - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow config core.symlinks true - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_login_ghcr()|indent }} + {{ macros.github_install_archery()|indent }} - ############################ Docker Registry ############################ - - name: Login to GitHub Container Registry - shell: bash - run: docker login ghcr.io -u {{ '${{ github.repository_owner }}' }} -p {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }} - - ############################ Archery Installation ####################### - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Install Archery and Crossbow dependencies - shell: bash - run: pip install -e arrow/dev/archery[all] - - ############################ Build & Test ############################### - name: Build wheel shell: cmd run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 + - name: Test wheel shell: cmd run: archery docker run python-wheel-windows-test - ############################ Artifact Uploading ######################### - - name: Upload artifacts - shell: bash - run: | - python arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --pattern "arrow/python/dist/*.whl" \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} - env: - CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN }}' }} - - {% if arrow.branch == 'master' %} - - name: Upload to gemfury - shell: bash - run: | - WHEEL_PATH=$(echo arrow/python/dist/*.whl) - curl.exe \ - -F "package=@${WHEEL_PATH}" \ - "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/" - env: - CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} - CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} - {% endif %} + {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml b/dev/tasks/python-wheels/travis.linux.arm64.yml index 01ae4871659..137ad6b2a56 100644 --- a/dev/tasks/python-wheels/travis.linux.arm64.yml +++ b/dev/tasks/python-wheels/travis.linux.arm64.yml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +{% import 'macros.jinja' as macros with context %} + arch: arm64-graviton2 virt: vm os: linux @@ -43,23 +45,14 @@ env: - PYTHON={{ python_version }} before_script: - - git clone --no-checkout {{ arrow.remote }} arrow - - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - - git -C arrow checkout FETCH_HEAD - - git -C arrow submodule update --init --recursive + {{ macros.travis_checkout_arrow() }} + {{ macros.travis_docker_login() }} script: # Install Archery and Crossbow dependencies - - sudo -H pip3 install --upgrade pip - - sudo -H pip3 install docker-compose - - sudo -H pip3 install -e arrow/dev/archery[docker] - - sudo -H pip3 install -r arrow/dev/tasks/requirements-crossbow.txt pygit2==1.0 - - # Docker Login: - - echo "${DOCKERHUB_TOKEN}" | docker login --username "${DOCKERHUB_USER}" --password-stdin + {{ macros.travis_install_archery() }} # Build and Test packages - # output something every minutes to prevent travis from killing the build - while sleep 1m; do echo "=====[ $SECONDS seconds still running ]====="; done & - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} @@ -69,23 +62,11 @@ script: after_success: # Upload wheel as github artifact - - | - python3 arrow/dev/tasks/crossbow.py \ - --queue-path . \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --pattern "arrow/python/repaired_wheels/*.whl" \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} + {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }} + {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }} {% if arrow.branch == 'master' %} # Push the docker image to dockerhub - archery docker push python-wheel-manylinux-{{ manylinux_version }} - archery docker push python-wheel-manylinux-test-unittests - # Upload as nightly wheel to gemfury - - | - WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl) - curl \ - -F "package=@${WHEEL_PATH}" \ - "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/" {% endif %} diff --git a/dev/tasks/requirements-crossbow.txt b/dev/tasks/requirements-crossbow.txt deleted file mode 100644 index 2436b4d1173..00000000000 --- a/dev/tasks/requirements-crossbow.txt +++ /dev/null @@ -1,7 +0,0 @@ -click>=7.1 -github3.py -jinja2 -pygit2 -ruamel.yaml -setuptools_scm -toolz diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a3f05a82a02..725bc4ca0a6 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -40,7 +40,7 @@ groups: linux-arm64: - debian-*-arm64 - ubuntu-*-arm64 - - centos-*-aarch64 + - centos-*-arm64 gandiva: - gandiva-* @@ -236,7 +236,7 @@ tasks: - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 - conda-linux-gcc-py36-aarch64: + conda-linux-gcc-py36-arm64: ci: drone template: conda-recipes/drone.yml params: @@ -245,7 +245,7 @@ tasks: - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 - conda-linux-gcc-py37-aarch64: + conda-linux-gcc-py37-arm64: ci: drone template: conda-recipes/drone.yml params: @@ -254,7 +254,7 @@ tasks: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 - conda-linux-gcc-py38-aarch64: + conda-linux-gcc-py38-arm64: ci: drone template: conda-recipes/drone.yml params: @@ -263,7 +263,7 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - conda-linux-gcc-py39-aarch64: + conda-linux-gcc-py39-arm64: ci: drone template: conda-recipes/drone.yml params: @@ -343,239 +343,62 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - ############################## Wheel Linux ################################## - # AMD64 +{% for python_version, python_tag, abi_tag in [("3.6", "cp36", "cp36m"), + ("3.7", "cp37", "cp37m"), + ("3.8", "cp38", "cp38"), + ("3.9", "cp39", "cp39")] %} - wheel-manylinux2010-cp36m-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.6 - manylinux_version: 2010 - artifacts: - - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2010_x86_64.whl - - wheel-manylinux2010-cp37m-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.7 - manylinux_version: 2010 - artifacts: - - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2010_x86_64.whl - - wheel-manylinux2010-cp38-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.8 - manylinux_version: 2010 - artifacts: - - pyarrow-{no_rc_version}-cp38-cp38-manylinux2010_x86_64.whl - - wheel-manylinux2010-cp39-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.9 - manylinux_version: 2010 - artifacts: - - pyarrow-{no_rc_version}-cp39-cp39-manylinux2010_x86_64.whl - - wheel-manylinux2014-cp36m-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.6 - manylinux_version: 2014 - artifacts: - - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2014_x86_64.whl - - wheel-manylinux2014-cp37m-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.7 - manylinux_version: 2014 - artifacts: - - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2014_x86_64.whl - - wheel-manylinux2014-cp38-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.8 - manylinux_version: 2014 - artifacts: - - pyarrow-{no_rc_version}-cp38-cp38-manylinux2014_x86_64.whl - - wheel-manylinux2014-cp39-amd64: - ci: github - template: python-wheels/github.linux.amd64.yml - params: - python_version: 3.9 - manylinux_version: 2014 - artifacts: - - pyarrow-{no_rc_version}-cp39-cp39-manylinux2014_x86_64.whl - - # ARM64v8 +{############################## Wheel Linux ##################################} - wheel-manylinux2014-cp36m-arm64: - ci: travis - template: python-wheels/travis.linux.arm64.yml - params: - python_version: 3.6 - manylinux_version: 2014 - artifacts: - - pyarrow-{no_rc_version}-cp36-cp36m-manylinux2014_aarch64.whl - - wheel-manylinux2014-cp37m-arm64: - ci: travis - template: python-wheels/travis.linux.arm64.yml - params: - python_version: 3.7 - manylinux_version: 2014 - artifacts: - - pyarrow-{no_rc_version}-cp37-cp37m-manylinux2014_aarch64.whl - - wheel-manylinux2014-cp38-arm64: - ci: travis - template: python-wheels/travis.linux.arm64.yml - params: - python_version: 3.8 - manylinux_version: 2014 - artifacts: - - pyarrow-{no_rc_version}-cp38-cp38-manylinux2014_aarch64.whl +{% for ci, arch, arch_alias, manylinux in [("github", "amd64", "x86_64", "2010"), + ("github", "amd64", "x86_64", "2014"), + ("travis", "arm64", "aarch64", "2014")] %} + {% set platform_tag = "manylinux{}_{}".format(manylinux, arch_alias) %} - wheel-manylinux2014-cp39-arm64: - ci: travis - template: python-wheels/travis.linux.arm64.yml + wheel-manylinux{{ manylinux }}-{{ python_tag }}-{{ arch }}: + ci: {{ ci }} + template: python-wheels/{{ ci }}.linux.{{ arch }}.yml params: - python_version: 3.9 - manylinux_version: 2014 + python_version: {{ python_version }} + manylinux_version: {{ manylinux }} artifacts: - - pyarrow-{no_rc_version}-cp39-cp39-manylinux2014_aarch64.whl + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl - ############################## Wheel OSX #################################### +{% endfor %} - wheel-osx-mavericks-cp36m: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: 3.6 - macos_deployment_target: 10.9 - arrow_s3: "OFF" - artifacts: - - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_9_x86_64.whl +{############################## Wheel OSX ####################################} - wheel-osx-mavericks-cp37m: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: 3.7 - macos_deployment_target: 10.9 - arrow_s3: "OFF" - artifacts: - - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_9_x86_64.whl +# enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl +{% for macos_version, macos_codename, arrow_s3 in [("10.9", "mavericks", "OFF"), + ("10.13", "high-sierra", "ON")] %} + {% set platform_tag = "macosx_{}_{}".format(macos_version.replace('.', '_'), arch_alias) %} - wheel-osx-mavericks-cp38: + wheel-osx-{{ macos_codename }}-{{ python_tag }}: ci: github template: python-wheels/github.osx.yml params: - python_version: 3.8 - macos_deployment_target: 10.9 - arrow_s3: "OFF" + python_version: {{ python_version }} + macos_deployment_target: {{ macos_version }} + arrow_s3: {{ arrow_s3 }} artifacts: - - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl - wheel-osx-mavericks-cp39: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: 3.9 - macos_deployment_target: 10.9 - arrow_s3: "OFF" - artifacts: - - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_9_x86_64.whl +{% endfor %} - # enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl +{############################## Wheel Windows ################################} - wheel-osx-high-sierra-cp36m: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: 3.6 - macos_deployment_target: 10.13 - arrow_s3: "ON" - artifacts: - - pyarrow-{no_rc_version}-cp36-cp36m-macosx_10_13_x86_64.whl - - wheel-osx-high-sierra-cp37m: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: 3.7 - macos_deployment_target: 10.13 - arrow_s3: "ON" - artifacts: - - pyarrow-{no_rc_version}-cp37-cp37m-macosx_10_13_x86_64.whl - - wheel-osx-high-sierra-cp38: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: 3.8 - macos_deployment_target: 10.13 - arrow_s3: "ON" - artifacts: - - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_13_x86_64.whl - - wheel-osx-high-sierra-cp39: - ci: github - template: python-wheels/github.osx.yml - params: - python_version: 3.9 - macos_deployment_target: 10.13 - arrow_s3: "ON" - artifacts: - - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_13_x86_64.whl - - ############################## Wheel Windows ################################ - - wheel-windows-cp36m: - ci: github - template: python-wheels/github.windows.yml - params: - python_version: 3.6 - artifacts: - - pyarrow-{no_rc_version}-cp36-cp36m-win_amd64.whl - - wheel-windows-cp37m: - ci: github - template: python-wheels/github.windows.yml - params: - python_version: 3.7 - artifacts: - - pyarrow-{no_rc_version}-cp37-cp37m-win_amd64.whl - - wheel-windows-cp38: + wheel-windows-{{ python_tag }}: ci: github template: python-wheels/github.windows.yml params: - python_version: 3.8 + python_version: {{ python_version }} artifacts: - - pyarrow-{no_rc_version}-cp38-cp38-win_amd64.whl + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl - wheel-windows-cp39: - ci: github - template: python-wheels/github.windows.yml - params: - python_version: 3.9 - artifacts: - - pyarrow-{no_rc_version}-cp39-cp39-win_amd64.whl +{% endfor %} - ############################ Python sdist #################################### +{############################ Python sdist ####################################} python-sdist: ci: github @@ -1259,7 +1082,7 @@ tasks: - plasma-store-server-debuginfo-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - plasma-store-server-{no_rc_version}-1.el8.[a-z0-9_]+.rpm - centos-8-aarch64: + centos-8-arm64: ci: travis template: linux-packages/travis.linux.arm64.yml params: @@ -1360,198 +1183,49 @@ tasks: ########################### Release verification ############################ - verify-rc-binaries-binary: +{% for target in ["binary", "yum", "apt"] %} + verify-rc-binaries-{{ target }}: ci: github template: verify-rc/github.linux.yml params: env: TEST_DEFAULT: 0 - TEST_BINARY: 1 + TEST_{{ target|upper }}: 1 artifact: "binaries" +{% endfor %} - verify-rc-binaries-apt: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_APT: 1 - artifact: "binaries" - - verify-rc-binaries-yum: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_YUM: 1 - artifact: "binaries" +{% for platform in ["linux", "macos"] %} - verify-rc-wheels-linux: + verify-rc-wheels-{{ platform }}: ci: github - template: verify-rc/github.linux.yml + template: verify-rc/github.{{ platform }}.yml params: env: TEST_DEFAULT: 0 artifact: "wheels" - verify-rc-wheels-macos: - ci: github - template: verify-rc/github.osx.yml - params: - env: - TEST_DEFAULT: 0 - artifact: "wheels" - - verify-rc-source-macos-java: - ci: github - template: verify-rc/github.osx.yml - params: - env: - TEST_DEFAULT: 0 - TEST_JAVA: 1 - artifact: "source" - - verify-rc-source-macos-csharp: - ci: github - template: verify-rc/github.osx.yml - params: - env: - TEST_DEFAULT: 0 - TEST_CSHARP: 1 - artifact: "source" - - verify-rc-source-macos-ruby: - ci: github - template: verify-rc/github.osx.yml - params: - env: - TEST_DEFAULT: 0 - TEST_RUBY: 1 - artifact: "source" - - verify-rc-source-macos-python: - ci: github - template: verify-rc/github.osx.yml - params: - env: - TEST_DEFAULT: 0 - TEST_PYTHON: 1 - # https://stackoverflow.com/questions/56083725/macos-build-issues-lstdc-not-found-while-building-python-package - MACOSX_DEPLOYMENT_TARGET: "10.15" - artifact: "source" - - verify-rc-source-macos-js: - ci: github - template: verify-rc/github.osx.yml - params: - env: - INSTALL_NODE: 0 - TEST_DEFAULT: 0 - TEST_JS: 1 - artifact: "source" - - verify-rc-source-macos-go: - ci: github - template: verify-rc/github.osx.yml - params: - env: - TEST_DEFAULT: 0 - TEST_GO: 1 - artifact: "source" +{% for target in ["csharp", + "go", + "integration" + "java", + "js", + "python", + "ruby", + "rust"] %} - verify-rc-source-macos-rust: + verify-rc-source-{{ platform }}-{{ target }}: ci: github - template: verify-rc/github.osx.yml - params: - env: - TEST_DEFAULT: 0 - TEST_RUST: 1 - artifact: "source" - - verify-rc-source-macos-integration: - ci: github - template: verify-rc/github.osx.yml + template: verify-rc/github.{{ platform }}.yml params: env: INSTALL_NODE: 0 TEST_DEFAULT: 0 - TEST_INTEGRATION: 1 - artifact: "source" - - verify-rc-source-linux-java: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_JAVA: 1 - artifact: "source" - - verify-rc-source-linux-csharp: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_CSHARP: 1 - artifact: "source" - - verify-rc-source-linux-ruby: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_RUBY: 1 + TEST_{{ target|upper }}: 1 artifact: "source" - verify-rc-source-linux-python: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_PYTHON: 1 - artifact: "source" +{% endfor %} - verify-rc-source-linux-js: - ci: github - template: verify-rc/github.linux.yml - params: - env: - INSTALL_NODE: 0 - TEST_DEFAULT: 0 - TEST_JS: 1 - artifact: "source" - - verify-rc-source-linux-go: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_GO: 1 - artifact: "source" - - verify-rc-source-linux-rust: - ci: github - template: verify-rc/github.linux.yml - params: - env: - TEST_DEFAULT: 0 - TEST_RUST: 1 - artifact: "source" - - verify-rc-source-linux-integration: - ci: github - template: verify-rc/github.linux.yml - params: - env: - INSTALL_NODE: 0 - TEST_DEFAULT: 0 - TEST_INTEGRATION: 1 - artifact: "source" +{% endfor %} verify-rc-source-windows: ci: github @@ -1565,19 +1239,20 @@ tasks: params: script: "verify-release-candidate-wheels.bat" - ############################## Docker tests ################################# - - test-conda-cpp: - ci: github - template: docker-tests/github.linux.yml - params: - run: conda-cpp +{############################## Docker tests #################################} - test-conda-cpp-valgrind: +{% for image in ["conda-cpp", + "conda-cpp-valgrind", + "debian-c-glib", + "ubuntu-c-glib", + "debian-ruby", + "ubuntu-ruby"] %} + test-{{ image }}: ci: github template: docker-tests/github.linux.yml params: - run: conda-cpp-valgrind + run: {{ image }} +{% endfor %} test-debian-10-cpp: ci: github @@ -1627,21 +1302,15 @@ tasks: UBUNTU: 20.04 run: ubuntu-cpp - test-ubuntu-20.04-cpp-14: - ci: github - template: docker-tests/github.linux.yml - params: - env: - UBUNTU: 20.04 - run: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD=14 ubuntu-cpp" - - test-ubuntu-20.04-cpp-17: +{% for cpp_standard in [14, 17] %} + test-ubuntu-20.04-cpp-{{ cpp_standard }}: ci: github template: docker-tests/github.linux.yml params: env: UBUNTU: 20.04 - run: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD=17 ubuntu-cpp" + run: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD={{ cpp_standard }} ubuntu-cpp" +{% endfor %} test-ubuntu-20.04-cpp-thread-sanitizer: ci: github @@ -1652,53 +1321,15 @@ tasks: UBUNTU: 20.04 run: ubuntu-cpp-thread-sanitizer - test-debian-c-glib: - ci: github - template: docker-tests/github.linux.yml - params: - run: debian-c-glib - - test-ubuntu-c-glib: - ci: github - template: docker-tests/github.linux.yml - params: - run: ubuntu-c-glib - - test-debian-ruby: - ci: github - template: docker-tests/github.linux.yml - params: - run: debian-ruby - - test-ubuntu-ruby: - ci: azure - template: docker-tests/azure.linux.yml - params: - run: ubuntu-ruby - - test-conda-python-3.6: +{% for python_version in ["3.6", "3.7", "3.8", "3.9"] %} + test-conda-python-{{ python_version }}: ci: github template: docker-tests/github.linux.yml params: env: - PYTHON: 3.6 - run: conda-python - - test-conda-python-3.7: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.7 - run: conda-python - - test-conda-python-3.8: - ci: github - template: docker-tests/github.linux.yml - params: - env: - PYTHON: 3.8 + PYTHON: {{ python_version }} run: conda-python +{% endfor %} test-conda-python-3.8-hypothesis: ci: github @@ -1740,7 +1371,7 @@ tasks: ci: github template: r/github.linux.cran.yml params: - MATRIX: "${{ matrix.r_image }}" + MATRIX: {{ "${{ matrix.r_image }}" }} test-r-version-compatibility: ci: github @@ -1750,7 +1381,7 @@ tasks: ci: github template: r/github.linux.versions.yml params: - MATRIX: "${{ matrix.r_version }}" + MATRIX: {{ "${{ matrix.r_version }}" }} test-r-install-local: ci: github diff --git a/dev/tasks/verify-rc/github.linux.yml b/dev/tasks/verify-rc/github.linux.yml index 4da78c80b04..8729426fd13 100644 --- a/dev/tasks/verify-rc/github.linux.yml +++ b/dev/tasks/verify-rc/github.linux.yml @@ -15,35 +15,24 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: verify: name: "Verify release candidate Ubuntu {{ artifact }}" runs-on: ubuntu-20.04 - {%- if env is defined %} + {% if env is defined %} env: - {%- for key, value in env.items() %} + {% for key, value in env.items() %} {{ key }}: {{ value }} - {%- endfor %} - {%- endif %} + {% endfor %} + {% endif %} + steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh + {{ macros.github_checkout_arrow()|indent }} + - name: Install System Dependencies run: | # TODO: don't require removing newer llvms diff --git a/dev/tasks/verify-rc/github.osx.yml b/dev/tasks/verify-rc/github.macos.yml similarity index 68% rename from dev/tasks/verify-rc/github.osx.yml rename to dev/tasks/verify-rc/github.macos.yml index a0f6fc4af4e..ab0c6563bdc 100644 --- a/dev/tasks/verify-rc/github.osx.yml +++ b/dev/tasks/verify-rc/github.macos.yml @@ -15,35 +15,24 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: verify: name: "Verify release candidate macOS {{ artifact }}" runs-on: macos-latest - {%- if env is defined %} + {% if env is defined %} env: - {%- for key, value in env.items() %} + {% for key, value in env.items() %} {{ key }}: {{ value }} - {%- endfor %} - {%- endif %} + {% endfor %} + {% endif %} + steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh + {{ macros.github_checkout_arrow()|indent }} + - name: Install System Dependencies shell: bash run: | diff --git a/dev/tasks/verify-rc/github.win.yml b/dev/tasks/verify-rc/github.win.yml index 7a96f89de51..5406327e874 100644 --- a/dev/tasks/verify-rc/github.win.yml +++ b/dev/tasks/verify-rc/github.win.yml @@ -15,35 +15,24 @@ # specific language governing permissions and limitations # under the License. -# NOTE: must set "Crossbow" as name to have the badge links working in the -# github comment reports! -name: Crossbow +{% import 'macros.jinja' as macros with context %} -on: - push: - branches: - - "*-github-*" +{{ macros.github_header() }} jobs: verify: name: "Verify release candidate Windows source" runs-on: windows-2016 - {%- if env is defined %} + {% if env is defined %} env: - {%- for key, value in env.items() %} + {% for key, value in env.items() %} {{ key }}: {{ value }} - {%- endfor %} - {%- endif %} + {% endfor %} + {% endif %} + steps: - - name: Checkout Arrow - run: | - git clone --no-checkout {{ arrow.remote }} arrow - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} - git -C arrow checkout FETCH_HEAD - git -C arrow submodule update --init --recursive - - name: Fetch Submodules and Tags - shell: bash - run: cd arrow && ci/scripts/util_checkout.sh + {{ macros.github_checkout_arrow()|indent }} + - uses: conda-incubator/setup-miniconda@v2 - name: Install System Dependencies run: | diff --git a/docs/source/developers/crossbow.rst b/docs/source/developers/crossbow.rst index d0ad05cd39b..7d5a3caa683 100644 --- a/docs/source/developers/crossbow.rst +++ b/docs/source/developers/crossbow.rst @@ -125,30 +125,17 @@ Install Miniconda is preferred, see installation instructions: https://conda.io/docs/user-guide/install/index.html -8. Install the python dependencies for the script: +8. Install the archery toolset containing crossbow itself: .. code:: bash - conda install -c conda-forge -y --file arrow/ci/conda_env_crossbow.txt - - .. code:: bash - - # pygit2 requires libgit2: http://www.pygit2.org/install.html - pip install \ - jinja2 \ - pygit2 \ - click \ - ruamel.yaml \ - setuptools_scm \ - github3.py \ - toolz \ - jira + pip install -e arrow/dev/archery[crossbow] 9. Try running it: .. code:: bash - $ python crossbow.py --help + $ archery crossbow --help Usage ----- @@ -165,8 +152,8 @@ The script does the following: $ git clone https://github.com/kszucs/crossbow $ cd arrow/dev/tasks - $ python crossbow.py submit --help # show the available options - $ python crossbow.py submit conda-win conda-linux conda-osx + $ archery crossbow submit --help # show the available options + $ archery crossbow submit conda-win conda-linux conda-osx 2. Gets the HEAD commit of the currently checked out branch and generates the version number based on `setuptools_scm`_. So to build @@ -175,7 +162,7 @@ The script does the following: .. code:: bash git checkout ARROW- - python dev/tasks/crossbow.py submit --dry-run conda-linux conda-osx + archery crossbow submit --dry-run conda-linux conda-osx .. @@ -201,14 +188,14 @@ by the ``submit`` command. .. code:: bash - python crossbow.py status + archery crossbow status Download the build artifacts ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code:: bash - python crossbow.py artifacts + archery crossbow artifacts Examples ~~~~~~~~ @@ -220,7 +207,7 @@ Run multiple builds: .. code:: bash - $ python crossbow.py submit debian-stretch conda-linux-gcc-py37-r40 + $ archery crossbow submit debian-stretch conda-linux-gcc-py37-r40 Repository: https://github.com/kszucs/arrow@tasks Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 Version: 0.9.1.dev48+g810a7188.d20180414 @@ -232,35 +219,34 @@ Just render without applying or committing the changes: .. code:: bash - $ python crossbow.py submit --dry-run task_name + $ archery crossbow submit --dry-run task_name Run only ``conda`` package builds and a Linux one: .. code:: bash - $ python crossbow.py submit --group conda centos-7 + $ archery crossbow submit --group conda centos-7 Run ``wheel`` builds: .. code:: bash - $ python crossbow.py submit --group wheel + $ archery crossbow submit --group wheel There are multiple task groups in the ``tasks.yml`` like docker, integration and cpp-python for running docker based tests. -``python crossbow.py submit`` supports multiple options and arguments, for more +``archery crossbow submit`` supports multiple options and arguments, for more see its help page: .. code:: bash - $ python crossbow.py submit --help + $ archery crossbow submit --help .. _conda-forge packages: conda-recipes .. _Wheels: python-wheels .. _Linux packages: linux-packages -.. _Crossbow.py: crossbow.py .. _Create the queue repository: https://help.github.com/articles/creating-a-new-repository .. _TravisCI: https://travis-ci.org/getting_started .. _Appveyor: https://www.appveyor.com/docs/