diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5f29bdaa15c..9ac59e97df2 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -343,6 +343,9 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARROW_CXXFLAGS}") # For any C code, use the same flags. set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS}") +# Remove --std=c++11 to avoid errors from C compilers +string(REPLACE "-std=c++11" "" CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) + # Add C++-only flags, like -std=c++11 set(CMAKE_CXX_FLAGS "${CXX_ONLY_FLAGS} ${CMAKE_CXX_FLAGS}") diff --git a/dev/tasks/README.md b/dev/tasks/README.md index f1f0128f03b..35140fa44b3 100644 --- a/dev/tasks/README.md +++ b/dev/tasks/README.md @@ -80,12 +80,12 @@ submission. The tasks are defined in `tasks.yml` 6. Install the python dependencies for the script: ```bash - conda install -y jinja2 pygit2 click pyyaml + conda install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py ``` ```bash # pygit2 requires libgit2: http://www.pygit2.org/install.html - pip install -y jinja2 pygit2 click pyyaml + pip install -y jinja2 pygit2 click pyyaml setuptools_scm github3.py ``` 7. Try running it: @@ -106,7 +106,7 @@ The script does the following: $ git clone https://github.com/kszucs/crossbow $ cd arrow/dev/tasks - $ python crossbow.py + $ python crossbow.py submit conda-win conda-linux conda-osx ``` 2. Gets the HEAD commit of the currently checked out branch and generates @@ -115,7 +115,7 @@ The script does the following: ```bash git checkout ARROW- - python dev/tasks/crossbow.py --dry-run + python dev/tasks/crossbow.py submit --dry-run conda-linux conda-osx ``` > Note that the arrow branch must be pushed beforehand, because the script @@ -123,86 +123,57 @@ The script does the following: 3. Reads and renders the required build configurations with the parameters substituted. -2. Create a commit per build configuration to its own branch. For example - to build `travis-linux-conda.yml` it will place a commit to the tip of - `crossbow@travis-linux-conda` branch. +2. Create a branch per task, prefixed with the job id. For example + to build conda recipes on linux it will create a new branch: + `crossbow@build--conda-linux`. 3. Pushes the modified branches to GitHub which triggers the builds. For authentication it uses github oauth tokens described in the install section. -### Examples - -The script accepts a pattern as a first argument to narrow the build scope: - -Run all builds: +### Query the build status ```bash -$ python crossbow.py -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 -Version: 0.9.1.dev48+g810a7188.d20180414 -Pushed branches: - - travis-osx-wheel - - travis-linux-packages - - travis-linux-wheel - - appveyor-win-wheel - - appveyor-win-conda - - travis-linux-conda - - travis-osx-conda +python crossbow.py status ``` -Just render without applying or committing the changes: +### Download the build artifacts ```bash -$ python crossbow.py --dry-run +python crossbow.py artifacts ``` -Run only `conda` package builds but on all platforms: +### Examples + +The script accepts a pattern as a first argument to narrow the build scope: + +Run multiple builds: ```bash -$ python crossbow.py conda +$ python crossbow.py submit linux-packages conda-linux wheel-win Repository: https://github.com/kszucs/arrow@tasks Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 Version: 0.9.1.dev48+g810a7188.d20180414 Pushed branches: - - appveyor-win-conda - - travis-linux-conda - - travis-osx-conda + - linux-packages + - conda-linux + - wheel-win ``` -Run `wheel` builds: +Just render without applying or committing the changes: ```bash -$ python crossbow.py wheel -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 -Version: 0.9.1.dev48+g810a7188.d20180414 -Pushed branches: - - travis-osx-wheel - - travis-linux-wheel - - appveyor-win-wheel +$ python crossbow.py submit --dry-run task_name ``` -Run `osx` builds: +Run only `conda` package builds but on all platforms: ```bash -$ python crossbow.py osx -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: cad1df2c7f650ad3434319bbbefed0d4abe45e4a -Version: 0.9.1.dev130+gcad1df2c.d20180414 -Pushed branches: - - travis-osx-wheel - - travis-osx-conda +$ python crossbow.py submit conda-win conda-osx conda-linux ``` -Run only `linux-conda` package build: +Run `wheel` builds: ```bash -$ python crossbow.py linux-conda -Repository: https://github.com/kszucs/arrow@tasks -Commit SHA: 810a718836bb3a8cefc053055600bdcc440e6702 -Version: 0.9.1.dev48+g810a7188.d20180414 -Pushed branches: - - travis-linux-conda +$ python crossbow.py submit wheel-osx wheel-linux wheel-win ``` diff --git a/dev/tasks/conda-recipes/appveyor.yml b/dev/tasks/conda-recipes/appveyor.yml index 10ef7044ceb..3d3f3305e86 100644 --- a/dev/tasks/conda-recipes/appveyor.yml +++ b/dev/tasks/conda-recipes/appveyor.yml @@ -23,7 +23,6 @@ environment: - TARGET_ARCH: x64 CONDA_PY: 36 CONDA_INSTALL_LOCN: C:\\Miniconda36-x64 - ARROW_SRC: C:\apache-arrow ARROW_VERSION: {{ ARROW_VERSION }} install: @@ -45,11 +44,28 @@ install: build: off test_script: - - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} %ARROW_SRC% || exit /B - - git checkout {{ ARROW_SHA }} || exit /B - - pushd %ARROW_SRC%\dev\tasks\conda-recipes - - conda.exe build parquet-cpp arrow-cpp pyarrow + - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow || exit /B + - git -C arrow checkout {{ ARROW_SHA }} || exit /B + - pushd arrow\dev\tasks\conda-recipes + - conda.exe build --output-folder . parquet-cpp arrow-cpp pyarrow + - pushd win-64 + - for %%f in (*.tar.bz2) do ( + set %%g=%%~nf + ren "%%f" "%%~ng-win-64.tar.bz2" + ) +artifacts: + # this must be relative and child of the build C:\projects\crossbow directory + - path: arrow\dev\tasks\conda-recipes\win-64\*.tar.bz2 + +deploy: + release: {{ BUILD_TAG }} + provider: GitHub + auth_token: "%CROSSBOW_GITHUB_TOKEN%" + artifact: /.*\.tar\.bz2/ + draft: false + prerelease: false + force_update: true notifications: - provider: Email diff --git a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml index 0f5a619010a..e7be2a1c7d7 100644 --- a/dev/tasks/conda-recipes/parquet-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/parquet-cpp/meta.yaml @@ -36,9 +36,8 @@ source: build: number: 0 - skip: true # [win and not (py35 and win64)] features: - - vc14 # [win and py35] + - vc14 # [win] requirements: build: @@ -46,16 +45,12 @@ requirements: - boost-cpp 1.66.0 - cmake - thrift-cpp >=0.11 - - python # [win] - arrow-cpp {{ ARROW_VERSION }} run: - arrow-cpp {{ ARROW_VERSION }} test: - requires: - - python {{ environ['PY_VER'] + '*' }} # [win] - commands: - test -f $PREFIX/lib/libparquet.so # [linux] - test -f $PREFIX/lib/libparquet.dylib # [osx] diff --git a/dev/tasks/conda-recipes/travis.linux.yml b/dev/tasks/conda-recipes/travis.linux.yml index c5266810c6d..84ca83b27a0 100644 --- a/dev/tasks/conda-recipes/travis.linux.yml +++ b/dev/tasks/conda-recipes/travis.linux.yml @@ -26,6 +26,7 @@ env: - CONDA_PY=35 - CONDA_PY=36 global: + - TRAVIS_TAG={{ BUILD_TAG }} - ARROW_VERSION={{ ARROW_VERSION }} install: @@ -55,7 +56,22 @@ script: - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow - git -C arrow checkout {{ ARROW_SHA }} - pushd arrow/dev/tasks/conda-recipes - - conda build parquet-cpp arrow-cpp pyarrow + - conda build --output-folder . parquet-cpp arrow-cpp pyarrow + - | + pushd linux-64 + for file in *.tar.bz2; do + mv "$file" "$(basename "$file" .tar.bz2)-linux-64.tar.bz2" + done + popd + +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/linux-64/*.tar.bz2 + skip_cleanup: true + on: + tags: true notifications: email: diff --git a/dev/tasks/conda-recipes/travis.osx.yml b/dev/tasks/conda-recipes/travis.osx.yml index 8a38d333489..31c1c244b5f 100644 --- a/dev/tasks/conda-recipes/travis.osx.yml +++ b/dev/tasks/conda-recipes/travis.osx.yml @@ -26,6 +26,7 @@ env: - CONDA_PY=35 - CONDA_PY=36 global: + - TRAVIS_TAG={{ BUILD_TAG }} - ARROW_VERSION={{ ARROW_VERSION }} before_install: @@ -64,7 +65,22 @@ script: - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow - git -C arrow checkout {{ ARROW_SHA }} - pushd arrow/dev/tasks/conda-recipes - - conda build parquet-cpp arrow-cpp pyarrow + - conda build --output-folder . parquet-cpp arrow-cpp pyarrow + - | + pushd osx-64 + for file in *.tar.bz2; do + mv "$file" "$(basename "$file" .tar.bz2)-osx-64.tar.bz2" + done + popd + +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: $TRAVIS_BUILD_DIR/arrow/dev/tasks/conda-recipes/osx-64/*.tar.bz2 + skip_cleanup: true + on: + tags: true notifications: email: diff --git a/dev/tasks/config-nightlies/travis.linux.yml b/dev/tasks/config-nightlies/travis.linux.yml new file mode 100644 index 00000000000..bf2774b2ed5 --- /dev/null +++ b/dev/tasks/config-nightlies/travis.linux.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +branches: + # don't attempt to build branches intented for windows builds + except: + - /.*win.*/ + +os: linux +dist: trusty +language: generic + +before_install: + # Install Miniconda. + - echo `pwd` + - | + echo "" + echo "Installing a fresh version of Miniconda." + MINICONDA_URL="https://repo.continuum.io/miniconda" + MINICONDA_FILE="Miniconda3-latest-Linux-x86_64.sh" + curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" + bash $MINICONDA_FILE -b + + # Configure conda. + - | + echo "" + echo "Configuring conda." + source /home/travis/miniconda3/bin/activate root + conda config --remove channels defaults + conda config --add channels defaults + conda config --add channels conda-forge + conda config --set show_channel_urls true + +install: + - conda install -y -q jinja2 pygit2 click pyyaml setuptools_scm + +script: + # fetch all branches of crossbow + - git config remote.origin.fetch +refs/heads/*:refs/remotes/origin/* + + # clone arrow with crossbow tool + - pushd .. + - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} + + # submit packaging tasks + - | + python arrow/dev/tasks/crossbow.py \ + conda-linux \ + conda-win \ + conda-osx \ + wheel-linux \ + wheel-win \ + wheel-osx \ + linux-packages diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index fd4f732ea82..a2a29ebad48 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -17,12 +17,13 @@ # specific language governing permissions and limitations # under the License. +import os import re -import sys import yaml import time import click import pygit2 +import github3 import logging from enum import Enum @@ -30,15 +31,6 @@ from textwrap import dedent from jinja2 import Template from setuptools_scm import get_version -from setuptools_scm.version import simplified_semver_version, meta - - -logging.basicConfig( - level=logging.INFO, - format="[%(asctime)s] %(levelname)s Crossbow %(message)s", - datefmt="%H:%M:%S", - stream=click.get_text_stream('stdout') -) class GitRemoteCallbacks(pygit2.RemoteCallbacks): @@ -70,134 +62,81 @@ def credentials(self, url, username_from_url, allowed_types): return None -class Platform(Enum): - # in alphabetical order - LINUX = 0 - OSX = 1 - WIN = 2 - - @property - def ci(self): - if self is self.WIN: - return 'appveyor' - else: - return 'travis' - - @property - def filename(self): - if self.ci == 'appveyor': - return 'appveyor.yml' - else: - return '.travis.yml' - +class Repo(object): -class Target(object): - - def __init__(self, repo_path, template_directory=None): + def __init__(self, repo_path): self.path = Path(repo_path).absolute() - - # relative to repository's path - if template_directory is None: - self.templates = self.path - else: - self.templates = self.path / template_directory - - # initialize a repo object to interact with arrow's git data self.repo = pygit2.Repository(str(self.path)) - msg = dedent(''' - Repository: {remote}@{branch} - Commit SHA: {sha} - Version: {version} + def __str__(self): + tpl = dedent(''' + Repo: {remote}@{branch} + Commit: {head} ''') - logging.info(msg.format( - remote=self.current_remote.url, - branch=self.current_branch.branch_name, - sha=self.sha, - version=self.version - )) + return tpl.format( + remote=self.remote.url, + branch=self.branch.branch_name, + head=self.head + ) + + def fetch(self): + self.origin.fetch() @property - def sha(self): + def head(self): """Currently checked out commit's sha""" return self.repo.head.target @property - def version(self): - """Generate version number based on version control history""" - # TODO(kszucs) use self.repo.describe() instead - return get_version(self.path) + def branch(self): + """Currently checked out branch""" + reference = self.repo.head.shorthand + return self.repo.branches[reference] @property - def current_remote(self): - remote_name = self.current_branch.upstream.remote_name + def remote(self): + """Currently checked out branch's remote counterpart""" + remote_name = self.branch.upstream.remote_name return self.repo.remotes[remote_name] @property - def current_branch(self): - reference = self.repo.head.shorthand - return self.repo.branches[reference] + def origin(self): + return self.repo.remotes['origin'] @property - def description(self): - return '[BUILD] {} of {}@{}'.format(self.version, - self.current_remote.url, - self.current_branch.branch_name) - - -class Build(object): - - def __init__(self, target, name, platform, template, **params): - assert isinstance(target, Target) - assert isinstance(platform, Platform) - - self.name = name - self.target = target - self.platform = platform - self.template = template - self.params = params - - def render(self): - path = Path(self.template) - template = Template(path.read_text()) - return template.render(**self.params) - - def config_files(self): - return {self.platform.filename: self.render()} + def email(self): + return next(self.repo.config.get_multivar('user.email')) @property - def branch(self): - return self.name + def signature(self): + name = next(self.repo.config.get_multivar('user.name')) + return pygit2.Signature(name, self.email, int(time.time())) - @property - def description(self): - return self.target.description + def parse_user_repo(self): + m = re.match('.*\/([^\/]+)\/([^\/\.]+)(\.git)?$', self.remote.url) + user, repo = m.group(1), m.group(2) + return user, repo -class Queue(object): +class Queue(Repo): def __init__(self, repo_path): - self.path = Path(repo_path).absolute() - self.repo = pygit2.Repository(str(self.path)) - self.updated_branches = [] - - def _get_parent_commit(self): - """Currently this always returns the HEAD of master""" - master = self.repo.branches['master'] - return self.repo[master.target] - - def _get_or_create_branch(self, name): - try: - return self.repo.branches[name] - except KeyError: - parent = self._get_parent_commit() - return self.repo.branches.create(name, parent) - - def _create_tree(self, files): - parent = self._get_parent_commit() + super(Queue, self).__init__(repo_path) + self._updated_refs = [] + + def next_job_id(self, prefix): + """Auto increments the branch's identifier based on the prefix""" + pattern = re.compile(prefix + '-(\d+)') + matches = list(filter(None, map(pattern.match, self.repo.branches))) + if matches: + latest = max(int(m.group(1)) for m in matches) + else: + latest = 0 + return '{}-{}'.format(prefix, latest + 1) - # creating the tree we are going to push based on master's tree - builder = self.repo.TreeBuilder(parent.tree) + def _create_branch(self, branch_name, files, parents=[], message=''): + # 1. create tree + builder = self.repo.TreeBuilder() for filename, content in files.items(): # insert the file and creating the new filetree @@ -205,119 +144,293 @@ def _create_tree(self, files): builder.insert(filename, blob_id, pygit2.GIT_FILEMODE_BLOB) tree_id = builder.write() - return tree_id - def put(self, build): - assert isinstance(build, Build) + # 2. create commit with the tree created above + author = committer = self.signature + commit_id = self.repo.create_commit(None, author, committer, message, + tree_id, parents) + commit = self.repo[commit_id] - branch = self._get_or_create_branch(build.branch) - tree_id = self._create_tree(build.config_files()) + # 3. create branch pointing to the previously created commit + branch = self.repo.create_branch(branch_name, commit) + # append to the pushable references + self._updated_refs.append('refs/heads/{}'.format(branch_name)) - # creating the new commit - timestamp = int(time.time()) + return branch - name = next(self.repo.config.get_multivar('user.name')) - email = next(self.repo.config.get_multivar('user.email')) + def _create_tag(self, tag_name, commit_id, message=''): + tag_id = self.repo.create_tag(tag_name, commit_id, + pygit2.GIT_OBJ_COMMIT, self.signature, + message) + + # append to the pushable references + self._updated_refs.append('refs/tags/{}'.format(tag_name)) + + return self.repo[tag_id] + + def put(self, job, prefix='build'): + assert isinstance(job, Job) + assert job.branch is not None - author = pygit2.Signature('crossbow', 'mailing@list.com', - int(timestamp)) - committer = pygit2.Signature(name, email, int(timestamp)) - message = build.description + # create tasks' branches + for task_name, task in job.tasks.items(): + branch = self._create_branch(task.branch, files=task.files()) + task.commit = str(branch.target) - reference = 'refs/heads/{}'.format(branch.branch_name) - commit_id = self.repo.create_commit(reference, author, committer, - message, tree_id, [branch.target]) - logging.info('{} created on {}'.format( - commit_id, branch.branch_name)) + # create job's branch + branch = self._create_branch(job.branch, files=job.files()) + self._create_tag(job.branch, branch.target) - self.updated_branches.append(branch) + return branch def push(self, token): callbacks = GitRemoteCallbacks(token) + self.origin.push(self._updated_refs, callbacks=callbacks) + self.updated_refs = [] - remote = self.repo.remotes['origin'] - refs = [branch.name for branch in self.updated_branches] - shorthands = [b.shorthand for b in self.updated_branches] - remote.push(refs, callbacks=callbacks) - self.updated_branches = [] +class Platform(Enum): + # in alphabetical order + LINUX = 0 + OSX = 1 + WIN = 2 + + @property + def ci(self): + if self is self.WIN: + return 'appveyor' + else: + return 'travis' + + @property + def filename(self): + if self.ci == 'appveyor': + return 'appveyor.yml' + else: + return '.travis.yml' + + +class Task(object): + + def __init__(self, platform, template, commit=None, branch=None, **params): + assert isinstance(platform, Platform) + assert isinstance(template, Path) + self.platform = platform + self.template = template + self.branch = branch + self.commit = commit + self.params = params + + def to_dict(self): + return {'branch': self.branch, + 'commit': str(self.commit), + 'platform': self.platform.name, + 'template': str(self.template), + 'params': self.params} + + @classmethod + def from_dict(cls, data): + return Task(platform=Platform[data['platform'].upper()], + template=Path(data['template']), + commit=data.get('commit'), + branch=data.get('branch'), + **data.get('params', {})) + + def files(self): + template = Template(self.template.read_text()) + rendered = template.render(**self.params) + return {self.platform.filename: rendered} + + +class Job(object): + + def __init__(self, tasks, branch=None): + assert all(isinstance(task, Task) for task in tasks.values()) + self.branch = branch + self.tasks = tasks + + def to_dict(self): + tasks = {name: task.to_dict() for name, task in self.tasks.items()} + return {'branch': self.branch, + 'tasks': tasks} - logging.info('\n - '.join(['\nUpdated branches:'] + shorthands)) + @classmethod + def from_dict(cls, data): + tasks = {name: Task.from_dict(task) + for name, task in data['tasks'].items()} + return Job(tasks=tasks, branch=data.get('branch')) + + def files(self): + return {'job.yml': yaml.dump(self.to_dict(), default_flow_style=False)} # this should be the mailing list MESSAGE_EMAIL = 'szucs.krisztian@gmail.com' +CWD = Path(__file__).absolute() + +DEFAULT_CONFIG_PATH = CWD.parent / 'tasks.yml' +DEFAULT_ARROW_PATH = CWD.parents[2] +DEFAULT_QUEUE_PATH = CWD.parents[3] / 'crossbow' + + +@click.group() +def crossbow(): + pass -@click.command() -@click.argument('task-regex', required=False) -@click.option('--config', help='Task configuration yml. Defaults to tasks.yml') + +def github_token_validation_callback(ctx, param, value): + if value is None: + raise click.ClickException( + 'Could not determine GitHub token. Please set the ' + 'CROSSBOW_GITHUB_TOKEN environment variable to a ' + 'valid github access token or pass one to --github-token.' + ) + return value + + +github_token = click.option( + '--github-token', + default=None, + envvar='CROSSBOW_GITHUB_TOKEN', + help='OAuth token for Github authentication', + callback=github_token_validation_callback, +) + + +def config_path_validation_callback(ctx, param, value): + with Path(value).open() as fp: + config = yaml.load(fp) + task_names = ctx.params['task_names'] + valid_tasks = set(config['tasks'].keys()) + invalid_tasks = {task for task in task_names if task not in valid_tasks} + if invalid_tasks: + raise click.ClickException( + 'Invalid task(s) {!r}. Must be one of {!r}'.format( + invalid_tasks, + valid_tasks + ) + ) + return value + + +@crossbow.command() +@click.argument('task-names', nargs=-1, required=True) +@click.option('--job-prefix', default='build', + help='Arbitrary prefix for branch names, e.g. nightly') +@click.option('--config-path', default=DEFAULT_CONFIG_PATH, + type=click.Path(exists=True), + callback=config_path_validation_callback, + help='Task configuration yml. Defaults to tasks.yml') @click.option('--dry-run/--push', default=False, help='Just display the rendered CI configurations without ' 'submitting them') -@click.option('--arrow-repo', default=None, +@click.option('--arrow-path', default=DEFAULT_ARROW_PATH, help='Arrow\'s repository path. Defaults to the repository of ' 'this script') -@click.option('--queue-repo', default=None, +@click.option('--queue-path', default=DEFAULT_QUEUE_PATH, help='The repository path used for scheduling the tasks. ' 'Defaults to crossbow directory placed next to arrow') -@click.option('--github-token', default=False, - help='Oauth token for Github authentication') -def build(task_regex, config, dry_run, arrow_repo, queue_repo, github_token): - if config is None: - config = Path(__file__).absolute().parent / 'tasks.yml' - else: - config = Path(config) - - if arrow_repo is None: - arrow_repo = Path(__file__).absolute().parents[2] - else: - arrow_repo = Path(arrow_repo) - - if queue_repo is None: - queue_repo = arrow_repo.parent / 'crossbow' - else: - queue_repo = Path(queue_repo) - - arrow = Target(arrow_repo, template_directory='cd') - queue = Queue(queue_repo) +@github_token +def submit(task_names, job_prefix, config_path, dry_run, arrow_path, + queue_path, github_token): + target = Repo(arrow_path) + queue = Queue(queue_path) + + logging.info(target) + logging.info(queue) + + queue.fetch() + + version = get_version(arrow_path, local_scheme=lambda v: '') + job_id = queue.next_job_id(prefix=job_prefix) variables = { # these should be renamed 'PLAT': 'x86_64', - 'EMAIL': MESSAGE_EMAIL, - 'BUILD_REF': arrow.sha, - 'ARROW_SHA': arrow.sha, - 'ARROW_REPO': arrow.current_remote.url, - 'ARROW_BRANCH': arrow.current_branch.branch_name, - 'ARROW_VERSION': arrow.version, - 'PYARROW_VERSION': arrow.version, + 'EMAIL': os.environ.get('CROSSBOW_EMAIL', target.email), + 'BUILD_TAG': job_id, + 'BUILD_REF': str(target.head), + 'ARROW_SHA': str(target.head), + 'ARROW_REPO': target.remote.url, + 'ARROW_BRANCH': target.branch.branch_name, + 'ARROW_VERSION': version, + 'PYARROW_VERSION': version, } - with config.open() as fp: - tasks = yaml.load(fp)['tasks'] + with Path(config_path).open() as fp: + config = yaml.load(fp) + + # create and filter tasks + tasks = {name: Task.from_dict(task) + for name, task in config['tasks'].items()} + tasks = {name: tasks[name] for name in task_names} - for task in tasks: - name = task['name'] - template = config.parent / task['template'] - platform = Platform[task['platform'].upper()] - params = task.get('params') or {} - params.update(variables) + for task_name, task in tasks.items(): + task.branch = '{}-{}'.format(job_id, task_name) + task.params.update(variables) - build = Build(arrow, name=name, platform=platform, template=template, - **params) + # create job + job = Job(tasks) + job.branch = job_id - # Regex pattern the task name is matched against - if task_regex is None or re.search(task_regex, build.name): - if dry_run: - logging.info('{}\n\n{}'.format(build.name, build.render())) - else: - queue.put(build) # create the commit + yaml_format = yaml.dump(job.to_dict(), default_flow_style=False) + click.echo(yaml_format.strip()) if not dry_run: - # push the changed branches + queue.put(job) queue.push(token=github_token) + click.echo('Pushed job identifier is: `{}`'.format(job_id)) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--queue-path', default=DEFAULT_QUEUE_PATH, + help='The repository path used for scheduling the tasks. ' + 'Defaults to crossbow directory placed next to arrow') +@github_token +def status(job_name, queue_path, github_token): + queue = Queue(queue_path) + username, reponame = queue.parse_user_repo() + + gh = github3.login(token=github_token) + repo = gh.repository(username, reponame) + content = repo.file_contents('job.yml', job_name) + + job = Job.from_dict(yaml.load(content.decoded)) + + tpl = '[{:>7}] {:<24} {:<40}' + header = tpl.format('status', 'branch', 'sha') + click.echo(header) + click.echo('-' * len(header)) + + for name, task in job.tasks.items(): + commit = repo.commit(task.commit) + status = commit.status() + + click.echo(tpl.format(status.state, task.branch, task.commit)) + + +@crossbow.command() +@click.argument('job-name', required=True) +@click.option('--target-dir', default=DEFAULT_ARROW_PATH, + help='Directory to download the build artifacts') +@click.option('--queue-path', default=DEFAULT_QUEUE_PATH, + help='The repository path used for scheduling the tasks. ' + 'Defaults to crossbow directory placed next to arrow') +@github_token +def artifacts(job_name, target_dir, queue_path, github_token): + queue = Queue(queue_path) + username, reponame = queue.parse_user_repo() + + gh = github3.login(token=github_token) + repo = gh.repository(username, reponame) + release = repo.release_from_tag(job_name) + + for asset in release.assets(): + click.echo('Downloading asset {} ...'.format(asset.name)) + asset.download(target_dir / asset.name) if __name__ == '__main__': - build(auto_envvar_prefix='CROSSBOW') + crossbow(auto_envvar_prefix='CROSSBOW') diff --git a/dev/tasks/linux-packages/travis.linux.yml b/dev/tasks/linux-packages/travis.linux.yml index afab39471c9..8cc637728a3 100644 --- a/dev/tasks/linux-packages/travis.linux.yml +++ b/dev/tasks/linux-packages/travis.linux.yml @@ -22,6 +22,7 @@ language: ruby env: global: + - TRAVIS_TAG={{ BUILD_TAG }} - PLAT={{ PLAT }} - BUILD_REF={{ BUILD_REF }} - PYARROW_VERSION={{ PYARROW_VERSION }} @@ -29,11 +30,11 @@ env: matrix: include: - script: - - (cd arrow/dev/tasks/linux-packages && travis_wait 40 rake apt:build APT_TARGETS=debian-stretch,ubuntu-trusty,ubuntu-xenial PARALLEL=yes DEBUG=no) + - (cd arrow/dev/tasks/linux-packages && travis_wait 40 && rake version:update && rake apt:build APT_TARGETS=debian-stretch,ubuntu-trusty,ubuntu-xenial PARALLEL=yes DEBUG=no) - script: - - (cd arrow/dev/tasks/linux-packages && travis_wait 40 rake apt:build APT_TARGETS=ubuntu-artful PARALLEL=yes DEBUG=no) + - (cd arrow/dev/tasks/linux-packages && travis_wait 40 && rake version:update && rake apt:build APT_TARGETS=ubuntu-artful PARALLEL=yes DEBUG=no) - script: - - (cd arrow/dev/tasks/linux-packages && rake yum:build PARALLEL=yes DEBUG=no) + - (cd arrow/dev/tasks/linux-packages && rake version:update && rake yum:build PARALLEL=yes DEBUG=no) before_install: - sudo apt update -y -qq @@ -62,6 +63,15 @@ before_script: - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} arrow - git -C arrow checkout {{ ARROW_SHA }} +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: /path/to/pachages/*.tar.gz # FIXME(kszucs) after the builds pass + skip_cleanup: true + on: + tags: true + notifications: email: - {{ EMAIL }} diff --git a/dev/tasks/python-wheels/appveyor.yml b/dev/tasks/python-wheels/appveyor.yml index 3bd1804832f..b3407fb178c 100644 --- a/dev/tasks/python-wheels/appveyor.yml +++ b/dev/tasks/python-wheels/appveyor.yml @@ -39,7 +39,6 @@ init: - set MINICONDA=C:\Miniconda35-x64 - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH% - build_script: - mkdir wheels - git clone -b {{ ARROW_BRANCH }} {{ ARROW_REPO }} %ARROW_SRC% || exit /B @@ -52,6 +51,14 @@ after_build: artifacts: - path: wheels\*.whl +deploy: + release: {{ BUILD_TAG }} + provider: GitHub + auth_token: "%CROSSBOW_GITHUB_TOKEN%" + artifact: /.*\.whl/ + draft: false + prerelease: false + notifications: - provider: Email to: diff --git a/dev/tasks/python-wheels/travis.linux.yml b/dev/tasks/python-wheels/travis.linux.yml index 8f2435cec4e..2685ad4bb49 100644 --- a/dev/tasks/python-wheels/travis.linux.yml +++ b/dev/tasks/python-wheels/travis.linux.yml @@ -16,6 +16,7 @@ env: global: + - TRAVIS_TAG={{ BUILD_TAG }} - PLAT={{ PLAT }} - BUILD_REF={{ BUILD_REF }} - PYARROW_VERSION={{ PYARROW_VERSION }} @@ -45,6 +46,15 @@ script: - sudo mv arrow/python/manylinux1/dist/* dist/ +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: dist/*.whl + skip_cleanup: true + on: + tags: true + notifications: email: - {{ EMAIL }} diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index a92e274fabb..f346297b5fb 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -14,12 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +language: generic + os: osx +osx_image: xcode8.3 + sudo: required -language: objective-c env: global: + - TRAVIS_TAG={{ BUILD_TAG }} - PLAT={{ PLAT }} - BUILD_REF={{ BUILD_REF }} - PYARROW_VERSION={{ PYARROW_VERSION }} @@ -69,13 +73,14 @@ install: - build_wheel arrow $PLAT - mv -v arrow/python/dist/* dist/ -script: - - echo "SCRIPT" - - pwd - -after_success: - - echo "After success" - - pwd +deploy: + provider: releases + api_key: $CROSSBOW_GITHUB_TOKEN + file_glob: true + file: dist/*.whl + skip_cleanup: true + on: + tags: true notifications: email: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 890c00edd3e..8aaf469f44e 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -16,36 +16,39 @@ # under the License. tasks: + # arbitrary_task_name: + # branch: defaults to name + # platform: osx|linux|win + # template: path of jinja2 templated yml + # params: optional extra parameters + # conda packages - - name: conda-linux + conda-linux: platform: linux template: conda-recipes/travis.linux.yml - params: - - name: conda-osx + + conda-osx: platform: osx template: conda-recipes/travis.osx.yml - params: - - name: conda-win + + conda-win: platform: win template: conda-recipes/appveyor.yml - params: # python wheels - - name: python-wheel-linux + wheel-linux: platform: linux template: python-wheels/travis.linux.yml - params: - - name: python-wheel-osx + + wheel-osx: platform: osx template: python-wheels/travis.osx.yml - params: - - name: python-wheel-win + + wheel-win: platform: win template: python-wheels/appveyor.yml - params: # linux packages - - name: linux-packages + linux-packages: platform: linux template: linux-packages/travis.linux.yml - params: