diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d72b126ef34..e70eaceaf41 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: hooks: - id: flake8 name: Python Format - files: ^(python|crossbow|integration)/ + files: ^(python|dev|integration)/ types: - file - python diff --git a/dev/archery/archery/bot.py b/dev/archery/archery/bot.py index b4ea24c1f3b..baa5210130d 100644 --- a/dev/archery/archery/bot.py +++ b/dev/archery/archery/bot.py @@ -122,7 +122,12 @@ class CrossbowCommentFormatter: title='Appveyor', url='https://ci.appveyor.com/project/{repo}/history', badge='https://img.shields.io/appveyor/ci/{repo}/{branch}.svg' - ) + ), + 'drone': _markdown_badge.format( + title='Drone', + url='https://cloud.drone.io/{repo}', + badge='https://img.shields.io/drone/build/{repo}/{branch}.svg' + ), } def __init__(self, crossbow_repo): diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 7be499fb576..454f9f20e39 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -67,6 +67,7 @@ cmake \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DCMAKE_RANLIB=${RANLIB} \ -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ + -DCMAKE_UNITY_BUILD=ON \ -GNinja \ ${EXTRA_CMAKE_ARGS} \ .. diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index cc256b6ad87..ba7d62921eb 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -87,7 +87,6 @@ outputs: run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - aws-sdk-cpp - - boost-cpp - brotli - c-ares - gflags @@ -193,9 +192,7 @@ outputs: run: - {{ pin_subpackage('arrow-cpp', exact=True) }} - {{ pin_compatible('numpy', lower_bound='1.16') }} - - boost-cpp - glog - - pandas # empty parquet-cpp metapackage, force old versions to be uninstalled - parquet-cpp 1.5.1.* - python diff --git a/dev/tasks/conda-recipes/drone-steps.sh b/dev/tasks/conda-recipes/drone-steps.sh new file mode 100755 index 00000000000..da72cffacf4 --- /dev/null +++ b/dev/tasks/conda-recipes/drone-steps.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -ex + +OUTPUT_DIR=$1 +QUEUE_REMOTE_URL=$2 +TASK_BRANCH=$3 +TASK_TAG=$4 +UPLOAD_TO_ANACONDA=$5 + +conda install -y mamba +$FEEDSTOCK_ROOT/build_steps.sh ${OUTPUT_DIR} + +# Upload as Github release +mamba install -y click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz anaconda-client shyaml -c conda-forge +pushd $DRONE_WORKSPACE +python arrow/dev/tasks/crossbow.py \ + --queue-path . \ + --queue-remote ${QUEUE_REMOTE_URL} \ + upload-artifacts \ + --sha ${TASK_BRANCH} \ + --tag ${TASK_TAG} \ + --pattern "${OUTPUT_DIR}/linux-aarch64/*.tar.bz2" + +if [[ "${UPLOAD_TO_ANACONDA}" == "1" ]]; then + anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force build_artifacts/linux-aarch64/*.tar.bz2 +fi diff --git a/dev/tasks/conda-recipes/drone.yml b/dev/tasks/conda-recipes/drone.yml new file mode 100644 index 00000000000..a461c79b9d4 --- /dev/null +++ b/dev/tasks/conda-recipes/drone.yml @@ -0,0 +1,43 @@ +--- +kind: pipeline +name: {{ config }} + +platform: + os: linux + arch: arm64 + +# Omit double builds with crossbow +trigger: + event: + - push + +steps: +- name: Install and build + image: condaforge/linux-anvil-aarch64 + environment: + CONFIG: {{ config }} + UPLOAD_PACKAGES: False + ARROW_VERSION: {{ arrow.no_rc_version }} + PLATFORM: linux-aarch64 + BINSTAR_TOKEN: + from_secret: BINSTAR_TOKEN + FEEDSTOCK_TOKEN: + from_secret: FEEDSTOCK_TOKEN + STAGING_BINSTAR_TOKEN: + from_secret: STAGING_BINSTAR_TOKEN + CROSSBOW_GITHUB_TOKEN: + from_secret: CROSSBOW_GITHUB_TOKEN + CROSSBOW_ANACONDA_TOKEN: + from_secret: CROSSBOW_ANACONDA_TOKEN + commands: + - export RECIPE_ROOT="$FEEDSTOCK_ROOT/arrow-cpp" + - export CI=drone + - export GIT_BRANCH="{{ arrow.branch }}" + - export FEEDSTOCK_NAME=arrow-cpp + - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE/arrow/dev/tasks/conda-recipes" + - sed -i '$ichown -R conda:conda "$FEEDSTOCK_ROOT"' /opt/docker/bin/entrypoint + - yum install -y git + - git clone --no-checkout {{ arrow.remote }} arrow + - pushd arrow && git fetch -t {{ arrow.remote }} {{ arrow.branch }} && git checkout FETCH_HEAD && git submodule update --init --recursive && popd + - mkdir -p $(pwd)/build_artifacts && chmod a+rwx $(pwd)/build_artifacts + - /opt/docker/bin/entrypoint $FEEDSTOCK_ROOT/drone-steps.sh $(pwd)/build_artifacts {{ queue_remote_url }} {{ task.branch }} {{ task.tag }} {% if arrow.branch == 'master' %}1{% else %}0{% endif %} diff --git a/dev/tasks/crossbow.py b/dev/tasks/crossbow.py index cb14de6415b..5981d56613e 100755 --- a/dev/tasks/crossbow.py +++ b/dev/tasks/crossbow.py @@ -691,7 +691,14 @@ class Task(Serializable): """ def __init__(self, ci, template, artifacts=None, params=None): - assert ci in {'circle', 'travis', 'appveyor', 'azure', 'github'} + assert ci in { + 'circle', + 'travis', + 'appveyor', + 'azure', + 'github', + 'drone', + } self.ci = ci self.template = template self.artifacts = artifacts or [] @@ -733,6 +740,7 @@ def filename(self): 'appveyor': 'appveyor.yml', 'azure': 'azure-pipelines.yml', 'github': '.github/workflows/crossbow.yml', + 'drone': '.drone.yml', } return config_files[self.ci] diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a064a420a4e..76ce517cdb0 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -187,6 +187,33 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 + conda-linux-gcc-py36-aarch64: + ci: drone + template: conda-recipes/drone.yml + params: + config: linux_aarch64_python3.6.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-linux-gcc-py37-aarch64: + ci: drone + template: conda-recipes/drone.yml + params: + config: linux_aarch64_python3.7.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-linux-gcc-py38-aarch64: + ci: drone + template: conda-recipes/drone.yml + params: + config: linux_aarch64_python3.8.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Conda OSX #################################### conda-osx-clang-py36: