Arrow 8359

.pre-commit-config.yaml

@@ -55,7 +55,7 @@ repos:
     hooks:
      - id: flake8
        name: Python Format
-       files: ^(python|crossbow|integration)/
+       files: ^(python|dev|integration)/
        types:
          - file
          - python

dev/archery/archery/bot.py

@@ -122,7 +122,12 @@ class CrossbowCommentFormatter:
             title='Appveyor',
             url='https://ci.appveyor.com/project/{repo}/history',
             badge='https://img.shields.io/appveyor/ci/{repo}/{branch}.svg'
-        )
+        ),
+        'drone': _markdown_badge.format(
+            title='Drone',
+            url='https://cloud.drone.io/{repo}',
+            badge='https://img.shields.io/drone/build/{repo}/{branch}.svg'
+        ),
     }
 
     def __init__(self, crossbow_repo):

dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh

@@ -67,6 +67,7 @@ cmake \
     -DCMAKE_INSTALL_PREFIX=$PREFIX \
     -DCMAKE_RANLIB=${RANLIB} \
     -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \
+    -DCMAKE_UNITY_BUILD=ON \
     -GNinja \
     ${EXTRA_CMAKE_ARGS} \
     ..

dev/tasks/conda-recipes/arrow-cpp/meta.yaml

@@ -87,7 +87,6 @@ outputs:
       run:
         - {{ pin_compatible('numpy', lower_bound='1.16') }}
         - aws-sdk-cpp
-        - boost-cpp
         - brotli
         - c-ares
         - gflags
@@ -193,9 +192,7 @@ outputs:
       run:
         - {{ pin_subpackage('arrow-cpp', exact=True) }}
         - {{ pin_compatible('numpy', lower_bound='1.16') }}
-        - boost-cpp
         - glog
-        - pandas
         # empty parquet-cpp metapackage, force old versions to be uninstalled
         - parquet-cpp 1.5.1.*
         - python

dev/tasks/conda-recipes/drone-steps.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -ex
+
+OUTPUT_DIR=$1
+QUEUE_REMOTE_URL=$2
+TASK_BRANCH=$3
+TASK_TAG=$4
+UPLOAD_TO_ANACONDA=$5
+
+conda install -y mamba
+$FEEDSTOCK_ROOT/build_steps.sh ${OUTPUT_DIR}
+
+# Upload as Github release
+mamba install -y click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz anaconda-client shyaml -c conda-forge
+pushd $DRONE_WORKSPACE
+python arrow/dev/tasks/crossbow.py \
+  --queue-path . \
+  --queue-remote ${QUEUE_REMOTE_URL} \
+  upload-artifacts \
+  --sha ${TASK_BRANCH} \
+  --tag ${TASK_TAG} \
+  --pattern "${OUTPUT_DIR}/linux-aarch64/*.tar.bz2"
+
+if [[ "${UPLOAD_TO_ANACONDA}" == "1" ]]; then
+  anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force build_artifacts/linux-aarch64/*.tar.bz2
+fi

dev/tasks/conda-recipes/drone.yml

@@ -0,0 +1,43 @@
+---
+kind: pipeline
+name: {{ config }}
+
+platform:
+  os: linux
+  arch: arm64
+
+# Omit double builds with crossbow
+trigger:
+  event:
+    - push
+
+steps:
+- name: Install and build
+  image: condaforge/linux-anvil-aarch64
+  environment:
+    CONFIG: {{ config }}
+    UPLOAD_PACKAGES: False
+    ARROW_VERSION: {{ arrow.no_rc_version }}
+    PLATFORM: linux-aarch64
+    BINSTAR_TOKEN:
+      from_secret: BINSTAR_TOKEN
+    FEEDSTOCK_TOKEN:
+      from_secret: FEEDSTOCK_TOKEN
+    STAGING_BINSTAR_TOKEN:
+      from_secret: STAGING_BINSTAR_TOKEN
+    CROSSBOW_GITHUB_TOKEN:
+      from_secret: CROSSBOW_GITHUB_TOKEN
+    CROSSBOW_ANACONDA_TOKEN:
+      from_secret: CROSSBOW_ANACONDA_TOKEN
+  commands:
+    - export RECIPE_ROOT="$FEEDSTOCK_ROOT/arrow-cpp"
+    - export CI=drone
+    - export GIT_BRANCH="{{ arrow.branch }}"
+    - export FEEDSTOCK_NAME=arrow-cpp
+    - export FEEDSTOCK_ROOT="$DRONE_WORKSPACE/arrow/dev/tasks/conda-recipes"
+    - sed -i '$ichown -R conda:conda "$FEEDSTOCK_ROOT"' /opt/docker/bin/entrypoint
+    - yum install -y git
+    - git clone --no-checkout {{ arrow.remote }} arrow
+    - pushd arrow && git fetch -t {{ arrow.remote }} {{ arrow.branch }} && git checkout FETCH_HEAD && git submodule update --init --recursive && popd
+    - mkdir -p $(pwd)/build_artifacts && chmod a+rwx $(pwd)/build_artifacts
+    - /opt/docker/bin/entrypoint $FEEDSTOCK_ROOT/drone-steps.sh $(pwd)/build_artifacts {{ queue_remote_url }} {{ task.branch }} {{ task.tag }} {% if arrow.branch == 'master' %}1{% else %}0{% endif %}

dev/tasks/crossbow.py

@@ -691,7 +691,14 @@ class Task(Serializable):
     """
 
     def __init__(self, ci, template, artifacts=None, params=None):
-        assert ci in {'circle', 'travis', 'appveyor', 'azure', 'github'}
+        assert ci in {
+            'circle',
+            'travis',
+            'appveyor',
+            'azure',
+            'github',
+            'drone',
+        }
         self.ci = ci
         self.template = template
         self.artifacts = artifacts or []
@@ -733,6 +740,7 @@ def filename(self):
             'appveyor': 'appveyor.yml',
             'azure': 'azure-pipelines.yml',
             'github': '.github/workflows/crossbow.yml',
+            'drone': '.drone.yml',
         }
         return config_files[self.ci]
 

dev/tasks/tasks.yml

@@ -187,6 +187,33 @@ tasks:
       - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2
       - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2
 
+  conda-linux-gcc-py36-aarch64:
+    ci: drone
+    template: conda-recipes/drone.yml
+    params:
+      config: linux_aarch64_python3.6.____cpython
+    artifacts:
+      - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
+      - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2
+
+  conda-linux-gcc-py37-aarch64:
+    ci: drone
+    template: conda-recipes/drone.yml
+    params:
+      config: linux_aarch64_python3.7.____cpython
+    artifacts:
+      - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
+      - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
+
+  conda-linux-gcc-py38-aarch64:
+    ci: drone
+    template: conda-recipes/drone.yml
+    params:
+      config: linux_aarch64_python3.8.____cpython
+    artifacts:
+      - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+      - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+
   ############################## Conda OSX ####################################
 
   conda-osx-clang-py36: