ARROW-6477: [Packaging][Crossbow] Use Azure Pipelines to build linux packages

dev/tasks/linux-packages/travis.linux.yml → ci/conda_env_crossbow.txt

@@ -15,45 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
-os: linux
-dist: xenial
-language: ruby
-
-services:
-  - docker
-
-# don't build twice
-if: tag IS blank
-
-env:
-  global:
-    - TRAVIS_TAG={{ task.tag }}
-    - BUILD_REF={{ arrow.head }}
-    - ARROW_VERSION={{ arrow.version }}
-
-before_script:
-  - git clone --no-checkout {{ arrow.remote }} arrow
-  - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
-  - git -C arrow checkout FETCH_HEAD
-
-script:
-  - pushd arrow/dev/tasks/linux-packages
-  - rake version:update
-  - rake dist
-  - {{ build_command }}
-
-deploy:
-  provider: releases
-  api_key: $CROSSBOW_GITHUB_TOKEN
-  file_glob: true
-  file:
-  {% for extension in upload_extensions -%}
-    - "**/*{{ extension }}"
-  {% endfor -%}
-  skip_cleanup: true
-  on:
-    tags: true
-
-notifications:
-  email:
-    - {{ job.email }}
+click
+github3.py
+jinja2
+jira
+pygit2
+ruamel.yaml
+setuptools_scm
+toolz

dev/tasks/conda-recipes/azure.linux.yml

@@ -30,23 +30,24 @@ jobs:
     displayName: Run docker build
     workingDirectory: arrow/dev/tasks/conda-recipes
 
-  # Using github release tries to find a common ancestor between the currently
-  # pushed tag and the latest tag of the github repository (don't know why).
-  # The tag upload took 43 minutes because of this scan, so use an alternative
-  # upload script.
-  - task: UsePythonVersion@0
+  # Using github release tries to find a common ancestor between the
+  # currently pushed tag and the latest tag of the github repository
+  # (don't know why).
+  # The tag upload took 43 minutes because of this scan, so use an
+  # alternative upload script.
+  - task: CondaEnvironment@1
     inputs:
-      versionSpec: '3.6'
-      architecture: 'x64'
-
+      packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz'
+      installOptions: '-c conda-forge'
+      updateConda: false
   - script: |
-      python -m pip install github3-py click
-      python upload-assets.py \
+      python arrow/dev/tasks/crossbow.py \
+        --queue-path . \
+        --queue-remote {{ queue.remote_url }} \
+        upload-artifacts \
         --sha {{ task.branch }} \
         --tag {{ task.tag }} \
-        --pattern "conda-recipes/build_artifacts/linux-64/*.tar.bz2"
+        --pattern "arrow/dev/tasks/conda-recipes/build_artifacts/linux-64/*.tar.bz2"
     env:
-      CROSSBOW_GITHUB_REPO: $(Build.Repository.Name)
       CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
     displayName: Upload packages as a GitHub release
-    workingDirectory: arrow/dev/tasks

dev/tasks/conda-recipes/azure.osx.yml

@@ -73,19 +73,24 @@ jobs:
     workingDirectory: arrow/dev/tasks/conda-recipes
     displayName: Build recipes
 
-  # Using github release tries to find a common ancestor between the currently
-  # pushed tag and the latest tag of the github repository (don't know why).
-  # The tag upload took 43 minutes because of this scan, so use an alternative
-  # upload script.
+  # Using github release tries to find a common ancestor between the
+  # currently pushed tag and the latest tag of the github repository
+  # (don't know why).
+  # The tag upload took 43 minutes because of this scan, so use an
+  # alternative upload script.
+  - task: CondaEnvironment@1
+    inputs:
+      packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz'
+      installOptions: '-c conda-forge'
+      updateConda: false
   - script: |
-      source activate base
-      conda install -c conda-forge -y github3.py click
-      python upload-assets.py \
+      python arrow/dev/tasks/crossbow.py \
+        --queue-path . \
+        --queue-remote {{ queue.remote_url }} \
+        upload-artifacts \
         --sha {{ task.branch }} \
         --tag {{ task.tag }} \
-        --pattern "conda-recipes/build_artifacts/osx-64/*.tar.bz2"
+        --pattern "arrow/dev/tasks/conda-recipes/build_artifacts/osx-64/*.tar.bz2"
     env:
-      CROSSBOW_GITHUB_REPO: $(Build.Repository.Name)
       CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
     displayName: Upload packages as a GitHub release
-    workingDirectory: arrow/dev/tasks

dev/tasks/conda-recipes/azure.win.yml

@@ -88,22 +88,24 @@ jobs:
       PYTHONUNBUFFERED: 1
     condition: not(contains(variables['CONFIG'], 'vs2008'))
 
-  # Using github release tries to find a common ancestor between the currently
-  # pushed tag and the latest tag of the github repository (don't know why).
-  # The tag upload took 43 minutes because of this scan, so use an alternative upload script.
-  - task: UsePythonVersion@0
+  # Using github release tries to find a common ancestor between the
+  # currently pushed tag and the latest tag of the github repository
+  # (don't know why).
+  # The tag upload took 43 minutes because of this scan, so use an
+  # alternative upload script.
+  - task: CondaEnvironment@1
     inputs:
-      versionSpec: '3.6'
-      architecture: 'x64'
-
+      packageSpecs: 'click github3.py jinja2 jira pygit2 ruamel.yaml setuptools_scm toolz'
+      installOptions: '-c conda-forge'
+      updateConda: false
   - script: |
-      python -m pip install github3-py click
-      python upload-assets.py \
+      python arrow/dev/tasks/crossbow.py \
+        --queue-path . \
+        --queue-remote {{ queue.remote_url }} \
+        upload-artifacts \
         --sha {{ task.branch }} \
         --tag {{ task.tag }} \
-        --pattern "conda-recipes/build_artifacts/*.tar.bz2"
+        --pattern "arrow/dev/tasks/conda-recipes/build_artifacts/win-64/*.tar.bz2"
     env:
-      CROSSBOW_GITHUB_REPO: $(Build.Repository.Name)
       CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN)
     displayName: Upload packages as a GitHub release
-    workingDirectory: arrow/dev/tasks