Skip to content

Commit 780eb22

Browse files
yongtangyongtang
committed
Travis CI improvement
Travis CI build is getting longer and longer, so we have to think about improvement with various methods. PyPI.org's nightly build is also setup. Signed-off-by: Yong Tang <[email protected]>
1 parent 5275afd commit 780eb22

File tree

10 files changed

+319
-151
lines changed

10 files changed

+319
-151
lines changed

‎.travis.yml‎

Lines changed: 58 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,91 +1,70 @@
1-
dist: xenial
1+
dist: trusty
22
sudo: required
33
language: minimal
44

5-
addons:
6-
apt:
7-
update: true
8-
9-
env:
10-
- BAZEL_VERSION=0.20.0
5+
git:
6+
quiet: true
117

128
services:
13-
- docker
9+
- docker
10+
11+
stages:
12+
- test
13+
- release
1414

15-
before_script:
16-
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka
17-
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh
15+
script:
16+
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka
17+
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh
18+
- docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ${BUILD_IMAGE} bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION}
19+
- if [[ ! -z ${TEST_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi
20+
- if [[ ! -z ${TEST_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi
21+
- if [[ ! -z ${TEST_R_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi
22+
- if [[ ! -z ${TEST_R_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi
23+
- echo Test Success
1824

19-
matrix:
25+
jobs:
2026
include:
21-
- name: "Ubuntu 14.04 (Custom-op) Python 2.7 Tests"
22-
script:
23-
- docker pull tensorflow/tensorflow:custom-op
24-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
25-
- name: "Ubuntu 14.04 (Custom-op) Python 3.4 Tests"
26-
script:
27-
- docker pull tensorflow/tensorflow:custom-op
28-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.4 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
29-
- name: "Ubuntu 14.04 (Custom-op) Python 3.5 Tests"
30-
script:
31-
- docker pull tensorflow/tensorflow:custom-op
32-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
33-
- name: "Ubuntu 14.04 (Custom-op) Python 3.6 Tests"
34-
script:
35-
- docker pull tensorflow/tensorflow:custom-op
36-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
37-
- name: "Ubuntu 16.04 Python 2.7 Tests"
38-
script:
39-
- docker pull ubuntu:16.04
40-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
41-
- name: "Ubuntu 16.04 Python 3.5 Tests"
42-
script:
43-
- docker pull ubuntu:16.04
44-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
45-
- name: "Ubuntu 18.04 Python 2.7 Tests"
46-
script:
47-
- docker pull ubuntu:18.04
48-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
49-
- name: "Ubuntu 18.04 Python 3.6 Tests"
50-
script:
51-
- docker pull ubuntu:18.04
52-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..."
53-
- name: "Ubuntu 16.04 Python 2.7 Build Tests"
54-
script:
55-
- docker pull tensorflow/tensorflow:custom-op
56-
- docker pull ubuntu:16.04
57-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
58-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)"
59-
- name: "Ubuntu 16.04 Python 3.5 Build Tests"
60-
script:
61-
- docker pull tensorflow/tensorflow:custom-op
62-
- docker pull ubuntu:16.04
63-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5
64-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp35-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)"
65-
- name: "Ubuntu 18.04 Python 2.7 Build Tests"
66-
script:
67-
- docker pull tensorflow/tensorflow:custom-op
68-
- docker pull ubuntu:18.04
69-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
70-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)"
71-
- name: "Ubuntu 18.04 Python 3.6 Build Tests"
72-
script:
73-
- docker pull tensorflow/tensorflow:custom-op
74-
- docker pull ubuntu:18.04
75-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6
76-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp36-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)"
77-
- name: "Ubuntu 16.04 R 3.5 (Python 2.7) Build Tests"
78-
script:
79-
- docker pull tensorflow/tensorflow:custom-op
80-
- docker pull ubuntu:16.04
81-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
82-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'"
83-
- name: "Ubuntu 18.04 R 3.5 (Python 2.7) Build Tests"
27+
- stage: test
28+
name: "Python 2.7 Build"
29+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true TEST_UBUNTU1804=true
30+
- stage: test
31+
name: "Python 3.4 Build"
32+
env: PYTHON_VERSION=3.4 BUILD_IMAGE=tensorflow/tensorflow:custom-op
33+
- stage: test
34+
name: "Python 3.5 Build"
35+
env: PYTHON_VERSION=3.5 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true
36+
- stage: test
37+
name: "Python 3.6 Build"
38+
env: PYTHON_VERSION=3.6 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1804=true
39+
- stage: test
40+
name: "Python 2.7 Build R Ubuntu 16.04"
41+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1604=true
42+
- stage: test
43+
name: "Python 2.7 Build R Ubuntu 18.04"
44+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1804=true
45+
- stage: test
46+
name: "Python 2.7 Ubuntu 16.04 Build"
47+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true
48+
- stage: test
49+
name: "Python 2.7 Ubuntu 18.04 Build"
50+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true
51+
- stage: test
52+
name: "Python 3.5 Ubuntu 16.04 Build"
53+
env: PYTHON_VERSION=3.5 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true
54+
- stage: test
55+
name: "Python 3.6 Ubuntu 18.04 Build"
56+
env: PYTHON_VERSION=3.6 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true
57+
- stage: release
58+
name: "Release Nightly"
59+
if: branch = travis
8460
script:
85-
- docker pull tensorflow/tensorflow:custom-op
86-
- docker pull ubuntu:18.04
87-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7
88-
- docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'"
61+
- export TENSORFLOW_IO_VERSION=0.3.0.dev$(date '+%Y%m%d%H%M%S')
62+
- echo $TENSORFLOW_IO_VERSION
63+
- docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION
64+
- docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.4 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION
65+
- docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION
66+
- docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION
67+
- twine upload artifacts/*
8968

9069
notifications:
9170
email: false

‎.travis/bazel.install.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Install bazel
2+
set -e -x
3+
4+
apt-get -y -qq update
5+
apt-get -y -qq install unzip curl > /dev/null
6+
BAZEL_OS=${1}
7+
BAZEL_VERSION=${2}
8+
curl -sOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
9+
chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
10+
./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
11+
rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh

‎.travis/python.configure.sh‎

Lines changed: 0 additions & 54 deletions
This file was deleted.

0 commit comments

Comments
 (0)