Skip to content

Commit 7a98019

Browse files
yongtangyongtang
committed
Set up CI with Azure Pipelines
Signed-off-by: Yong Tang <[email protected]>
1 parent de9b9cf commit 7a98019

File tree

9 files changed

+212
-106
lines changed

9 files changed

+212
-106
lines changed

‎.travis.yml‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,25 +2,45 @@ dist: trusty
22
sudo: required
33
language: minimal
44

5+
git:
6+
quiet: true
7+
58
services:
6-
- docker
9+
- docker
710

811
before_script:
9-
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka
10-
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh
12+
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka
13+
- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh
1114

1215
matrix:
1316
include:
1417
- name: "Python 2.7 Build"
15-
env: PYTHON_VERSION=2.7
18+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true TEST_UBUNTU1804=true
1619
- name: "Python 3.4 Build"
17-
env: PYTHON_VERSION=3.4
20+
env: PYTHON_VERSION=3.4 BUILD_IMAGE=tensorflow/tensorflow:custom-op
1821
- name: "Python 3.5 Build"
19-
env: PYTHON_VERSION=3.5
22+
env: PYTHON_VERSION=3.5 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true
2023
- name: "Python 3.6 Build"
21-
env: PYTHON_VERSION=3.6
24+
env: PYTHON_VERSION=3.6 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1804=true
25+
- name: "Python 2.7 Build R Ubuntu 16.04"
26+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1604=true
27+
- name: "Python 2.7 Build R Ubuntu 18.04"
28+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1804=true
29+
- name: "Python 2.7 Ubuntu 16.04 Build"
30+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true
31+
- name: "Python 2.7 Ubuntu 18.04 Build"
32+
env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true
33+
- name: "Python 3.5 Ubuntu 16.04 Build"
34+
env: PYTHON_VERSION=3.5 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true
35+
- name: "Python 3.6 Ubuntu 18.04 Build"
36+
env: PYTHON_VERSION=3.6 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true
37+
2238
script:
23-
- docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION}
39+
- docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ${BUILD_IMAGE} bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION}
40+
- if [[ ! -z ${TEST_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi
41+
- if [[ ! -z ${TEST_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi
42+
- if [[ ! -z ${TEST_R_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi
43+
- if [[ ! -z ${TEST_R_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi
2444

2545
notifications:
2646
email: false

‎.travis/bazel.install.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# Install bazel
22
set -e -x
33

4+
apt-get -y -qq update
5+
apt-get -y -qq install unzip curl > /dev/null
46
BAZEL_OS=${1}
57
BAZEL_VERSION=${2}
68
curl -sOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh

‎.travis/pip.install.sh‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎.travis/python.configure.sh‎

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎.travis/python.install.sh‎

Lines changed: 47 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,22 @@
22
set -x -e
33

44
apt-get -y -qq update
5-
apt-get -y -qq install lsb-core
5+
apt-get -y -qq install lsb-core > /dev/null
66
if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then
7-
apt-get -y -qq install libav-tools
7+
apt-get -y -qq install libav-tools > /dev/null
88
if [[ ${1} == "2.7" ]]; then
9-
rm -f /usr/local/bin/pip
10-
ln -s /usr/local/bin/pip2 /usr/local/bin/pip
11-
pip install auditwheel
9+
echo "Python 2.7"
1210
elif [[ ${1} == "3.4" ]]; then
1311
rm -f /usr/local/bin/pip
14-
ln -s /usr/local/bin/pip3 /usr/local/bin/pip
12+
ln -s /usr/local/bin/pip3.4 /usr/local/bin/pip
1513
rm -f /usr/bin/python
16-
ln -s /usr/bin/python3 /usr/bin/python
17-
pip install auditwheel
14+
ln -s /usr/bin/python3.4 /usr/bin/python
1815
elif [[ ${1} == "3.5" ]]; then
1916
curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh
20-
chmod +x install_python3.5_pip_packages.sh
2117
sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.5_pip_packages.sh
2218
sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.5_pip_packages.sh
2319
sed -i 's/pip3.5 install/pip3.5 -q install/g' install_python3.5_pip_packages.sh
24-
./install_python3.5_pip_packages.sh
20+
bash install_python3.5_pip_packages.sh
2521
rm -f install_python3.5_pip_packages.sh
2622
rm -f /usr/bin/python
2723
ln -s /usr/bin/python3.5 /usr/bin/python
@@ -30,7 +26,6 @@ if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then
3026
elif [[ ${1} == "3.6" ]]; then
3127
rm -f /usr/local/bin/pip3
3228
curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh
33-
chmod +x install_python3.6_pip_packages.sh
3429
sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh
3530
sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh
3631
sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh
@@ -39,19 +34,58 @@ if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then
3934
sed -i 's/configure/configure -q/g' install_python3.6_pip_packages.sh
4035
sed -i 's/make altinstall/make altinstall>make.log/g' install_python3.6_pip_packages.sh
4136
sed -i 's/wget /wget -q /g' install_python3.6_pip_packages.sh
42-
./install_python3.6_pip_packages.sh
37+
bash install_python3.6_pip_packages.sh
4338
rm -f install_python3.6_pip_packages.sh make.log
4439
rm -rf Python-3.6.1*
4540
rm -f /usr/bin/python
4641
ln -s /usr/local/bin/python3.6 /usr/bin/python
4742
rm -f /usr/local/bin/pip
48-
ln -s /usr/local/bin/pip3 /usr/local/bin/pip
43+
ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip
4944
else
5045
echo Python ${1} not supported!
5146
exit 1
5247
fi
48+
elif [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then
49+
if [[ ${1} == "2.7" ]]; then
50+
echo "Python 2.7"
51+
apt-get -y -qq install ffmpeg python-pip > /dev/null
52+
pip install -q auditwheel==1.5.0
53+
pip install -q wheel==0.31.1
54+
elif [[ ${1} == "3.5" ]]; then
55+
echo "Python 3.5"
56+
apt-get -y -qq install ffmpeg python3-pip > /dev/null
57+
pip3 install -q auditwheel==1.5.0
58+
pip3 install -q wheel==0.31.1
59+
rm -f /usr/bin/python
60+
ln -s /usr/bin/python3 /usr/bin/python
61+
rm -f /usr/bin/pip
62+
ln -s /usr/bin/pip3 /usr/bin/pip
63+
else
64+
echo Platform $(lsb_release -r | awk '{ print $2 }') not supported!
65+
exit 1
66+
fi
67+
elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then
68+
if [[ ${1} == "2.7" ]]; then
69+
echo "Python 2.7"
70+
apt-get -y -qq install ffmpeg python-pip > /dev/null
71+
pip install -q auditwheel==1.5.0
72+
pip install -q wheel==0.31.1
73+
elif [[ ${1} == "3.6" ]]; then
74+
echo "Python 3.6"
75+
apt-get -y -qq install ffmpeg python3-pip > /dev/null
76+
pip3 install -q auditwheel==1.5.0
77+
pip3 install -q wheel==0.31.1
78+
rm -f /usr/bin/python
79+
ln -s /usr/bin/python3 /usr/bin/python
80+
rm -f /usr/bin/pip
81+
ln -s /usr/bin/pip3 /usr/bin/pip
82+
else
83+
echo Platform $(lsb_release -r | awk '{ print $2 }') not supported!
84+
exit 1
85+
fi
5386
else
5487
echo Platform $(lsb_release -r | awk '{ print $2 }') not supported!
88+
exit 1
5589
fi
5690
python --version
5791
pip --version

‎.travis/python.test.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#/bin/bash
2+
set -x -e
3+
4+
PYTHON_VERSION=${1}
5+
shift
6+
7+
TENSORFLOW_VERSION=1.12.0
8+
if [[ ! -z ${1} ]]; then
9+
TENSORFLOW_VERSION=${1}
10+
shift
11+
fi
12+
13+
apt-get -y -qq update
14+
apt-get -y -qq install lsb-core > /dev/null
15+
if [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then
16+
if [[ ${PYTHON_VERSION} == "2.7" ]]; then
17+
echo Python 2.7 Ubuntu 16.04
18+
apt-get -y -qq install ffmpeg python-pip > /dev/null
19+
pip install -q tensorflow==${TENSORFLOW_VERSION}
20+
pip install -q artifacts/tensorflow_io-*-cp27-*.whl
21+
pip install -q pytest
22+
(cd tests && python -m pytest -s .)
23+
echo Success
24+
exit 0
25+
elif [[ ${PYTHON_VERSION} == "3.5" ]]; then
26+
echo Python 3.5 Ubuntu 16.04
27+
apt-get -y -qq install ffmpeg python3-pip > /dev/null
28+
pip3 install -q tensorflow==${TENSORFLOW_VERSION}
29+
pip3 install -q artifacts/tensorflow_io-*-cp35-*.whl
30+
pip3 install -q pytest
31+
(cd tests && python3 -m pytest -s .)
32+
echo Success
33+
exit 0
34+
fi
35+
elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then
36+
apt-get -y -qq install ffmpeg > /dev/null
37+
if [[ ${PYTHON_VERSION} == "2.7" ]]; then
38+
echo Python 2.7 Ubuntu 18.04
39+
apt-get -y -qq install ffmpeg python-pip > /dev/null
40+
pip install -q tensorflow==${TENSORFLOW_VERSION}
41+
pip install -q artifacts/tensorflow_io-*-cp27-*.whl
42+
pip install -q pytest
43+
(cd tests && python -m pytest -s .)
44+
echo Success
45+
exit 0
46+
elif [[ ${PYTHON_VERSION} == "3.6" ]]; then
47+
echo Python 3.6 Ubuntu 18.04
48+
apt-get -y -qq install ffmpeg python3-pip > /dev/null
49+
pip3 install -q tensorflow==${TENSORFLOW_VERSION}
50+
pip3 install -q artifacts/tensorflow_io-*-cp36-*.whl
51+
pip3 install -q pytest
52+
(cd tests && python3 -m pytest -s .)
53+
echo Success
54+
exit 0
55+
fi
56+
fi
57+
echo Python ${1} on Ubuntu $(lsb_release -r | awk '{ print $2 }') not tested

‎.travis/r.configure.sh‎ renamed to ‎.travis/r.install.sh‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1+
#/bin/bash
2+
set -x -e
3+
14
# Install needed repo
2-
apt-get -y -qqq update
3-
apt-get -y -qqq install build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev
4-
apt-get -y -qqq install apt-transport-https ca-certificates curl gnupg2 software-properties-common
5+
apt-get -y -qq update
6+
apt-get -y -qq install build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev > /dev/null
7+
apt-get -y -qq install apt-transport-https ca-certificates curl gnupg2 software-properties-common > /dev/null
58
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9
69
add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran35/"
7-
apt-get -y -qqq update
8-
DEBIAN_FRONTEND=nonexteractive apt-get -y -qqq install r-base
10+
apt-get -y -qq update
11+
DEBIAN_FRONTEND=nonexteractive apt-get -y -qqq install r-base > /dev/null
912
echo "options(repos = c(CRAN='http://cran.rstudio.com'))" >> ~/.Rprofile
10-
R -e 'install.packages(c("Rcpp", "reticulate", "knitr", "tensorflow", "tfdatasets", "forge", "tidyselect", "testthat", "devtools"))'
13+
R -e 'install.packages(c("Rcpp", "reticulate", "knitr", "tensorflow", "tfdatasets", "forge", "tidyselect", "testthat", "devtools"), quiet = TRUE)'

‎.travis/r.test.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#/bin/bash
2+
set -x -e
3+
4+
PYTHON_VERSION=${1}
5+
shift
6+
7+
TENSORFLOW_VERSION=1.12.0
8+
if [[ ! -z ${1} ]]; then
9+
TENSORFLOW_VERSION=${1}
10+
shift
11+
fi
12+
13+
apt-get -y -qq update
14+
apt-get -y -qq install lsb-core > /dev/null
15+
if [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then
16+
if [[ ${PYTHON_VERSION} == "2.7" ]]; then
17+
echo R with Python 2.7 Ubuntu 16.04
18+
.travis/r.install.sh
19+
apt-get -y -qq install ffmpeg python-pip > /dev/null
20+
pip install -q tensorflow==${TENSORFLOW_VERSION}
21+
pip install -q artifacts/tensorflow_io-*-cp27-*.whl
22+
(cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())$failed == 0L))')
23+
echo Success
24+
exit 0
25+
fi
26+
elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then
27+
if [[ ${PYTHON_VERSION} == "2.7" ]]; then
28+
echo R with Python 2.7 Ubuntu 18.04
29+
.travis/r.install.sh
30+
apt-get -y -qq install ffmpeg python-pip > /dev/null
31+
pip install -q tensorflow==${TENSORFLOW_VERSION}
32+
pip install -q artifacts/tensorflow_io-*-cp27-*.whl
33+
(cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())$failed == 0L))')
34+
echo Success
35+
exit 0
36+
fi
37+
fi
38+
echo R with Python ${1} on Ubuntu $(lsb_release -r | awk '{ print $2 }') not tested

0 commit comments

Comments
 (0)