From e4ce408276d2bce8b81912baa7d1c8191ed9ebd1 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 8 Feb 2019 20:10:48 -0800 Subject: [PATCH 1/3] Set up CI with Azure Pipelines Signed-off-by: Yong Tang --- azure-pipelines.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 000000000..a0c1286e4 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,20 @@ +trigger: +- azure-pipelines + +pool: + vmImage: 'Ubuntu-16.04' + +steps: +- script: | + sudo apt-get -y -qq update + sudo apt-get -y -qq install apt-transport-https ca-certificates curl gnupg-agent software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo apt-key fingerprint 0EBFCD88 + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + sudo apt-get -y -qq update + sudo apt-get -y -qq install docker-ce docker-ce-cli containerd.io + echo Success! + displayName: 'Install Docker CE' +- script: | + pip install twine + displayName: 'Install Twine' From 4c19e924f692cbbca491620450e2b86cdad4b8f4 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 8 Feb 2019 21:14:14 +0000 Subject: [PATCH 2/3] Travis CI improvement Travis CI build is getting longer and longer, so we have to think about improvement with various methods. Signed-off-by: Yong Tang --- .travis.yml | 87 +++++---------------------------------- .travis/bazel.install.sh | 9 ++++ .travis/pip.install.sh | 24 +++++++++++ .travis/python.install.sh | 58 ++++++++++++++++++++++++++ .travis/python.release.sh | 31 ++++++++++---- 5 files changed, 124 insertions(+), 85 deletions(-) create mode 100755 .travis/bazel.install.sh create mode 100755 .travis/pip.install.sh create mode 100755 .travis/python.install.sh diff --git a/.travis.yml b/.travis.yml index e202cb05d..2c9988ebb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,7 @@ -dist: xenial +dist: trusty sudo: required language: minimal -addons: - apt: - update: true - -env: - - BAZEL_VERSION=0.20.0 - services: - docker @@ -18,74 +11,16 @@ before_script: matrix: include: - - name: "Ubuntu 14.04 (Custom-op) Python 2.7 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.4 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.4 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.5 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.6 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 2.7 Tests" - script: - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 3.5 Tests" - script: - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 18.04 Python 2.7 Tests" - script: - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 18.04 Python 3.6 Tests" - script: - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 2.7 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)" - - name: "Ubuntu 16.04 Python 3.5 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp35-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)" - - name: "Ubuntu 18.04 Python 2.7 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)" - - name: "Ubuntu 18.04 Python 3.6 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp36-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)" - - name: "Ubuntu 16.04 R 3.5 (Python 2.7) Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'" - - name: "Ubuntu 18.04 R 3.5 (Python 2.7) Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'" + - name: "Python 2.7 Build" + env: PYTHON_VERSION=2.7 + - name: "Python 3.4 Build" + env: PYTHON_VERSION=3.4 + - name: "Python 3.5 Build" + env: PYTHON_VERSION=3.5 + - name: "Python 3.6 Build" + env: PYTHON_VERSION=3.6 +script: + - docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION} notifications: email: false diff --git a/.travis/bazel.install.sh b/.travis/bazel.install.sh new file mode 100755 index 000000000..5fb5df7f8 --- /dev/null +++ b/.travis/bazel.install.sh @@ -0,0 +1,9 @@ +# Install bazel +set -e -x + +BAZEL_OS=${1} +BAZEL_VERSION=${2} +curl -sOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh diff --git a/.travis/pip.install.sh b/.travis/pip.install.sh new file mode 100755 index 000000000..db5e5a516 --- /dev/null +++ b/.travis/pip.install.sh @@ -0,0 +1,24 @@ +# Pin wheel==0.31.1 to work around issue +# https://github.com/pypa/auditwheel/issues/102 +pip install -q wheel==0.31.1 + +# Install last working version of setuptools. This must happen before we install +# absl-py, which uses install_requires notation introduced in setuptools 20.5. +pip install -q --upgrade setuptools==39.1.0 + +# Install six. +pip install -q --upgrade six==1.10.0 + +# Install protobuf. +pip install -q --upgrade protobuf==3.6.0 + +# numpy needs to be installed from source to fix segfaults. See: +# https://github.com/tensorflow/tensorflow/issues/6968 +# This workaround isn't needed for Ubuntu 16.04 or later. +pip install -q --no-binary=:all: --upgrade numpy==1.14.5 + +# Keras +pip install -q keras_applications==1.0.6 --no-deps +pip install -q keras_preprocessing==1.0.5 --no-deps + +pip install auditwheel==1.5.0 diff --git a/.travis/python.install.sh b/.travis/python.install.sh new file mode 100755 index 000000000..295b72d88 --- /dev/null +++ b/.travis/python.install.sh @@ -0,0 +1,58 @@ +#/bin/bash +set -x -e + +apt-get -y -qq update +apt-get -y -qq install lsb-core +if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then + apt-get -y -qq install libav-tools + if [[ ${1} == "2.7" ]]; then + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip2 /usr/local/bin/pip + pip install auditwheel==1.5.0 + elif [[ ${1} == "3.4" ]]; then + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3 /usr/local/bin/pip + rm -f /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python + pip install auditwheel==1.5.0 + elif [[ ${1} == "3.5" ]]; then + curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh + chmod +x install_python3.5_pip_packages.sh + sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.5_pip_packages.sh + sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.5_pip_packages.sh + sed -i 's/pip3.5 install/pip3.5 -q install/g' install_python3.5_pip_packages.sh + ./install_python3.5_pip_packages.sh + rm -f install_python3.5_pip_packages.sh + rm -f /usr/bin/python + ln -s /usr/bin/python3.5 /usr/bin/python + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3.5 /usr/local/bin/pip + elif [[ ${1} == "3.6" ]]; then + rm -f /usr/local/bin/pip3 + curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh + chmod +x install_python3.6_pip_packages.sh + sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh + sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh + sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh + sed -i 's/pip3 install/pip3 -q install/g' install_python3.6_pip_packages.sh + sed -i 's/tar xvf/tar xf/g' install_python3.6_pip_packages.sh + sed -i 's/configure/configure -q/g' install_python3.6_pip_packages.sh + sed -i 's/make altinstall/make altinstall>make.log/g' install_python3.6_pip_packages.sh + sed -i 's/wget /wget -q /g' install_python3.6_pip_packages.sh + ./install_python3.6_pip_packages.sh + rm -f install_python3.6_pip_packages.sh make.log + rm -rf Python-3.6.1* + rm -f /usr/bin/python + ln -s /usr/local/bin/python3.6 /usr/bin/python + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3 /usr/local/bin/pip + else + echo Python ${1} not supported! + exit 1 + fi +else + echo Platform $(lsb_release -r | awk '{ print $2 }') not supported! +fi +python --version +pip --version +pip freeze diff --git a/.travis/python.release.sh b/.travis/python.release.sh index 4341fb435..3046d2a06 100755 --- a/.travis/python.release.sh +++ b/.travis/python.release.sh @@ -13,24 +13,37 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -set -e -set -x +set -e -x + # Release: -# docker run -it -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> +# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> +# Nightly: +# export TENSORFLOW_IO_VERSION=0.3.0.dev$(date '+%Y%m%d%H%M%S') +# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> $TENSORFLOW_IO_VERSION if [[ -z "${1}" ]]; then echo "usage:" $0 "<2.7|3.4|3.5|3.6>" exit 1 fi -export PYTHON_VERSION=$1 -export BAZEL_VERSION=${BAZEL_VERSION} -export CUSTOM_OP=True -export BAZEL_OS=linux -.travis/python.configure.sh -bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg +BAZEL_VERSION=0.20.0 +PYTHON_VERSION=${1} +shift +TENSORFLOW_VERSION=1.12.0 +if [[ ! -z ${1} ]]; then + TENSORFLOW_VERSION=${1} + shift +fi +.travis/bazel.install.sh linux ${BAZEL_VERSION} + +.travis/python.install.sh ${PYTHON_VERSION} + +pip install -q tensorflow==${TENSORFLOW_VERSION} +./configure.sh +bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/... +bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg bazel-bin/build_pip_pkg artifacts exit 0 From e932744f6e0ccd2c8235b6a7ef707ea510f767b1 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Fri, 8 Feb 2019 20:10:48 -0800 Subject: [PATCH 3/3] Set up CI with Azure Pipelines Signed-off-by: Yong Tang --- azure-pipelines.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a0c1286e4..2f9e3a082 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,4 +17,14 @@ steps: displayName: 'Install Docker CE' - script: | pip install twine + echo Success! displayName: 'Install Twine' +- script: | + sudo bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka + sudo bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh + echo Success! + displayName: 'Setup Ignite/Kafka' +- script: | + sudo docker run --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 + echo Success! + displayName: 'Build TensorFlow IO'