diff --git a/.travis.yml b/.travis.yml index e202cb05d..fb5cf3e3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,91 +1,55 @@ -dist: xenial +dist: trusty sudo: required language: minimal -addons: - apt: - update: true - -env: - - BAZEL_VERSION=0.20.0 +git: + quiet: true services: - - docker +- docker before_script: - - bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka - - bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh +- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka +- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh -matrix: +jobs: include: - - name: "Ubuntu 14.04 (Custom-op) Python 2.7 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.4 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.4 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.5 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.6 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 2.7 Tests" - script: - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 3.5 Tests" - script: - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 18.04 Python 2.7 Tests" - script: - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 18.04 Python 3.6 Tests" - script: - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 2.7 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)" - - name: "Ubuntu 16.04 Python 3.5 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp35-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)" - - name: "Ubuntu 18.04 Python 2.7 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)" - - name: "Ubuntu 18.04 Python 3.6 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp36-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)" - - name: "Ubuntu 16.04 R 3.5 (Python 2.7) Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'" - - name: "Ubuntu 18.04 R 3.5 (Python 2.7) Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'" + - stage: Build + script: echo Build Success + - stage: Test + script: echo Test Success + - stage: Nightly + script: echo PyPI.org + +#matrix: +# include: +# - name: "Python 2.7 Build" +# env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true TEST_UBUNTU1804=true +# - name: "Python 3.4 Build" +# env: PYTHON_VERSION=3.4 BUILD_IMAGE=tensorflow/tensorflow:custom-op +# - name: "Python 3.5 Build" +# env: PYTHON_VERSION=3.5 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true +# - name: "Python 3.6 Build" +# env: PYTHON_VERSION=3.6 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1804=true +# - name: "Python 2.7 Build R Ubuntu 16.04" +# env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1604=true +# - name: "Python 2.7 Build R Ubuntu 18.04" +# env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1804=true +# - name: "Python 2.7 Ubuntu 16.04 Build" +# env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true +# - name: "Python 2.7 Ubuntu 18.04 Build" +# env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true +# - name: "Python 3.5 Ubuntu 16.04 Build" +# env: PYTHON_VERSION=3.5 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true +# - name: "Python 3.6 Ubuntu 18.04 Build" +# env: PYTHON_VERSION=3.6 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true + +#script: +#- docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ${BUILD_IMAGE} bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION} +#- if [[ ! -z ${TEST_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi +#- if [[ ! -z ${TEST_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi +#- if [[ ! -z ${TEST_R_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi +#- if [[ ! -z ${TEST_R_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi notifications: email: false diff --git a/.travis/bazel.install.sh b/.travis/bazel.install.sh new file mode 100755 index 000000000..9a827d158 --- /dev/null +++ b/.travis/bazel.install.sh @@ -0,0 +1,11 @@ +# Install bazel +set -e -x + +apt-get -y -qq update +apt-get -y -qq install unzip curl > /dev/null +BAZEL_OS=${1} +BAZEL_VERSION=${2} +curl -sOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh diff --git a/.travis/python.configure.sh b/.travis/python.configure.sh deleted file mode 100755 index acccd5ada..000000000 --- a/.travis/python.configure.sh +++ /dev/null @@ -1,54 +0,0 @@ -apt-get -y -qq update -apt-get -y -qq install $INSTALL_PACKAGES -# Update Python and Pip alias -echo PYTHON_VERSION="${PYTHON_VERSION}" -if [ "${PYTHON_VERSION}" != "2.7" ]; then - echo CUSTOM_OP="${CUSTOM_OP}" - if [ -n "${CUSTOM_OP}" ]; then - if [ "${PYTHON_VERSION}" == "3.4" ]; then - rm -f /usr/bin/python - ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python - fi - if [ "${PYTHON_VERSION}" == "3.5" ]; then - curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh - chmod +x install_python3.5_pip_packages.sh - ./install_python3.5_pip_packages.sh >install_python3.5_pip_packages.log 2>&1 - rm install_python3.5_pip_packages.sh - rm -f /usr/bin/python - ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python - fi - if [ "${PYTHON_VERSION}" == "3.6" ]; then - curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh - chmod +x install_python3.6_pip_packages.sh - sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh - sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh - sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh - ./install_python3.6_pip_packages.sh >install_python3.6_pip_packages.log 2>&1 - rm install_python3.6_pip_packages.sh - rm -f /usr/bin/python - ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/bin/python - rm -f /usr/local/bin/python - ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python - fi - rm -f /usr/local/bin/pip - ln -s /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip - else - rm -f /usr/bin/pip - ln -s /usr/bin/pip3 /usr/bin/pip - rm -f /usr/bin/python - ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python - fi -fi -# Show gcc and python version in Travis CI -gcc -v -python --version -pip --version -# Install bazel -curl -OL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -# Install tensorflow -pip install tensorflow==1.12.0 -# Configure TensorFlow -./configure.sh diff --git a/.travis/python.install.sh b/.travis/python.install.sh new file mode 100755 index 000000000..d23399ea9 --- /dev/null +++ b/.travis/python.install.sh @@ -0,0 +1,100 @@ +#/bin/bash +set -x -e + +apt-get -y -qq update +apt-get -y -qq install lsb-core > /dev/null +if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then + apt-get -y -qq install libav-tools > /dev/null + if [[ ${1} == "2.7" ]]; then + echo "Python 2.7" + elif [[ ${1} == "3.4" ]]; then + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3.4 /usr/local/bin/pip + rm -f /usr/bin/python + ln -s /usr/bin/python3.4 /usr/bin/python + elif [[ ${1} == "3.5" ]]; then + curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh + sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.5_pip_packages.sh + sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.5_pip_packages.sh + sed -i 's/pip3.5 install/pip3.5 -q install/g' install_python3.5_pip_packages.sh + bash install_python3.5_pip_packages.sh + rm -f install_python3.5_pip_packages.sh + rm -f /usr/bin/python + ln -s /usr/bin/python3.5 /usr/bin/python + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3.5 /usr/local/bin/pip + elif [[ ${1} == "3.6" ]]; then + rm -f /usr/local/bin/pip3 + curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh + sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh + sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh + sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh + sed -i 's/pip3 install/pip3 -q install/g' install_python3.6_pip_packages.sh + sed -i 's/tar xvf/tar xf/g' install_python3.6_pip_packages.sh + sed -i 's/configure/configure -q/g' install_python3.6_pip_packages.sh + sed -i 's/make altinstall/make altinstall>make.log/g' install_python3.6_pip_packages.sh + sed -i 's/wget /wget -q /g' install_python3.6_pip_packages.sh + bash install_python3.6_pip_packages.sh + rm -f install_python3.6_pip_packages.sh make.log + rm -rf Python-3.6.1* + rm -f /usr/bin/python + ln -s /usr/local/bin/python3.6 /usr/bin/python + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip + else + echo Python ${1} not supported! + exit 1 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then + if [[ ${1} == "2.7" ]]; then + echo "Python 2.7" + apt-get -y -qq install ffmpeg python-pip python3-pip patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install -q wheel==0.31.1 + elif [[ ${1} == "3.5" ]]; then + echo "Python 3.5" + apt-get -y -qq install ffmpeg python3-pip patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install wheel==0.31.1 + rm -f /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python + rm -f /usr/bin/pip + ln -s /usr/bin/pip3 /usr/bin/pip + else + echo Platform $(lsb_release -r | awk '{ print $2 }') not supported! + exit 1 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then + if [[ ${1} == "2.7" ]]; then + echo "Python 2.7" + apt-get -y -qq install ffmpeg python-pip python3-pip python3-wheel patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install wheel==0.31.1 + elif [[ ${1} == "3.6" ]]; then + echo "Python 3.6" + apt-get -y -qq install ffmpeg python3-pip python3-wheel patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install wheel==0.31.1 + rm -f /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python + rm -f /usr/bin/pip + ln -s /usr/bin/pip3 /usr/bin/pip + else + echo Platform $(lsb_release -r | awk '{ print $2 }') not supported! + exit 1 + fi +else + echo Platform $(lsb_release -r | awk '{ print $2 }') not supported! + exit 1 +fi +python --version +pip --version +pip freeze diff --git a/.travis/python.release.sh b/.travis/python.release.sh index 4341fb435..3046d2a06 100755 --- a/.travis/python.release.sh +++ b/.travis/python.release.sh @@ -13,24 +13,37 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -set -e -set -x +set -e -x + # Release: -# docker run -it -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> +# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> +# Nightly: +# export TENSORFLOW_IO_VERSION=0.3.0.dev$(date '+%Y%m%d%H%M%S') +# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> $TENSORFLOW_IO_VERSION if [[ -z "${1}" ]]; then echo "usage:" $0 "<2.7|3.4|3.5|3.6>" exit 1 fi -export PYTHON_VERSION=$1 -export BAZEL_VERSION=${BAZEL_VERSION} -export CUSTOM_OP=True -export BAZEL_OS=linux -.travis/python.configure.sh -bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg +BAZEL_VERSION=0.20.0 +PYTHON_VERSION=${1} +shift +TENSORFLOW_VERSION=1.12.0 +if [[ ! -z ${1} ]]; then + TENSORFLOW_VERSION=${1} + shift +fi +.travis/bazel.install.sh linux ${BAZEL_VERSION} + +.travis/python.install.sh ${PYTHON_VERSION} + +pip install -q tensorflow==${TENSORFLOW_VERSION} +./configure.sh +bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/... +bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg bazel-bin/build_pip_pkg artifacts exit 0 diff --git a/.travis/python.test.sh b/.travis/python.test.sh new file mode 100755 index 000000000..1bf858ec8 --- /dev/null +++ b/.travis/python.test.sh @@ -0,0 +1,57 @@ +#/bin/bash +set -x -e + +PYTHON_VERSION=${1} +shift + +TENSORFLOW_VERSION=1.12.0 +if [[ ! -z ${1} ]]; then + TENSORFLOW_VERSION=${1} + shift +fi + +apt-get -y -qq update +apt-get -y -qq install lsb-core > /dev/null +if [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo Python 2.7 Ubuntu 16.04 + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io-*-cp27-*.whl + pip install -q pytest + (cd tests && python -m pytest -s .) + echo Success + exit 0 + elif [[ ${PYTHON_VERSION} == "3.5" ]]; then + echo Python 3.5 Ubuntu 16.04 + apt-get -y -qq install ffmpeg python3-pip > /dev/null + pip3 install -q tensorflow==${TENSORFLOW_VERSION} + pip3 install -q artifacts/tensorflow_io-*-cp35-*.whl + pip3 install -q pytest + (cd tests && python3 -m pytest -s .) + echo Success + exit 0 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then + apt-get -y -qq install ffmpeg > /dev/null + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo Python 2.7 Ubuntu 18.04 + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io-*-cp27-*.whl + pip install -q pytest + (cd tests && python -m pytest -s .) + echo Success + exit 0 + elif [[ ${PYTHON_VERSION} == "3.6" ]]; then + echo Python 3.6 Ubuntu 18.04 + apt-get -y -qq install ffmpeg python3-pip > /dev/null + pip3 install -q tensorflow==${TENSORFLOW_VERSION} + pip3 install -q artifacts/tensorflow_io-*-cp36-*.whl + pip3 install -q pytest + (cd tests && python3 -m pytest -s .) + echo Success + exit 0 + fi +fi +echo Python ${1} on Ubuntu $(lsb_release -r | awk '{ print $2 }') not tested diff --git a/.travis/r.configure.sh b/.travis/r.install.sh similarity index 53% rename from .travis/r.configure.sh rename to .travis/r.install.sh index c3cee13ff..e36912a98 100755 --- a/.travis/r.configure.sh +++ b/.travis/r.install.sh @@ -1,10 +1,13 @@ +#/bin/bash +set -x -e + # Install needed repo -apt-get -y -qqq update -apt-get -y -qqq install build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev -apt-get -y -qqq install apt-transport-https ca-certificates curl gnupg2 software-properties-common +apt-get -y -qq update +apt-get -y -qq install build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev > /dev/null +apt-get -y -qq install apt-transport-https ca-certificates curl gnupg2 software-properties-common > /dev/null apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran35/" -apt-get -y -qqq update -DEBIAN_FRONTEND=nonexteractive apt-get -y -qqq install r-base +apt-get -y -qq update +DEBIAN_FRONTEND=nonexteractive apt-get -y -qqq install r-base > /dev/null echo "options(repos = c(CRAN='http://cran.rstudio.com'))" >> ~/.Rprofile -R -e 'install.packages(c("Rcpp", "reticulate", "knitr", "tensorflow", "tfdatasets", "forge", "tidyselect", "testthat", "devtools"))' +R -e 'install.packages(c("Rcpp", "reticulate", "knitr", "tensorflow", "tfdatasets", "forge", "tidyselect", "testthat", "devtools"), quiet = TRUE)' diff --git a/.travis/r.test.sh b/.travis/r.test.sh new file mode 100755 index 000000000..429e86e5f --- /dev/null +++ b/.travis/r.test.sh @@ -0,0 +1,38 @@ +#/bin/bash +set -x -e + +PYTHON_VERSION=${1} +shift + +TENSORFLOW_VERSION=1.12.0 +if [[ ! -z ${1} ]]; then + TENSORFLOW_VERSION=${1} + shift +fi + +apt-get -y -qq update +apt-get -y -qq install lsb-core > /dev/null +if [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo R with Python 2.7 Ubuntu 16.04 + .travis/r.install.sh + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io-*-cp27-*.whl + (cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())$failed == 0L))') + echo Success + exit 0 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo R with Python 2.7 Ubuntu 18.04 + .travis/r.install.sh + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io-*-cp27-*.whl + (cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())$failed == 0L))') + echo Success + exit 0 + fi +fi +echo R with Python ${1} on Ubuntu $(lsb_release -r | awk '{ print $2 }') not tested diff --git a/HEAD~2 b/HEAD~2 new file mode 100644 index 000000000..07bfc5859 --- /dev/null +++ b/HEAD~2 @@ -0,0 +1,39 @@ +# This viminfo file was generated by Vim 8.0. +# You may edit it if you're careful! + +# Viminfo version +|1,4 + +# Value of 'encoding' when this file was written +*encoding=utf-8 + + +# hlsearch on (H) or off (h): +~h +# Command Line History (newest to oldest): +:q! +|2,0,1549691502,,"q!" + +# Search String History (newest to oldest): + +# Expression History (newest to oldest): + +# Input Line History (newest to oldest): + +# Debug Line History (newest to oldest): + +# Registers: + +# File marks: +'0 1 0 ~/io/rebase +|4,48,1,0,1549691502,"~/io/rebase" + +# Jumplist (newest first): +-' 1 0 ~/io/rebase +|4,39,1,0,1549691502,"~/io/rebase" + +# History of marks within files (newest to oldest): + +> ~/io/rebase + * 1549691501 0 + " 1 0 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a0c1286e4..b96454d19 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,8 +13,38 @@ steps: sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" sudo apt-get -y -qq update sudo apt-get -y -qq install docker-ce docker-ce-cli containerd.io - echo Success! displayName: 'Install Docker CE' - script: | pip install twine displayName: 'Install Twine' +- script: | + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka + sudo bash -x tensorflow_io/ignite/python/tests/start_ignite.sh + sudo docker run --rm -v ${PWD}:/working_dir -w /working_dir --net=host tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 + sudo bash -x tensorflow_io/ignite/python/tests/stop_ignite.sh + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh stop kafka + displayName: 'Build TensorFlow IO (Python 2.7)' +- script: | + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka + sudo bash -x tensorflow_io/ignite/python/tests/start_ignite.sh + sudo docker run --rm -v ${PWD}:/working_dir -w /working_dir --net=host tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.4 + sudo bash -x tensorflow_io/ignite/python/tests/stop_ignite.sh + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh stop kafka + displayName: 'Build TensorFlow IO (Python 3.4)' +- script: | + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka + sudo bash -x tensorflow_io/ignite/python/tests/start_ignite.sh + sudo docker run --rm -v ${PWD}:/working_dir -w /working_dir --net=host tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5 + sudo bash -x tensorflow_io/ignite/python/tests/stop_ignite.sh + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh stop kafka + displayName: 'Build TensorFlow IO (Python 3.5)' +- script: | + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka + sudo bash -x tensorflow_io/ignite/python/tests/start_ignite.sh + sudo docker run --rm -v ${PWD}:/working_dir -w /working_dir --net=host tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6 + sudo bash -x tensorflow_io/ignite/python/tests/stop_ignite.sh + sudo bash -x tensorflow_io/kafka/python/kernel_tests/kafka_test.sh stop kafka + displayName: 'Build TensorFlow IO (Python 3.6)' +- task: TwineAuthenticate@0 + inputs: + externalFeeds: 'tensorflow-io-nightly'