diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..d1d8ddecb --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,16 @@ +version: 2 +jobs: + build: + docker: + - image: debian:stretch + + steps: + - checkout + + - run: + name: Greeting + command: echo Hello, world. + + - run: + name: Print the Current Time + command: date diff --git a/.travis.yml b/.travis.yml index e202cb05d..a8549211c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,91 +1,70 @@ -dist: xenial +dist: trusty sudo: required language: minimal -addons: - apt: - update: true - -env: - - BAZEL_VERSION=0.20.0 +git: + quiet: true services: - - docker +- docker + +stages: + - test + - release -before_script: - - bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka - - bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh +script: +- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/kafka/python/kernel_tests/kafka_test.sh start kafka +- bash -x ${TRAVIS_BUILD_DIR}/tensorflow_io/ignite/python/tests/start_ignite.sh +- docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ${BUILD_IMAGE} bash -x /working_dir/.travis/python.release.sh ${PYTHON_VERSION} +- if [[ ! -z ${TEST_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi +- if [[ ! -z ${TEST_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/python.test.sh ${PYTHON_VERSION} ; fi +- if [[ ! -z ${TEST_R_UBUNTU1604} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:16.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi +- if [[ ! -z ${TEST_R_UBUNTU1804} ]]; then docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir --net=host ubuntu:18.04 bash -x /working_dir/.travis/r.test.sh ${PYTHON_VERSION} ; fi +- echo Test Success -matrix: +jobs: include: - - name: "Ubuntu 14.04 (Custom-op) Python 2.7 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.4 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.4 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.5 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 14.04 (Custom-op) Python 3.6 Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="libav-tools" -e CUSTOM_OP=True -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host tensorflow/tensorflow:custom-op bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 2.7 Tests" - script: - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 3.5 Tests" - script: - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.5 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 18.04 Python 2.7 Tests" - script: - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python-pip ffmpeg" -e PYTHON_VERSION=2.7 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 18.04 Python 3.6 Tests" - script: - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e INSTALL_PACKAGES="curl unzip python3-pip ffmpeg" -e PYTHON_VERSION=3.6 -e BAZEL_VERSION=$BAZEL_VERSION -e BAZEL_OS=linux --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/python.configure.sh && pip install pyarrow==0.11.1 && bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/..." - - name: "Ubuntu 16.04 Python 2.7 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)" - - name: "Ubuntu 16.04 Python 3.5 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp35-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)" - - name: "Ubuntu 18.04 Python 2.7 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python-pip ffmpeg && pip install artifacts/tensorflow_io-*-cp27-*.whl && pip install pytest && (cd tests && python -m pytest .)" - - name: "Ubuntu 18.04 Python 3.6 Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c "apt-get -y -qqq update && apt-get -y -qqq install python3-pip ffmpeg && pip3 install artifacts/tensorflow_io-*-cp36-*.whl && pip3 install pytest && (cd tests && python3 -m pytest .)" - - name: "Ubuntu 16.04 R 3.5 (Python 2.7) Build Tests" - script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:16.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:16.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'" - - name: "Ubuntu 18.04 R 3.5 (Python 2.7) Build Tests" + - stage: test + name: "Python 2.7 Build" + env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true TEST_UBUNTU1804=true + - stage: test + name: "Python 3.4 Build" + env: PYTHON_VERSION=3.4 BUILD_IMAGE=tensorflow/tensorflow:custom-op + - stage: test + name: "Python 3.5 Build" + env: PYTHON_VERSION=3.5 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1604=true + - stage: test + name: "Python 3.6 Build" + env: PYTHON_VERSION=3.6 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_UBUNTU1804=true + - stage: test + name: "Python 2.7 Build R Ubuntu 16.04" + env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1604=true + - stage: test + name: "Python 2.7 Build R Ubuntu 18.04" + env: PYTHON_VERSION=2.7 BUILD_IMAGE=tensorflow/tensorflow:custom-op TEST_R_UBUNTU1804=true + - stage: test + name: "Python 2.7 Ubuntu 16.04 Build" + env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true + - stage: test + name: "Python 2.7 Ubuntu 18.04 Build" + env: PYTHON_VERSION=2.7 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true + - stage: test + name: "Python 3.5 Ubuntu 16.04 Build" + env: PYTHON_VERSION=3.5 BUILD_IMAGE=ubuntu:16.04 TEST_UBUNTU1604=true + - stage: test + name: "Python 3.6 Ubuntu 18.04 Build" + env: PYTHON_VERSION=3.6 BUILD_IMAGE=ubuntu:18.04 TEST_UBUNTU1804=true + - stage: release + name: "Release Nightly" + if: branch = travis script: - - docker pull tensorflow/tensorflow:custom-op - - docker pull ubuntu:18.04 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 - - docker run -i -t -v $HOME/.cache/pip:/root/.cache/pip -e --rm -v $PWD:/workspace -w /workspace --net=host ubuntu:18.04 bash -x -c ".travis/r.configure.sh && apt-get -y -qqq install ffmpeg python-pip && pip install artifacts/tensorflow_io-*-cp27-*.whl && cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())\$failed == 0L))'" + - export TENSORFLOW_IO_VERSION=0.3.0.dev$(date '+%Y%m%d%H%M%S') + - echo $TENSORFLOW_IO_VERSION + - docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 2.7 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION + - docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.4 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION + - docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.5 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION + - docker run -i -t --rm -e TENSORFLOW_IO_TEST=no -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh 3.6 1.12.0 --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION + - twine upload artifacts/* notifications: email: false diff --git a/.travis/bazel.install.sh b/.travis/bazel.install.sh new file mode 100755 index 000000000..9a827d158 --- /dev/null +++ b/.travis/bazel.install.sh @@ -0,0 +1,11 @@ +# Install bazel +set -e -x + +apt-get -y -qq update +apt-get -y -qq install unzip curl > /dev/null +BAZEL_OS=${1} +BAZEL_VERSION=${2} +curl -sOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh +rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh diff --git a/.travis/python.configure.sh b/.travis/python.configure.sh deleted file mode 100755 index acccd5ada..000000000 --- a/.travis/python.configure.sh +++ /dev/null @@ -1,54 +0,0 @@ -apt-get -y -qq update -apt-get -y -qq install $INSTALL_PACKAGES -# Update Python and Pip alias -echo PYTHON_VERSION="${PYTHON_VERSION}" -if [ "${PYTHON_VERSION}" != "2.7" ]; then - echo CUSTOM_OP="${CUSTOM_OP}" - if [ -n "${CUSTOM_OP}" ]; then - if [ "${PYTHON_VERSION}" == "3.4" ]; then - rm -f /usr/bin/python - ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python - fi - if [ "${PYTHON_VERSION}" == "3.5" ]; then - curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh - chmod +x install_python3.5_pip_packages.sh - ./install_python3.5_pip_packages.sh >install_python3.5_pip_packages.log 2>&1 - rm install_python3.5_pip_packages.sh - rm -f /usr/bin/python - ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python - fi - if [ "${PYTHON_VERSION}" == "3.6" ]; then - curl -OL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh - chmod +x install_python3.6_pip_packages.sh - sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh - sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh - sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh - ./install_python3.6_pip_packages.sh >install_python3.6_pip_packages.log 2>&1 - rm install_python3.6_pip_packages.sh - rm -f /usr/bin/python - ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/bin/python - rm -f /usr/local/bin/python - ln -s /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python - fi - rm -f /usr/local/bin/pip - ln -s /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip - else - rm -f /usr/bin/pip - ln -s /usr/bin/pip3 /usr/bin/pip - rm -f /usr/bin/python - ln -s /usr/bin/python${PYTHON_VERSION} /usr/bin/python - fi -fi -# Show gcc and python version in Travis CI -gcc -v -python --version -pip --version -# Install bazel -curl -OL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -chmod +x bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -./bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -rm -f bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -# Install tensorflow -pip install tensorflow==1.12.0 -# Configure TensorFlow -./configure.sh diff --git a/.travis/python.install.sh b/.travis/python.install.sh new file mode 100755 index 000000000..d23399ea9 --- /dev/null +++ b/.travis/python.install.sh @@ -0,0 +1,100 @@ +#/bin/bash +set -x -e + +apt-get -y -qq update +apt-get -y -qq install lsb-core > /dev/null +if [[ $(lsb_release -r | awk '{ print $2 }') == "14.04" ]]; then + apt-get -y -qq install libav-tools > /dev/null + if [[ ${1} == "2.7" ]]; then + echo "Python 2.7" + elif [[ ${1} == "3.4" ]]; then + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3.4 /usr/local/bin/pip + rm -f /usr/bin/python + ln -s /usr/bin/python3.4 /usr/bin/python + elif [[ ${1} == "3.5" ]]; then + curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.5_pip_packages.sh + sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.5_pip_packages.sh + sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.5_pip_packages.sh + sed -i 's/pip3.5 install/pip3.5 -q install/g' install_python3.5_pip_packages.sh + bash install_python3.5_pip_packages.sh + rm -f install_python3.5_pip_packages.sh + rm -f /usr/bin/python + ln -s /usr/bin/python3.5 /usr/bin/python + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3.5 /usr/local/bin/pip + elif [[ ${1} == "3.6" ]]; then + rm -f /usr/local/bin/pip3 + curl -sOL https://raw.githubusercontent.com/tensorflow/tensorflow/v1.12.0/tensorflow/tools/ci_build/install/install_python3.6_pip_packages.sh + sed -i 's/apt-get update/apt-get -y -qq update/g' install_python3.6_pip_packages.sh + sed -i 's/apt-get install/apt-get -y -qq install/g' install_python3.6_pip_packages.sh + sed -i 's/apt-get upgrade/apt-get -y -qq upgrade/g' install_python3.6_pip_packages.sh + sed -i 's/pip3 install/pip3 -q install/g' install_python3.6_pip_packages.sh + sed -i 's/tar xvf/tar xf/g' install_python3.6_pip_packages.sh + sed -i 's/configure/configure -q/g' install_python3.6_pip_packages.sh + sed -i 's/make altinstall/make altinstall>make.log/g' install_python3.6_pip_packages.sh + sed -i 's/wget /wget -q /g' install_python3.6_pip_packages.sh + bash install_python3.6_pip_packages.sh + rm -f install_python3.6_pip_packages.sh make.log + rm -rf Python-3.6.1* + rm -f /usr/bin/python + ln -s /usr/local/bin/python3.6 /usr/bin/python + rm -f /usr/local/bin/pip + ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip + else + echo Python ${1} not supported! + exit 1 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then + if [[ ${1} == "2.7" ]]; then + echo "Python 2.7" + apt-get -y -qq install ffmpeg python-pip python3-pip patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install -q wheel==0.31.1 + elif [[ ${1} == "3.5" ]]; then + echo "Python 3.5" + apt-get -y -qq install ffmpeg python3-pip patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install wheel==0.31.1 + rm -f /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python + rm -f /usr/bin/pip + ln -s /usr/bin/pip3 /usr/bin/pip + else + echo Platform $(lsb_release -r | awk '{ print $2 }') not supported! + exit 1 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then + if [[ ${1} == "2.7" ]]; then + echo "Python 2.7" + apt-get -y -qq install ffmpeg python-pip python3-pip python3-wheel patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install wheel==0.31.1 + elif [[ ${1} == "3.6" ]]; then + echo "Python 3.6" + apt-get -y -qq install ffmpeg python3-pip python3-wheel patchelf > /dev/null + pip3 install -q auditwheel==1.5.0 + # Pin wheel==0.31.1 to work around issue + # https://github.com/pypa/auditwheel/issues/102 + pip3 install wheel==0.31.1 + rm -f /usr/bin/python + ln -s /usr/bin/python3 /usr/bin/python + rm -f /usr/bin/pip + ln -s /usr/bin/pip3 /usr/bin/pip + else + echo Platform $(lsb_release -r | awk '{ print $2 }') not supported! + exit 1 + fi +else + echo Platform $(lsb_release -r | awk '{ print $2 }') not supported! + exit 1 +fi +python --version +pip --version +pip freeze diff --git a/.travis/python.release.sh b/.travis/python.release.sh index 4341fb435..7cc550149 100755 --- a/.travis/python.release.sh +++ b/.travis/python.release.sh @@ -13,24 +13,39 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -set -e -set -x +set -e -x + # Release: -# docker run -it -e BAZEL_VERSION=$BAZEL_VERSION --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> +# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> +# Nightly: +# export TENSORFLOW_IO_VERSION=0.3.0.dev$(date '+%Y%m%d%H%M%S') +# docker run -i -t --rm -v ${PWD}:/working_dir -w /working_dir tensorflow/tensorflow:custom-op bash -x /working_dir/.travis/python.release.sh <2.7|3.4|3.5|3.6> --project_name tensorflow-io-nightly --project_version $TENSORFLOW_IO_VERSION if [[ -z "${1}" ]]; then echo "usage:" $0 "<2.7|3.4|3.5|3.6>" exit 1 fi -export PYTHON_VERSION=$1 -export BAZEL_VERSION=${BAZEL_VERSION} -export CUSTOM_OP=True -export BAZEL_OS=linux -.travis/python.configure.sh -bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg +BAZEL_VERSION=0.20.0 +PYTHON_VERSION=${1} +shift +TENSORFLOW_VERSION=1.12.0 +if [[ ! -z ${1} ]]; then + TENSORFLOW_VERSION=${1} + shift +fi + +.travis/bazel.install.sh linux ${BAZEL_VERSION} -bazel-bin/build_pip_pkg artifacts +.travis/python.install.sh ${PYTHON_VERSION} + +pip install -q tensorflow==${TENSORFLOW_VERSION} +./configure.sh +if [[ ${TENSORFLOW_IO_TEST} != "no" ]]; then + bazel test --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors -- //tensorflow_io/... +fi +bazel build --noshow_progress --noshow_loading_progress --spawn_strategy standalone --verbose_failures --test_output=errors build_pip_pkg +bazel-bin/build_pip_pkg artifacts "$@" exit 0 diff --git a/.travis/python.test.sh b/.travis/python.test.sh new file mode 100755 index 000000000..fcae6322b --- /dev/null +++ b/.travis/python.test.sh @@ -0,0 +1,57 @@ +#/bin/bash +set -x -e + +PYTHON_VERSION=${1} +shift + +TENSORFLOW_VERSION=1.12.0 +if [[ ! -z ${1} ]]; then + TENSORFLOW_VERSION=${1} + shift +fi + +apt-get -y -qq update +apt-get -y -qq install lsb-core > /dev/null +if [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo Python 2.7 Ubuntu 16.04 + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io*-cp27-*.whl + pip install -q pytest + (cd tests && python -m pytest -s .) + echo Success + exit 0 + elif [[ ${PYTHON_VERSION} == "3.5" ]]; then + echo Python 3.5 Ubuntu 16.04 + apt-get -y -qq install ffmpeg python3-pip > /dev/null + pip3 install -q tensorflow==${TENSORFLOW_VERSION} + pip3 install -q artifacts/tensorflow_io*-cp35-*.whl + pip3 install -q pytest + (cd tests && python3 -m pytest -s .) + echo Success + exit 0 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then + apt-get -y -qq install ffmpeg > /dev/null + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo Python 2.7 Ubuntu 18.04 + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io*-cp27-*.whl + pip install -q pytest + (cd tests && python -m pytest -s .) + echo Success + exit 0 + elif [[ ${PYTHON_VERSION} == "3.6" ]]; then + echo Python 3.6 Ubuntu 18.04 + apt-get -y -qq install ffmpeg python3-pip > /dev/null + pip3 install -q tensorflow==${TENSORFLOW_VERSION} + pip3 install -q artifacts/tensorflow_io*-cp36-*.whl + pip3 install -q pytest + (cd tests && python3 -m pytest -s .) + echo Success + exit 0 + fi +fi +echo Python ${1} on Ubuntu $(lsb_release -r | awk '{ print $2 }') not tested diff --git a/.travis/r.configure.sh b/.travis/r.install.sh similarity index 53% rename from .travis/r.configure.sh rename to .travis/r.install.sh index c3cee13ff..e36912a98 100755 --- a/.travis/r.configure.sh +++ b/.travis/r.install.sh @@ -1,10 +1,13 @@ +#/bin/bash +set -x -e + # Install needed repo -apt-get -y -qqq update -apt-get -y -qqq install build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev -apt-get -y -qqq install apt-transport-https ca-certificates curl gnupg2 software-properties-common +apt-get -y -qq update +apt-get -y -qq install build-essential libcurl4-gnutls-dev libxml2-dev libssl-dev > /dev/null +apt-get -y -qq install apt-transport-https ca-certificates curl gnupg2 software-properties-common > /dev/null apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran35/" -apt-get -y -qqq update -DEBIAN_FRONTEND=nonexteractive apt-get -y -qqq install r-base +apt-get -y -qq update +DEBIAN_FRONTEND=nonexteractive apt-get -y -qqq install r-base > /dev/null echo "options(repos = c(CRAN='http://cran.rstudio.com'))" >> ~/.Rprofile -R -e 'install.packages(c("Rcpp", "reticulate", "knitr", "tensorflow", "tfdatasets", "forge", "tidyselect", "testthat", "devtools"))' +R -e 'install.packages(c("Rcpp", "reticulate", "knitr", "tensorflow", "tfdatasets", "forge", "tidyselect", "testthat", "devtools"), quiet = TRUE)' diff --git a/.travis/r.test.sh b/.travis/r.test.sh new file mode 100755 index 000000000..429e86e5f --- /dev/null +++ b/.travis/r.test.sh @@ -0,0 +1,38 @@ +#/bin/bash +set -x -e + +PYTHON_VERSION=${1} +shift + +TENSORFLOW_VERSION=1.12.0 +if [[ ! -z ${1} ]]; then + TENSORFLOW_VERSION=${1} + shift +fi + +apt-get -y -qq update +apt-get -y -qq install lsb-core > /dev/null +if [[ $(lsb_release -r | awk '{ print $2 }') == "16.04" ]]; then + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo R with Python 2.7 Ubuntu 16.04 + .travis/r.install.sh + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io-*-cp27-*.whl + (cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())$failed == 0L))') + echo Success + exit 0 + fi +elif [[ $(lsb_release -r | awk '{ print $2 }') == "18.04" ]]; then + if [[ ${PYTHON_VERSION} == "2.7" ]]; then + echo R with Python 2.7 Ubuntu 18.04 + .travis/r.install.sh + apt-get -y -qq install ffmpeg python-pip > /dev/null + pip install -q tensorflow==${TENSORFLOW_VERSION} + pip install -q artifacts/tensorflow_io-*-cp27-*.whl + (cd R-package && R -e 'stopifnot(all(data.frame(devtools::test())$failed == 0L))') + echo Success + exit 0 + fi +fi +echo R with Python ${1} on Ubuntu $(lsb_release -r | awk '{ print $2 }') not tested diff --git a/build_pip_pkg.sh b/build_pip_pkg.sh index f523270f5..f308f980b 100755 --- a/build_pip_pkg.sh +++ b/build_pip_pkg.sh @@ -25,6 +25,12 @@ function main() { if [[ ${1} == "make" ]]; then echo "Using Makefile to build pip package." PIP_FILE_PREFIX="" + elif [[ ${1} == "--project_name" ]]; then + shift + PROJECT_NAME="--project_name ${1}" + elif [[ ${1} == "--project_version" ]]; then + shift + PROJECT_VERSION="--project_version ${1}" else DEST=${1} fi @@ -62,7 +68,7 @@ function main() { pushd ${TMPDIR} echo $(date) : "=== Building wheel" - python setup.py bdist_wheel + python setup.py bdist_wheel ${PROJECT_NAME} ${PROJECT_VERSION} auditwheel repair -w "${DEST}" dist/*.whl popd diff --git a/setup.py b/setup.py index 26eb43ad7..02484078a 100644 --- a/setup.py +++ b/setup.py @@ -17,15 +17,28 @@ from __future__ import division from __future__ import print_function +import sys + from setuptools import find_packages from setuptools import setup from setuptools.dist import Distribution -__version__ = '0.2.0' REQUIRED_PACKAGES = [ 'tensorflow == 1.12.0', ] +__version__ = '0.3.0' +if '--project_version' in sys.argv: + project_version_idx = sys.argv.index('--project_version') + __version__ = sys.argv[project_version_idx + 1] + sys.argv.remove('--project_version') + sys.argv.pop(project_version_idx) + project_name = 'tensorflow-io' +if '--project_name' in sys.argv: + project_name_idx = sys.argv.index('--project_name') + project_name = sys.argv[project_name_idx + 1] + sys.argv.remove('--project_name') + sys.argv.pop(project_name_idx) class BinaryDistribution(Distribution): """This class is needed in order to create OS specific wheels."""