diff --git a/.travis/install-dependencies.sh b/.travis/install-dependencies.sh index 5bae4ba87f8d..c247ee92cc5c 100755 --- a/.travis/install-dependencies.sh +++ b/.travis/install-dependencies.sh @@ -24,7 +24,7 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ + pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then sudo apt-get update @@ -33,7 +33,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ + pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed @@ -50,7 +50,7 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ + pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed @@ -67,7 +67,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ + pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \ feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout elif [[ "$LINT" == "1" ]]; then sudo apt-get update diff --git a/cmake/Modules/ArrowExternalProject.cmake b/cmake/Modules/ArrowExternalProject.cmake index 2add25a94887..14a88f6a8c96 100644 --- a/cmake/Modules/ArrowExternalProject.cmake +++ b/cmake/Modules/ArrowExternalProject.cmake @@ -15,10 +15,10 @@ # - PLASMA_SHARED_LIB set(arrow_URL https://github.com/apache/arrow.git) -# The PR for this commit is https://github.com/apache/arrow/pull/3117. We +# The PR for this commit is https://github.com/apache/arrow/pull/3124. We # include the link here to make it easier to find the right commit because # Arrow often rewrites git history and invalidates certain commits. -set(arrow_TAG 4d810b4a9c37e79fde6b134ac90ee0c5f7f6c9bf) +set(arrow_TAG b3bc3384f3068edebe69f1084518ccfb85a368f8) set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install) set(ARROW_HOME ${ARROW_INSTALL_PREFIX}) diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 56cff8de0ac5..b7fa7be6b9cf 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -26,6 +26,6 @@ RUN apt-get update \ && /opt/conda/bin/conda clean -y --all \ && /opt/conda/bin/pip install \ flatbuffers \ - cython==0.27.3 + cython==0.29.0 ENV PATH "/opt/conda/bin:$PATH" diff --git a/java/doc/installation.rst b/java/doc/installation.rst index 8daec29ace40..54be4094be48 100644 --- a/java/doc/installation.rst +++ b/java/doc/installation.rst @@ -26,7 +26,7 @@ For Ubuntu users, run the following commands: # If you are on Ubuntu 14.04, you need the following. pip install cmake - pip install cython==0.27.3 + pip install cython==0.29.0 For macOS users, run the following commands: :: @@ -34,7 +34,7 @@ For macOS users, run the following commands: brew update brew install maven cmake pkg-config automake autoconf libtool openssl bison wget - pip install cython==0.27.3 + pip install cython==0.29.0 Build Ray ^^^^^^^^^ diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 30e8b1936376..1b29cd77415b 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -70,7 +70,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PIP_CMD install -q setuptools_scm==2.1.0 # Fix the numpy version because this will be the oldest numpy version we can # support. - $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.27.3 + $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.0 # Install wheel to avoid the error "invalid command 'bdist_wheel'". $PIP_CMD install -q wheel # Add the correct Python to the path and build the wheel. This is only diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux1.sh index db31ff55a4e6..500fe5c491fb 100755 --- a/python/build-wheel-manylinux1.sh +++ b/python/build-wheel-manylinux1.sh @@ -21,7 +21,7 @@ for PYTHON in cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m; do pushd python # Fix the numpy version because this will be the oldest numpy version we can # support. - /opt/python/${PYTHON}/bin/pip install -q numpy==1.10.4 cython==0.27.3 + /opt/python/${PYTHON}/bin/pip install -q numpy==1.10.4 cython==0.29.0 INCLUDE_UI=1 PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel # In the future, run auditwheel here. mv dist/*.whl ../.whl/ diff --git a/python/ray/autoscaler/aws/development-example.yaml b/python/ray/autoscaler/aws/development-example.yaml index 273d1d7d9459..b9791799ff04 100644 --- a/python/ray/autoscaler/aws/development-example.yaml +++ b/python/ray/autoscaler/aws/development-example.yaml @@ -94,7 +94,7 @@ setup_commands: - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc # Build Ray. - git clone https://github.com/ray-project/ray || true - - pip install boto3==1.4.8 cython==0.27.3 + - pip install boto3==1.4.8 cython==0.29.0 - cd ray/python; pip install -e . --verbose # Custom commands that will be run on the head node after common setup. diff --git a/python/ray/autoscaler/gcp/example-full.yaml b/python/ray/autoscaler/gcp/example-full.yaml index 6afbb464fa6a..32ac1fa12ff8 100644 --- a/python/ray/autoscaler/gcp/example-full.yaml +++ b/python/ray/autoscaler/gcp/example-full.yaml @@ -123,7 +123,7 @@ setup_commands: - >- pip install google-api-python-client==1.6.7 - cython==0.27.3 + cython==0.29.0 # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp27-cp27mu-manylinux1_x86_64.whl # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp35-cp35m-manylinux1_x86_64.whl # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp36-cp36m-manylinux1_x86_64.whl diff --git a/python/ray/worker.py b/python/ray/worker.py index 06f7f03d2520..cd0e56fe1871 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -2061,7 +2061,7 @@ def connect(info, # Create an object store client. worker.plasma_client = thread_safe_client( - plasma.connect(info["store_socket_name"], "", 64)) + plasma.connect(info["store_socket_name"], "")) raylet_socket = info["raylet_socket_name"] diff --git a/python/setup.py b/python/setup.py index a323501d3b43..ee0bd2524570 100644 --- a/python/setup.py +++ b/python/setup.py @@ -165,7 +165,7 @@ def find_version(*filepath): # The BinaryDistribution argument triggers build_ext. distclass=BinaryDistribution, install_requires=requires, - setup_requires=["cython >= 0.27, < 0.28"], + setup_requires=["cython >= 0.29"], extras_require=extras, entry_points={ "console_scripts": [ diff --git a/src/ray/object_manager/object_buffer_pool.cc b/src/ray/object_manager/object_buffer_pool.cc index 5f101e795920..fde3221dc3f9 100644 --- a/src/ray/object_manager/object_buffer_pool.cc +++ b/src/ray/object_manager/object_buffer_pool.cc @@ -3,10 +3,10 @@ namespace ray { ObjectBufferPool::ObjectBufferPool(const std::string &store_socket_name, - uint64_t chunk_size, int release_delay) + uint64_t chunk_size) : default_chunk_size_(chunk_size) { store_socket_name_ = store_socket_name; - ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str(), "", release_delay)); + ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str(), "")); } ObjectBufferPool::~ObjectBufferPool() { diff --git a/src/ray/object_manager/object_buffer_pool.h b/src/ray/object_manager/object_buffer_pool.h index ed6594ed4b49..e4790dfe537c 100644 --- a/src/ray/object_manager/object_buffer_pool.h +++ b/src/ray/object_manager/object_buffer_pool.h @@ -40,10 +40,7 @@ class ObjectBufferPool { /// \param store_socket_name The socket name of the store to which plasma clients /// connect. /// \param chunk_size The chunk size into which objects are to be split. - /// \param release_delay The number of release calls before objects are released - /// from the store client (FIFO). - ObjectBufferPool(const std::string &store_socket_name, const uint64_t chunk_size, - const int release_delay); + ObjectBufferPool(const std::string &store_socket_name, const uint64_t chunk_size); ~ObjectBufferPool(); diff --git a/src/ray/object_manager/object_manager.cc b/src/ray/object_manager/object_manager.cc index 5f6a503b6f07..6afc5f3dc2c7 100644 --- a/src/ray/object_manager/object_manager.cc +++ b/src/ray/object_manager/object_manager.cc @@ -14,10 +14,7 @@ ObjectManager::ObjectManager(asio::io_service &main_service, : config_(config), object_directory_(std::move(object_directory)), store_notification_(main_service, config_.store_socket_name), - // release_delay of 2 * config_.max_sends is to ensure the pool does not release - // an object prematurely whenever we reach the maximum number of sends. - buffer_pool_(config_.store_socket_name, config_.object_chunk_size, - /*release_delay=*/2 * config_.max_sends), + buffer_pool_(config_.store_socket_name, config_.object_chunk_size), send_work_(send_service_), receive_work_(receive_service_), connection_pool_(), diff --git a/src/ray/object_manager/object_store_notification_manager.cc b/src/ray/object_manager/object_store_notification_manager.cc index e590e8efa549..dc846f536068 100644 --- a/src/ray/object_manager/object_store_notification_manager.cc +++ b/src/ray/object_manager/object_store_notification_manager.cc @@ -15,8 +15,7 @@ namespace ray { ObjectStoreNotificationManager::ObjectStoreNotificationManager( boost::asio::io_service &io_service, const std::string &store_socket_name) : store_client_(), socket_(io_service) { - ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str(), "", - plasma::kPlasmaDefaultReleaseDelay)); + ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str(), "")); ARROW_CHECK_OK(store_client_.Subscribe(&c_socket_)); boost::system::error_code ec; diff --git a/src/ray/object_manager/test/object_manager_stress_test.cc b/src/ray/object_manager/test/object_manager_stress_test.cc index 73121ada867a..d16503d9a7ff 100644 --- a/src/ray/object_manager/test/object_manager_stress_test.cc +++ b/src/ray/object_manager/test/object_manager_stress_test.cc @@ -154,8 +154,8 @@ class TestObjectManagerBase : public ::testing::Test { server2.reset(new MockServer(main_service, om_config_2, gcs_client_2)); // connect to stores. - ARROW_CHECK_OK(client1.Connect(store_id_1, "", plasma::kPlasmaDefaultReleaseDelay)); - ARROW_CHECK_OK(client2.Connect(store_id_2, "", plasma::kPlasmaDefaultReleaseDelay)); + ARROW_CHECK_OK(client1.Connect(store_id_1, "")); + ARROW_CHECK_OK(client2.Connect(store_id_2, "")); } void TearDown() { diff --git a/src/ray/object_manager/test/object_manager_test.cc b/src/ray/object_manager/test/object_manager_test.cc index 4184bca2b8aa..2d1e5a6a611c 100644 --- a/src/ray/object_manager/test/object_manager_test.cc +++ b/src/ray/object_manager/test/object_manager_test.cc @@ -139,8 +139,8 @@ class TestObjectManagerBase : public ::testing::Test { server2.reset(new MockServer(main_service, om_config_2, gcs_client_2)); // connect to stores. - ARROW_CHECK_OK(client1.Connect(store_id_1, "", plasma::kPlasmaDefaultReleaseDelay)); - ARROW_CHECK_OK(client2.Connect(store_id_2, "", plasma::kPlasmaDefaultReleaseDelay)); + ARROW_CHECK_OK(client1.Connect(store_id_1, "")); + ARROW_CHECK_OK(client2.Connect(store_id_2, "")); } void TearDown() { diff --git a/src/ray/raylet/object_manager_integration_test.cc b/src/ray/raylet/object_manager_integration_test.cc index 83c7a9f8f8be..a83c822231e1 100644 --- a/src/ray/raylet/object_manager_integration_test.cc +++ b/src/ray/raylet/object_manager_integration_test.cc @@ -74,8 +74,8 @@ class TestObjectManagerBase : public ::testing::Test { GetNodeManagerConfig("raylet_2", store_sock_2), om_config_2, gcs_client_2)); // connect to stores. - ARROW_CHECK_OK(client1.Connect(store_sock_1, "", plasma::kPlasmaDefaultReleaseDelay)); - ARROW_CHECK_OK(client2.Connect(store_sock_2, "", plasma::kPlasmaDefaultReleaseDelay)); + ARROW_CHECK_OK(client1.Connect(store_sock_1, "")); + ARROW_CHECK_OK(client2.Connect(store_sock_2, "")); } void TearDown() { diff --git a/test/stress_tests/stress_testing_config.yaml b/test/stress_tests/stress_testing_config.yaml index 6ef5285035e8..0887c9426a1b 100644 --- a/test/stress_tests/stress_testing_config.yaml +++ b/test/stress_tests/stress_testing_config.yaml @@ -94,7 +94,7 @@ setup_commands: - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc # # Build Ray. # - git clone https://github.com/ray-project/ray || true - - pip install boto3==1.4.8 cython==0.27.3 + - pip install boto3==1.4.8 cython==0.29.0 # - cd ray/python; git checkout master; git pull; pip install -e . --verbose - pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp36-cp36m-manylinux1_x86_64.whl