Update arrow to reduce plasma IPCs.

.travis/install-dependencies.sh

@@ -24,7 +24,7 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
@@ -33,7 +33,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
@@ -50,7 +50,7 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
@@ -67,7 +67,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$LINT" == "1" ]]; then
   sudo apt-get update

cmake/Modules/ArrowExternalProject.cmake

@@ -15,10 +15,10 @@
 #  - PLASMA_SHARED_LIB
 
 set(arrow_URL https://github.com/apache/arrow.git)
-# The PR for this commit is https://github.com/apache/arrow/pull/3117. We
+# The PR for this commit is https://github.com/apache/arrow/pull/3124. We
 # include the link here to make it easier to find the right commit because
 # Arrow often rewrites git history and invalidates certain commits.
-set(arrow_TAG 4d810b4a9c37e79fde6b134ac90ee0c5f7f6c9bf)
+set(arrow_TAG b3bc3384f3068edebe69f1084518ccfb85a368f8)
 
 set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
 set(ARROW_HOME ${ARROW_INSTALL_PREFIX})

docker/base-deps/Dockerfile

@@ -26,6 +26,6 @@ RUN apt-get update \
     && /opt/conda/bin/conda clean -y --all \
     && /opt/conda/bin/pip install \
         flatbuffers \
-        cython==0.27.3
+        cython==0.29.0
 
 ENV PATH "/opt/conda/bin:$PATH"

java/doc/installation.rst

@@ -26,15 +26,15 @@ For Ubuntu users, run the following commands:
   # If you are on Ubuntu 14.04, you need the following.
   pip install cmake
 
-  pip install cython==0.27.3
+  pip install cython==0.29.0
 
 For macOS users, run the following commands:
 ::
 
   brew update
   brew install maven cmake pkg-config automake autoconf libtool openssl bison wget
 
-  pip install cython==0.27.3
+  pip install cython==0.29.0
 
 Build Ray
 ^^^^^^^^^

python/build-wheel-macos.sh

@@ -70,7 +70,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     $PIP_CMD install -q setuptools_scm==2.1.0
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.27.3
+    $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.0
     # Install wheel to avoid the error "invalid command 'bdist_wheel'".
     $PIP_CMD install -q wheel
     # Add the correct Python to the path and build the wheel. This is only

python/build-wheel-manylinux1.sh

@@ -21,7 +21,7 @@ for PYTHON in cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m; do
   pushd python
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
-    /opt/python/${PYTHON}/bin/pip install -q numpy==1.10.4 cython==0.27.3
+    /opt/python/${PYTHON}/bin/pip install -q numpy==1.10.4 cython==0.29.0
     INCLUDE_UI=1 PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel
     # In the future, run auditwheel here.
     mv dist/*.whl ../.whl/

python/ray/autoscaler/aws/development-example.yaml

@@ -94,7 +94,7 @@ setup_commands:
     - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
     # Build Ray.
     - git clone https://github.com/ray-project/ray || true
-    - pip install boto3==1.4.8 cython==0.27.3
+    - pip install boto3==1.4.8 cython==0.29.0
     - cd ray/python; pip install -e . --verbose
 
 # Custom commands that will be run on the head node after common setup.

python/ray/autoscaler/gcp/example-full.yaml

@@ -123,7 +123,7 @@ setup_commands:
     - >-
       pip install
       google-api-python-client==1.6.7
-      cython==0.27.3
+      cython==0.29.0
     # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp27-cp27mu-manylinux1_x86_64.whl
     # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp35-cp35m-manylinux1_x86_64.whl
     # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp36-cp36m-manylinux1_x86_64.whl

python/ray/worker.py

@@ -2061,7 +2061,7 @@ def connect(info,
 
     # Create an object store client.
     worker.plasma_client = thread_safe_client(
-        plasma.connect(info["store_socket_name"], "", 64))
+        plasma.connect(info["store_socket_name"], ""))
 
     raylet_socket = info["raylet_socket_name"]
 

python/setup.py

@@ -165,7 +165,7 @@ def find_version(*filepath):
     # The BinaryDistribution argument triggers build_ext.
     distclass=BinaryDistribution,
     install_requires=requires,
-    setup_requires=["cython >= 0.27, < 0.28"],
+    setup_requires=["cython >= 0.29"],
     extras_require=extras,
     entry_points={
         "console_scripts": [

src/ray/object_manager/object_buffer_pool.cc

@@ -3,10 +3,10 @@
 namespace ray {
 
 ObjectBufferPool::ObjectBufferPool(const std::string &store_socket_name,
-                                   uint64_t chunk_size, int release_delay)
+                                   uint64_t chunk_size)
     : default_chunk_size_(chunk_size) {
   store_socket_name_ = store_socket_name;
-  ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str(), "", release_delay));
+  ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str(), ""));
 }
 
 ObjectBufferPool::~ObjectBufferPool() {

src/ray/object_manager/object_buffer_pool.h

@@ -40,10 +40,7 @@ class ObjectBufferPool {
   /// \param store_socket_name The socket name of the store to which plasma clients
   /// connect.
   /// \param chunk_size The chunk size into which objects are to be split.
-  /// \param release_delay The number of release calls before objects are released
-  /// from the store client (FIFO).
-  ObjectBufferPool(const std::string &store_socket_name, const uint64_t chunk_size,
-                   const int release_delay);
+  ObjectBufferPool(const std::string &store_socket_name, const uint64_t chunk_size);
 
   ~ObjectBufferPool();
 

src/ray/object_manager/object_manager.cc

@@ -14,10 +14,7 @@ ObjectManager::ObjectManager(asio::io_service &main_service,
     : config_(config),
       object_directory_(std::move(object_directory)),
       store_notification_(main_service, config_.store_socket_name),
-      // release_delay of 2 * config_.max_sends is to ensure the pool does not release
-      // an object prematurely whenever we reach the maximum number of sends.
-      buffer_pool_(config_.store_socket_name, config_.object_chunk_size,
-                   /*release_delay=*/2 * config_.max_sends),
+      buffer_pool_(config_.store_socket_name, config_.object_chunk_size),
       send_work_(send_service_),
       receive_work_(receive_service_),
       connection_pool_(),

src/ray/object_manager/object_store_notification_manager.cc

@@ -15,8 +15,7 @@ namespace ray {
 ObjectStoreNotificationManager::ObjectStoreNotificationManager(
     boost::asio::io_service &io_service, const std::string &store_socket_name)
     : store_client_(), socket_(io_service) {
-  ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str(), "",
-                                       plasma::kPlasmaDefaultReleaseDelay));
+  ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str(), ""));
 
   ARROW_CHECK_OK(store_client_.Subscribe(&c_socket_));
   boost::system::error_code ec;

src/ray/object_manager/test/object_manager_stress_test.cc

@@ -154,8 +154,8 @@ class TestObjectManagerBase : public ::testing::Test {
     server2.reset(new MockServer(main_service, om_config_2, gcs_client_2));
 
     // connect to stores.
-    ARROW_CHECK_OK(client1.Connect(store_id_1, "", plasma::kPlasmaDefaultReleaseDelay));
-    ARROW_CHECK_OK(client2.Connect(store_id_2, "", plasma::kPlasmaDefaultReleaseDelay));
+    ARROW_CHECK_OK(client1.Connect(store_id_1, ""));
+    ARROW_CHECK_OK(client2.Connect(store_id_2, ""));
   }
 
   void TearDown() {

src/ray/object_manager/test/object_manager_test.cc

@@ -139,8 +139,8 @@ class TestObjectManagerBase : public ::testing::Test {
     server2.reset(new MockServer(main_service, om_config_2, gcs_client_2));
 
     // connect to stores.
-    ARROW_CHECK_OK(client1.Connect(store_id_1, "", plasma::kPlasmaDefaultReleaseDelay));
-    ARROW_CHECK_OK(client2.Connect(store_id_2, "", plasma::kPlasmaDefaultReleaseDelay));
+    ARROW_CHECK_OK(client1.Connect(store_id_1, ""));
+    ARROW_CHECK_OK(client2.Connect(store_id_2, ""));
   }
 
   void TearDown() {

src/ray/raylet/object_manager_integration_test.cc

@@ -74,8 +74,8 @@ class TestObjectManagerBase : public ::testing::Test {
         GetNodeManagerConfig("raylet_2", store_sock_2), om_config_2, gcs_client_2));
 
     // connect to stores.
-    ARROW_CHECK_OK(client1.Connect(store_sock_1, "", plasma::kPlasmaDefaultReleaseDelay));
-    ARROW_CHECK_OK(client2.Connect(store_sock_2, "", plasma::kPlasmaDefaultReleaseDelay));
+    ARROW_CHECK_OK(client1.Connect(store_sock_1, ""));
+    ARROW_CHECK_OK(client2.Connect(store_sock_2, ""));
   }
 
   void TearDown() {

test/stress_tests/stress_testing_config.yaml

@@ -94,7 +94,7 @@ setup_commands:
     - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
     # # Build Ray.
     # - git clone https://github.com/ray-project/ray || true
-    - pip install boto3==1.4.8 cython==0.27.3
+    - pip install boto3==1.4.8 cython==0.29.0
     # - cd ray/python; git checkout master; git pull; pip install -e . --verbose
     - pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp36-cp36m-manylinux1_x86_64.whl