diff --git a/.travis/install-dependencies.sh b/.travis/install-dependencies.sh index 2050cdf69eff..ac2ac879a723 100755 --- a/.travis/install-dependencies.sh +++ b/.travis/install-dependencies.sh @@ -19,20 +19,20 @@ fi if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then sudo apt-get update - sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip + sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers + pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then sudo apt-get update - sudo apt-get install -y cmake python-dev python-numpy build-essential autoconf curl libtool libboost-all-dev unzip + sudo apt-get install -y cmake pkg-config python-dev python-numpy build-essential autoconf curl libtool libboost-all-dev unzip # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers + pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed which -s brew @@ -43,12 +43,12 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then echo "Updating brew." brew update fi - brew install cmake automake autoconf libtool boost + brew install cmake pkg-config automake autoconf libtool boost # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers + pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then # check that brew is installed which -s brew @@ -59,12 +59,12 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then echo "Updating brew." brew update fi - brew install cmake automake autoconf libtool boost + brew install cmake pkg-config automake autoconf libtool boost # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" - pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers + pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers elif [[ "$LINT" == "1" ]]; then sudo apt-get update sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c2bc929453f..73f8e2ba9c6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 2.8) project(ray) +set(ARROW_DIR "${CMAKE_CURRENT_LIST_DIR}/src/thirdparty/arrow/" + CACHE STRING "Path of the arrow source directory") + +include_directories("${ARROW_DIR}/cpp/src/") + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/) add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/) add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/) diff --git a/build.sh b/build.sh index 975d48106510..43bd1d13aa2d 100755 --- a/build.sh +++ b/build.sh @@ -1,10 +1,19 @@ #!/usr/bin/env bash +set -x + # Cause the script to exit if a single command fails. set -e ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +if [[ -z "$1" ]]; then + PYTHON_EXECUTABLE=`which python` +else + PYTHON_EXECUTABLE=$1 +fi +echo "Using Python executable $PYTHON_EXECUTABLE." + # Determine how many parallel jobs to use for make based on the number of cores unamestr="$(uname)" if [[ "$unamestr" == "Linux" ]]; then @@ -20,17 +29,25 @@ pushd "$ROOT_DIR/src/common/thirdparty/" bash build-redis.sh popd -bash "$ROOT_DIR/src/numbuf/thirdparty/download_thirdparty.sh" -bash "$ROOT_DIR/src/numbuf/thirdparty/build_thirdparty.sh" +bash "$ROOT_DIR/src/thirdparty/download_thirdparty.sh" +bash "$ROOT_DIR/src/thirdparty/build_thirdparty.sh" $PYTHON_EXECUTABLE # Now build everything. pushd "$ROOT_DIR/python/ray/core" + # We use these variables to set PKG_CONFIG_PATH, which is important so that + # in cmake, pkg-config can find plasma. + TP_DIR=$ROOT_DIR/src/thirdparty + ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install if [ "$VALGRIND" = "1" ] then - cmake -DCMAKE_BUILD_TYPE=Debug ../../.. + PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=Debug ../../.. else - cmake -DCMAKE_BUILD_TYPE=Release ../../.. + PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=Release ../../.. fi make clean make -j${PARALLEL} popd + +# Move stuff from Arrow to Ray. + +mv $ROOT_DIR/src/thirdparty/arrow/cpp/build/release/plasma_store $ROOT_DIR/python/ray/core/src/plasma/ diff --git a/doc/source/conf.py b/doc/source/conf.py index dfee7f9af5c4..c3e5a0866a95 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -18,7 +18,9 @@ # These lines added to enable Sphinx to work without installing Ray. import mock -MOCK_MODULES = ["ray.numbuf", +MOCK_MODULES = ["pyarrow", + "pyarrow.plasma", + "ray.numbuf", "ray.local_scheduler", "ray.plasma", "ray.core.generated.TaskInfo", diff --git a/doc/source/install-on-macosx.rst b/doc/source/install-on-macosx.rst index 247aa6d90735..1e5a66b92859 100644 --- a/doc/source/install-on-macosx.rst +++ b/doc/source/install-on-macosx.rst @@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using .. code-block:: bash brew update - brew install cmake automake autoconf libtool boost wget + brew install cmake pkg-config automake autoconf libtool boost wget pip install numpy cloudpickle funcsigs click colorama psutil redis flatbuffers --ignore-installed six diff --git a/doc/source/install-on-ubuntu.rst b/doc/source/install-on-ubuntu.rst index 608b16287498..ba422a4e5200 100644 --- a/doc/source/install-on-ubuntu.rst +++ b/doc/source/install-on-ubuntu.rst @@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using .. code-block:: bash sudo apt-get update - sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip + sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool libboost-all-dev unzip # If you are not using Anaconda, you need the following. sudo apt-get install python-dev # For Python 2. diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 5dabbfa44983..7ddb94f339cd 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:xenial RUN apt-get update \ && apt-get install -y vim git wget \ - && apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip + && apt-get install -y cmake pkg-config build-essential autoconf curl libtool libboost-all-dev unzip RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \ diff --git a/python/ray/experimental/state.py b/python/ray/experimental/state.py index 72b3956d8527..71ce78cbc39c 100644 --- a/python/ray/experimental/state.py +++ b/python/ray/experimental/state.py @@ -465,7 +465,7 @@ def task_profiles(self, start=None, end=None, num_tasks=None, fwd=True): **params) for (event, score) in event_list: - event_dict = json.loads(event) + event_dict = json.loads(event.decode()) task_id = "" for event in event_dict: if "task_id" in event[3]: diff --git a/python/ray/global_scheduler/test/test.py b/python/ray/global_scheduler/test/test.py index 7b4d193d92da..18634d768111 100644 --- a/python/ray/global_scheduler/test/test.py +++ b/python/ray/global_scheduler/test/test.py @@ -10,6 +10,7 @@ import time import unittest +import pyarrow as pa import ray.global_scheduler as global_scheduler import ray.local_scheduler as local_scheduler import ray.plasma as plasma @@ -87,8 +88,8 @@ def setUp(self): self.plasma_manager_pids.append(p3) plasma_address = "{}:{}".format(self.node_ip_address, plasma_manager_port) - plasma_client = plasma.PlasmaClient(plasma_store_name, - plasma_manager_name) + plasma_client = pa.plasma.connect(plasma_store_name, + plasma_manager_name, 64) self.plasma_clients.append(plasma_client) # Start the local scheduler. local_scheduler_name, p4 = local_scheduler.start_local_scheduler( @@ -203,9 +204,10 @@ def test_integration_single_task(self): # Sleep before submitting task to local scheduler. time.sleep(0.1) # Submit a task to Redis. - task = local_scheduler.Task(random_driver_id(), random_function_id(), - [local_scheduler.ObjectID(object_dep)], - num_return_vals[0], random_task_id(), 0) + task = local_scheduler.Task( + random_driver_id(), random_function_id(), + [local_scheduler.ObjectID(object_dep.binary())], + num_return_vals[0], random_task_id(), 0) self.local_scheduler_clients[0].submit(task) time.sleep(0.1) # There should now be a task in Redis, and it should get assigned to @@ -256,11 +258,11 @@ def integration_many_tasks_helper(self, timesync=True): # Give 10ms for object info handler to fire (long enough to # yield CPU). time.sleep(0.010) - task = local_scheduler.Task(random_driver_id(), - random_function_id(), - [local_scheduler.ObjectID(object_dep)], - num_return_vals[0], random_task_id(), - 0) + task = local_scheduler.Task( + random_driver_id(), + random_function_id(), + [local_scheduler.ObjectID(object_dep.binary())], + num_return_vals[0], random_task_id(), 0) self.local_scheduler_clients[0].submit(task) # Check that there are the correct number of tasks in Redis and that # they all get assigned to the local scheduler. diff --git a/python/ray/local_scheduler/test/test.py b/python/ray/local_scheduler/test/test.py index 6577119f546c..dd9f45f05aba 100644 --- a/python/ray/local_scheduler/test/test.py +++ b/python/ray/local_scheduler/test/test.py @@ -12,6 +12,7 @@ import ray.local_scheduler as local_scheduler import ray.plasma as plasma +import pyarrow as pa USE_VALGRIND = False ID_SIZE = 20 @@ -41,8 +42,7 @@ class TestLocalSchedulerClient(unittest.TestCase): def setUp(self): # Start Plasma store. plasma_store_name, self.p1 = plasma.start_plasma_store() - self.plasma_client = plasma.PlasmaClient(plasma_store_name, - release_delay=0) + self.plasma_client = pa.plasma.connect(plasma_store_name, "", 0) # Start a local scheduler. scheduler_name, self.p2 = local_scheduler.start_local_scheduler( plasma_store_name, use_valgrind=USE_VALGRIND) @@ -72,8 +72,8 @@ def test_submit_and_get_task(self): # Create and seal the objects in the object store so that we can # schedule all of the subsequent tasks. for object_id in object_ids: - self.plasma_client.create(object_id.id(), 0) - self.plasma_client.seal(object_id.id()) + self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0) + self.plasma_client.seal(pa.plasma.ObjectID(object_id.id())) # Define some arguments to use for the tasks. args_list = [ [], @@ -153,8 +153,8 @@ def get_task(): time.sleep(0.1) # Create and seal the object ID in the object store. This should # trigger a scheduling event. - self.plasma_client.create(object_id.id(), 0) - self.plasma_client.seal(object_id.id()) + self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0) + self.plasma_client.seal(pa.plasma.ObjectID(object_id.id())) # Wait until the thread finishes so that we know the task was # scheduled. t.join() @@ -175,8 +175,8 @@ def get_task(): t.start() # Make one of the dependencies available. - buf = self.plasma_client.create(object_id1.id(), 1) - self.plasma_client.seal(object_id1.id()) + buf = self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1) + self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id())) # Release the object. del buf # Check that the thread is still waiting for a task. @@ -188,23 +188,24 @@ def get_task(): time.sleep(0.1) self.assertTrue(t.is_alive()) # Check that the first object dependency was evicted. - object1 = self.plasma_client.get([object_id1.id()], timeout_ms=0) + object1 = self.plasma_client.get([pa.plasma.ObjectID(object_id1.id())], + timeout_ms=0) self.assertEqual(object1, [None]) # Check that the thread is still waiting for a task. time.sleep(0.1) self.assertTrue(t.is_alive()) # Create the second dependency. - self.plasma_client.create(object_id2.id(), 1) - self.plasma_client.seal(object_id2.id()) + self.plasma_client.create(pa.plasma.ObjectID(object_id2.id()), 1) + self.plasma_client.seal(pa.plasma.ObjectID(object_id2.id())) # Check that the thread is still waiting for a task. time.sleep(0.1) self.assertTrue(t.is_alive()) # Create the first dependency again. Both dependencies are now # available. - self.plasma_client.create(object_id1.id(), 1) - self.plasma_client.seal(object_id1.id()) + self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1) + self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id())) # Wait until the thread finishes so that we know the task was # scheduled. diff --git a/python/ray/plasma/__init__.py b/python/ray/plasma/__init__.py index c5238bd50aff..6c3dd7b0aaad 100644 --- a/python/ray/plasma/__init__.py +++ b/python/ray/plasma/__init__.py @@ -2,13 +2,8 @@ from __future__ import division from __future__ import print_function -from ray.plasma.plasma import (PlasmaBuffer, buffers_equal, PlasmaClient, - start_plasma_store, start_plasma_manager, - plasma_object_exists_error, - plasma_out_of_memory_error, +from ray.plasma.plasma import (start_plasma_store, start_plasma_manager, DEFAULT_PLASMA_STORE_MEMORY) -__all__ = ["PlasmaBuffer", "buffers_equal", "PlasmaClient", - "start_plasma_store", "start_plasma_manager", - "plasma_object_exists_error", "plasma_out_of_memory_error", +__all__ = ["start_plasma_store", "start_plasma_manager", "DEFAULT_PLASMA_STORE_MEMORY"] diff --git a/python/ray/plasma/plasma.py b/python/ray/plasma/plasma.py index b7f0aefdedde..5c0a0243f38f 100644 --- a/python/ray/plasma/plasma.py +++ b/python/ray/plasma/plasma.py @@ -5,315 +5,14 @@ import os import random import subprocess -import sys import time -import ray.core.src.plasma.libplasma as libplasma -from ray.core.src.plasma.libplasma import plasma_object_exists_error -from ray.core.src.plasma.libplasma import plasma_out_of_memory_error -__all__ = ["PlasmaBuffer", "buffers_equal", "PlasmaClient", - "start_plasma_store", "start_plasma_manager", - "plasma_object_exists_error", "plasma_out_of_memory_error", +__all__ = ["start_plasma_store", "start_plasma_manager", "DEFAULT_PLASMA_STORE_MEMORY"] PLASMA_WAIT_TIMEOUT = 2 ** 30 - -class PlasmaBuffer(object): - """This is the type returned by calls to get with a PlasmaClient. - - We define our own class instead of directly returning a buffer object so - that we can add a custom destructor which notifies Plasma that the object - is no longer being used, so the memory in the Plasma store backing the - object can potentially be freed. - - Attributes: - buffer (buffer): A buffer containing an object in the Plasma store. - plasma_id (PlasmaID): The ID of the object in the buffer. - plasma_client (PlasmaClient): The PlasmaClient that we use to communicate - with the store and manager. - """ - def __init__(self, buff, plasma_id, plasma_client): - """Initialize a PlasmaBuffer.""" - self.buffer = buff - self.plasma_id = plasma_id - self.plasma_client = plasma_client - - def __del__(self): - """Notify Plasma that the object is no longer needed. - - If the plasma client has been shut down, then don't do anything. - """ - if self.plasma_client.alive: - libplasma.release(self.plasma_client.conn, self.plasma_id) - - def __getitem__(self, index): - """Read from the PlasmaBuffer as if it were just a regular buffer.""" - # We currently don't allow slicing plasma buffers. We should handle - # this better, but it requires some care because the slice may be - # backed by the same memory in the object store, but the original - # plasma buffer may go out of scope causing the memory to no longer be - # accessible. - assert not isinstance(index, slice) - value = self.buffer[index] - if sys.version_info >= (3, 0) and not isinstance(index, slice): - value = chr(value) - return value - - def __setitem__(self, index, value): - """Write to the PlasmaBuffer as if it were just a regular buffer. - - This should fail because the buffer should be read only. - """ - # We currently don't allow slicing plasma buffers. We should handle - # this better, but it requires some care because the slice may be - # backed by the same memory in the object store, but the original - # plasma buffer may go out of scope causing the memory to no longer be - # accessible. - assert not isinstance(index, slice) - if sys.version_info >= (3, 0) and not isinstance(index, slice): - value = ord(value) - self.buffer[index] = value - - def __len__(self): - """Return the length of the buffer.""" - return len(self.buffer) - - -def buffers_equal(buff1, buff2): - """Compare two buffers. These buffers may be PlasmaBuffer objects. - - This method should only be used in the tests. We implement a special helper - method for doing this because doing comparisons by slicing is much faster, - but we don't want to expose slicing of PlasmaBuffer objects because it - currently is not safe. - """ - buff1_to_compare = (buff1.buffer if isinstance(buff1, PlasmaBuffer) - else buff1) - buff2_to_compare = (buff2.buffer if isinstance(buff2, PlasmaBuffer) - else buff2) - return buff1_to_compare[:] == buff2_to_compare[:] - - -class PlasmaClient(object): - """The PlasmaClient is used to interface with a plasma store and manager. - - The PlasmaClient can ask the PlasmaStore to allocate a new buffer, seal a - buffer, and get a buffer. Buffers are referred to by object IDs, which are - strings. - """ - - def __init__(self, store_socket_name, manager_socket_name=None, - release_delay=64): - """Initialize the PlasmaClient. - - Args: - store_socket_name (str): Name of the socket the plasma store is - listening at. - manager_socket_name (str): Name of the socket the plasma manager is - listening at. - release_delay (int): The maximum number of objects that the client - will keep and delay releasing (for caching reasons). - """ - self.store_socket_name = store_socket_name - self.manager_socket_name = manager_socket_name - self.alive = True - - if manager_socket_name is not None: - self.conn = libplasma.connect(store_socket_name, - manager_socket_name, - release_delay) - else: - self.conn = libplasma.connect(store_socket_name, "", release_delay) - - def shutdown(self): - """Shutdown the client so that it does not send messages. - - If we kill the Plasma store and Plasma manager that this client is - connected to, then we can use this method to prevent the client from - trying to send messages to the killed processes. - """ - if self.alive: - libplasma.disconnect(self.conn) - self.alive = False - - def create(self, object_id, size, metadata=None): - """Create a new buffer in the PlasmaStore for a particular object ID. - - The returned buffer is mutable until seal is called. - - Args: - object_id (str): A string used to identify an object. - size (int): The size in bytes of the created buffer. - metadata (buffer): An optional buffer encoding whatever metadata the - user wishes to encode. - - Raises: - plasma_object_exists_error: This exception is raised if the object - could not be created because there already is an object with the - same ID in the plasma store. - plasma_out_of_memory_error: This exception is raised if the object - could not be created because the plasma store is unable to evict - enough objects to create room for it. - """ - # Turn the metadata into the right type. - metadata = bytearray(b"") if metadata is None else metadata - buff = libplasma.create(self.conn, object_id, size, metadata) - return PlasmaBuffer(buff, object_id, self) - - def get(self, object_ids, timeout_ms=-1): - """Create a buffer from the PlasmaStore based on object ID. - - If the object has not been sealed yet, this call will block. The - retrieved buffer is immutable. - - Args: - object_ids (List[str]): A list of strings used to identify some - objects. - timeout_ms (int): The number of milliseconds that the get call should - block before timing out and returning. Pass -1 if the call should - block and 0 if the call should return immediately. - """ - results = libplasma.get(self.conn, object_ids, timeout_ms) - assert len(object_ids) == len(results) - returns = [] - for i in range(len(object_ids)): - if results[i] is None: - returns.append(None) - else: - returns.append(PlasmaBuffer(results[i][0], object_ids[i], - self)) - return returns - - def get_metadata(self, object_ids, timeout_ms=-1): - """Create a buffer from the PlasmaStore based on object ID. - - If the object has not been sealed yet, this call will block until the - object has been sealed. The retrieved buffer is immutable. - - Args: - object_ids (List[str]): A list of strings used to identify some - objects. - timeout_ms (int): The number of milliseconds that the get call should - block before timing out and returning. Pass -1 if the call should - block and 0 if the call should return immediately. - """ - results = libplasma.get(self.conn, object_ids, timeout_ms) - assert len(object_ids) == len(results) - returns = [] - for i in range(len(object_ids)): - if results[i] is None: - returns.append(None) - else: - returns.append(PlasmaBuffer(results[i][1], object_ids[i], - self)) - return returns - - def contains(self, object_id): - """Check if the object is present and has been sealed. - - Args: - object_id (str): A string used to identify an object. - """ - return libplasma.contains(self.conn, object_id) - - def hash(self, object_id): - """Compute the hash of an object in the object store. - - Args: - object_id (str): A string used to identify an object. - - Returns: - A digest string object's SHA256 hash. If the object isn't in the - object store, the string will have length zero. - """ - return libplasma.hash(self.conn, object_id) - - def seal(self, object_id): - """Seal the buffer in the PlasmaStore for a particular object ID. - - Once a buffer has been sealed, the buffer is immutable and can only be - accessed through get. - - Args: - object_id (str): A string used to identify an object. - """ - libplasma.seal(self.conn, object_id) - - def delete(self, object_id): - """Delete the buffer in the PlasmaStore for a particular object ID. - - Once a buffer has been deleted, the buffer is no longer accessible. - - Args: - object_id (str): A string used to identify an object. - """ - libplasma.delete(self.conn, object_id) - - def evict(self, num_bytes): - """Evict some objects until to recover some bytes. - - Recover at least num_bytes bytes if possible. - - Args: - num_bytes (int): The number of bytes to attempt to recover. - """ - return libplasma.evict(self.conn, num_bytes) - - def transfer(self, addr, port, object_id): - """Transfer local object with id object_id to another plasma instance - - Args: - addr (str): IPv4 address of the plasma instance the object is sent - to. - port (int): Port number of the plasma instance the object is sent to. - object_id (str): A string used to identify an object. - """ - return libplasma.transfer(self.conn, object_id, addr, port) - - def fetch(self, object_ids): - """Fetch the objects with the given IDs from other plasma managers. - - Args: - object_ids (List[str]): A list of strings used to identify the - objects. - """ - return libplasma.fetch(self.conn, object_ids) - - def wait(self, object_ids, timeout=PLASMA_WAIT_TIMEOUT, num_returns=1): - """Wait until num_returns objects in object_ids are ready. - - Currently, the object ID arguments to wait must be unique. - - Args: - object_ids (List[str]): List of object IDs to wait for. - timeout (int): Return to the caller after timeout milliseconds. - num_returns (int): We are waiting for this number of objects to be - ready. - - Returns: - ready_ids, waiting_ids (List[str], List[str]): List of object IDs - that are ready and list of object IDs we might still wait on - respectively. - """ - # Check that the object ID arguments are unique. The plasma manager - # currently crashes if given duplicate object IDs. - if len(object_ids) != len(set(object_ids)): - raise Exception("Wait requires a list of unique object IDs.") - ready_ids, waiting_ids = libplasma.wait(self.conn, object_ids, timeout, - num_returns) - return ready_ids, list(waiting_ids) - - def subscribe(self): - """Subscribe to notifications about sealed objects.""" - self.notification_fd = libplasma.subscribe(self.conn) - - def get_next_notification(self): - """Get the next notification from the notification socket.""" - return libplasma.receive_notification(self.notification_fd) - - DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9 diff --git a/python/ray/plasma/test/test.py b/python/ray/plasma/test/test.py index a56dc60a2b4d..dba6253d3bf9 100644 --- a/python/ray/plasma/test/test.py +++ b/python/ray/plasma/test/test.py @@ -3,16 +3,20 @@ from __future__ import print_function import numpy as np +from numpy.testing import assert_equal import os import random import signal +import subprocess import sys import threading import time import unittest -import ray.plasma as plasma -from ray.plasma.utils import (random_object_id, generate_metadata, +import pyarrow as pa +import pyarrow.plasma as plasma +import ray +from ray.plasma.utils import (random_object_id, create_object_with_id, create_object) from ray import services @@ -20,6 +24,10 @@ PLASMA_STORE_MEMORY = 1000000000 +def random_name(): + return str(random.randint(0, 99999999)) + + def assert_get_object_equal(unit_test, client1, client2, object_id, memory_buffer=None, metadata=None): client1_buff = client1.get([object_id])[0] @@ -29,473 +37,88 @@ def assert_get_object_equal(unit_test, client1, client2, object_id, unit_test.assertEqual(len(client1_buff), len(client2_buff)) unit_test.assertEqual(len(client1_metadata), len(client2_metadata)) # Check that the buffers from the two clients are the same. - unit_test.assertTrue(plasma.buffers_equal(client1_buff, client2_buff)) + assert_equal(np.frombuffer(client1_buff, dtype="uint8"), + np.frombuffer(client2_buff, dtype="uint8")) # Check that the metadata buffers from the two clients are the same. - unit_test.assertTrue(plasma.buffers_equal(client1_metadata, - client2_metadata)) + assert_equal(np.frombuffer(client1_metadata, dtype="uint8"), + np.frombuffer(client2_metadata, dtype="uint8")) # If a reference buffer was provided, check that it is the same as well. if memory_buffer is not None: - unit_test.assertTrue(plasma.buffers_equal(memory_buffer, client1_buff)) + assert_equal(np.frombuffer(memory_buffer, dtype="uint8"), + np.frombuffer(client1_buff, dtype="uint8")) # If reference metadata was provided, check that it is the same as well. if metadata is not None: - unit_test.assertTrue(plasma.buffers_equal(metadata, client1_metadata)) - - -class TestPlasmaClient(unittest.TestCase): - - def setUp(self): - # Start Plasma store. - plasma_store_name, self.p = plasma.start_plasma_store( - use_valgrind=USE_VALGRIND) - # Connect to Plasma. - self.plasma_client = plasma.PlasmaClient(plasma_store_name, None, 64) - # For the eviction test - self.plasma_client2 = plasma.PlasmaClient(plasma_store_name, None, 0) - - def tearDown(self): - # Check that the Plasma store is still alive. - self.assertEqual(self.p.poll(), None) - # Kill the plasma store process. - if USE_VALGRIND: - self.p.send_signal(signal.SIGTERM) - self.p.wait() - if self.p.returncode != 0: - os._exit(-1) - else: - self.p.kill() - - def test_create(self): - # Create an object id string. - object_id = random_object_id() - # Create a new buffer and write to it. - length = 50 - memory_buffer = self.plasma_client.create(object_id, length) - for i in range(length): - memory_buffer[i] = chr(i % 256) - # Seal the object. - self.plasma_client.seal(object_id) - # Get the object. - memory_buffer = self.plasma_client.get([object_id])[0] - for i in range(length): - self.assertEqual(memory_buffer[i], chr(i % 256)) - - def test_create_with_metadata(self): - for length in range(1000): - # Create an object id string. - object_id = random_object_id() - # Create a random metadata string. - metadata = generate_metadata(length) - # Create a new buffer and write to it. - memory_buffer = self.plasma_client.create(object_id, length, - metadata) - for i in range(length): - memory_buffer[i] = chr(i % 256) - # Seal the object. - self.plasma_client.seal(object_id) - # Get the object. - memory_buffer = self.plasma_client.get([object_id])[0] - for i in range(length): - self.assertEqual(memory_buffer[i], chr(i % 256)) - # Get the metadata. - metadata_buffer = self.plasma_client.get_metadata([object_id])[0] - self.assertEqual(len(metadata), len(metadata_buffer)) - for i in range(len(metadata)): - self.assertEqual(chr(metadata[i]), metadata_buffer[i]) - - def test_create_existing(self): - # This test is partially used to test the code path in which we create - # an object with an ID that already exists - length = 100 - for _ in range(1000): - object_id = random_object_id() - self.plasma_client.create(object_id, length, - generate_metadata(length)) - try: - self.plasma_client.create(object_id, length, - generate_metadata(length)) - except plasma.plasma_object_exists_error as e: - pass - else: - self.assertTrue(False) - - def test_get(self): - num_object_ids = 100 - # Test timing out of get with various timeouts. - for timeout in [0, 10, 100, 1000]: - object_ids = [random_object_id() for _ in range(num_object_ids)] - results = self.plasma_client.get(object_ids, timeout_ms=timeout) - self.assertEqual(results, num_object_ids * [None]) - - data_buffers = [] - metadata_buffers = [] - for i in range(num_object_ids): - if i % 2 == 0: - data_buffer, metadata_buffer = create_object_with_id( - self.plasma_client, object_ids[i], 2000, 2000) - data_buffers.append(data_buffer) - metadata_buffers.append(metadata_buffer) - - # Test timing out from some but not all get calls with various - # timeouts. - for timeout in [0, 10, 100, 1000]: - data_results = self.plasma_client.get(object_ids, - timeout_ms=timeout) - for i in range(num_object_ids): - if i % 2 == 0: - self.assertTrue(plasma.buffers_equal(data_buffers[i // 2], - data_results[i])) - else: - self.assertIsNone(results[i]) - - def test_store_full(self): - # The store is started with 1GB, so make sure that create throws an - # exception when it is full. - def assert_create_raises_plasma_full(unit_test, size): - partial_size = np.random.randint(size) - try: - _, memory_buffer, _ = create_object(unit_test.plasma_client, - partial_size, - size - partial_size) - except plasma.plasma_out_of_memory_error as e: - pass - else: - # For some reason the above didn't throw an exception, so fail. - unit_test.assertTrue(False) - - # Create a list to keep some of the buffers in scope. - memory_buffers = [] - _, memory_buffer, _ = create_object(self.plasma_client, 5 * 10 ** 8, 0) - memory_buffers.append(memory_buffer) - # Remaining space is 5 * 10 ** 8. Make sure that we can't create an - # object of size 5 * 10 ** 8 + 1, but we can create one of size - # 2 * 10 ** 8. - assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1) - _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0) - del memory_buffer - _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0) - del memory_buffer - assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1) - - _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0) - memory_buffers.append(memory_buffer) - # Remaining space is 3 * 10 ** 8. - assert_create_raises_plasma_full(self, 3 * 10 ** 8 + 1) - - _, memory_buffer, _ = create_object(self.plasma_client, 10 ** 8, 0) - memory_buffers.append(memory_buffer) - # Remaining space is 2 * 10 ** 8. - assert_create_raises_plasma_full(self, 2 * 10 ** 8 + 1) - - def test_contains(self): - fake_object_ids = [random_object_id() for _ in range(100)] - real_object_ids = [random_object_id() for _ in range(100)] - for object_id in real_object_ids: - self.assertFalse(self.plasma_client.contains(object_id)) - self.plasma_client.create(object_id, 100) - self.plasma_client.seal(object_id) - self.assertTrue(self.plasma_client.contains(object_id)) - for object_id in fake_object_ids: - self.assertFalse(self.plasma_client.contains(object_id)) - for object_id in real_object_ids: - self.assertTrue(self.plasma_client.contains(object_id)) - - def test_hash(self): - # Check the hash of an object that doesn't exist. - object_id1 = random_object_id() - self.plasma_client.hash(object_id1) - - length = 1000 - # Create a random object, and check that the hash function always - # returns the same value. - metadata = generate_metadata(length) - memory_buffer = self.plasma_client.create(object_id1, length, metadata) - for i in range(length): - memory_buffer[i] = chr(i % 256) - self.plasma_client.seal(object_id1) - self.assertEqual(self.plasma_client.hash(object_id1), - self.plasma_client.hash(object_id1)) - - # Create a second object with the same value as the first, and check - # that their hashes are equal. - object_id2 = random_object_id() - memory_buffer = self.plasma_client.create(object_id2, length, metadata) - for i in range(length): - memory_buffer[i] = chr(i % 256) - self.plasma_client.seal(object_id2) - self.assertEqual(self.plasma_client.hash(object_id1), - self.plasma_client.hash(object_id2)) - - # Create a third object with a different value from the first two, and - # check that its hash is different. - object_id3 = random_object_id() - metadata = generate_metadata(length) - memory_buffer = self.plasma_client.create(object_id3, length, metadata) - for i in range(length): - memory_buffer[i] = chr((i + 1) % 256) - self.plasma_client.seal(object_id3) - self.assertNotEqual(self.plasma_client.hash(object_id1), - self.plasma_client.hash(object_id3)) - - # Create a fourth object with the same value as the third, but - # different metadata. Check that its hash is different from any of the - # previous three. - object_id4 = random_object_id() - metadata4 = generate_metadata(length) - memory_buffer = self.plasma_client.create(object_id4, length, - metadata4) - for i in range(length): - memory_buffer[i] = chr((i + 1) % 256) - self.plasma_client.seal(object_id4) - self.assertNotEqual(self.plasma_client.hash(object_id1), - self.plasma_client.hash(object_id4)) - self.assertNotEqual(self.plasma_client.hash(object_id3), - self.plasma_client.hash(object_id4)) - - def test_many_hashes(self): - hashes = [] - length = 2 ** 10 - - for i in range(256): - object_id = random_object_id() - memory_buffer = self.plasma_client.create(object_id, length) - for j in range(length): - memory_buffer[j] = chr(i) - self.plasma_client.seal(object_id) - hashes.append(self.plasma_client.hash(object_id)) - - # Create objects of varying length. Each pair has two bits different. - for i in range(length): - object_id = random_object_id() - memory_buffer = self.plasma_client.create(object_id, length) - for j in range(length): - memory_buffer[j] = chr(0) - memory_buffer[i] = chr(1) - self.plasma_client.seal(object_id) - hashes.append(self.plasma_client.hash(object_id)) - - # Create objects of varying length, all with value 0. - for i in range(length): - object_id = random_object_id() - memory_buffer = self.plasma_client.create(object_id, i) - for j in range(i): - memory_buffer[j] = chr(0) - self.plasma_client.seal(object_id) - hashes.append(self.plasma_client.hash(object_id)) - - # Check that all hashes were unique. - self.assertEqual(len(set(hashes)), 256 + length + length) - - # def test_individual_delete(self): - # length = 100 - # # Create an object id string. - # object_id = random_object_id() - # # Create a random metadata string. - # metadata = generate_metadata(100) - # # Create a new buffer and write to it. - # memory_buffer = self.plasma_client.create(object_id, length, metadata) - # for i in range(length): - # memory_buffer[i] = chr(i % 256) - # # Seal the object. - # self.plasma_client.seal(object_id) - # # Check that the object is present. - # self.assertTrue(self.plasma_client.contains(object_id)) - # # Delete the object. - # self.plasma_client.delete(object_id) - # # Make sure the object is no longer present. - # self.assertFalse(self.plasma_client.contains(object_id)) - # - # def test_delete(self): - # # Create some objects. - # object_ids = [random_object_id() for _ in range(100)] - # for object_id in object_ids: - # length = 100 - # # Create a random metadata string. - # metadata = generate_metadata(100) - # # Create a new buffer and write to it. - # memory_buffer = self.plasma_client.create(object_id, length, - # metadata) - # for i in range(length): - # memory_buffer[i] = chr(i % 256) - # # Seal the object. - # self.plasma_client.seal(object_id) - # # Check that the object is present. - # self.assertTrue(self.plasma_client.contains(object_id)) - # - # # Delete the objects and make sure they are no longer present. - # for object_id in object_ids: - # # Delete the object. - # self.plasma_client.delete(object_id) - # # Make sure the object is no longer present. - # self.assertFalse(self.plasma_client.contains(object_id)) + assert_equal(np.frombuffer(metadata, dtype="uint8"), + np.frombuffer(client1_metadata, dtype="uint8")) + + +DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9 + + +def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, + use_valgrind=False, use_profiler=False, + stdout_file=None, stderr_file=None): + """Start a plasma store process. + Args: + use_valgrind (bool): True if the plasma store should be started inside + of valgrind. If this is True, use_profiler must be False. + use_profiler (bool): True if the plasma store should be started inside + a profiler. If this is True, use_valgrind must be False. + stdout_file: A file handle opened for writing to redirect stdout to. If + no redirection should happen, then this should be None. + stderr_file: A file handle opened for writing to redirect stderr to. If + no redirection should happen, then this should be None. + Return: + A tuple of the name of the plasma store socket and the process ID of + the plasma store process. + """ + if use_valgrind and use_profiler: + raise Exception("Cannot use valgrind and profiler at the same time.") + plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store") + plasma_store_name = "/tmp/plasma_store{}".format(random_name()) + command = [plasma_store_executable, + "-s", plasma_store_name, + "-m", str(plasma_store_memory)] + if use_valgrind: + pid = subprocess.Popen(["valgrind", + "--track-origins=yes", + "--leak-check=full", + "--show-leak-kinds=all", + "--leak-check-heuristics=stdstring", + "--error-exitcode=1"] + command, + stdout=stdout_file, stderr=stderr_file) + time.sleep(1.0) + elif use_profiler: + pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command, + stdout=stdout_file, stderr=stderr_file) + time.sleep(1.0) + else: + pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file) + time.sleep(0.1) + return plasma_store_name, pid - def test_illegal_functionality(self): - # Create an object id string. - object_id = random_object_id() - # Create a new buffer and write to it. - length = 1000 - memory_buffer = self.plasma_client.create(object_id, length) - # Make sure we cannot access memory out of bounds. - self.assertRaises(Exception, lambda: memory_buffer[length]) - # Seal the object. - self.plasma_client.seal(object_id) - # This test is commented out because it currently fails. - # # Make sure the object is ready only now. - # def illegal_assignment(): - # memory_buffer[0] = chr(0) - # self.assertRaises(Exception, illegal_assignment) - # Get the object. - memory_buffer = self.plasma_client.get([object_id])[0] - - # Make sure the object is read only. - def illegal_assignment(): - memory_buffer[0] = chr(0) - self.assertRaises(Exception, illegal_assignment) - - def test_evict(self): - client = self.plasma_client2 - object_id1 = random_object_id() - b1 = client.create(object_id1, 1000) - client.seal(object_id1) - del b1 - self.assertEqual(client.evict(1), 1000) - object_id2 = random_object_id() - object_id3 = random_object_id() - b2 = client.create(object_id2, 999) - b3 = client.create(object_id3, 998) - client.seal(object_id3) - del b3 - self.assertEqual(client.evict(1000), 998) - - object_id4 = random_object_id() - b4 = client.create(object_id4, 997) - client.seal(object_id4) - del b4 - client.seal(object_id2) - del b2 - self.assertEqual(client.evict(1), 997) - self.assertEqual(client.evict(1), 999) - - object_id5 = random_object_id() - object_id6 = random_object_id() - object_id7 = random_object_id() - b5 = client.create(object_id5, 996) - b6 = client.create(object_id6, 995) - b7 = client.create(object_id7, 994) - client.seal(object_id5) - client.seal(object_id6) - client.seal(object_id7) - del b5 - del b6 - del b7 - self.assertEqual(client.evict(2000), 996 + 995 + 994) - - def test_subscribe(self): - # Subscribe to notifications from the Plasma Store. - self.plasma_client.subscribe() - for i in [1, 10, 100, 1000, 10000, 100000]: - object_ids = [random_object_id() for _ in range(i)] - metadata_sizes = [np.random.randint(1000) for _ in range(i)] - data_sizes = [np.random.randint(1000) for _ in range(i)] - for j in range(i): - self.plasma_client.create( - object_ids[j], size=data_sizes[j], - metadata=bytearray(np.random.bytes(metadata_sizes[j]))) - self.plasma_client.seal(object_ids[j]) - # Check that we received notifications for all of the objects. - for j in range(i): - notification_info = self.plasma_client.get_next_notification() - recv_objid, recv_dsize, recv_msize = notification_info - self.assertEqual(object_ids[j], recv_objid) - self.assertEqual(data_sizes[j], recv_dsize) - self.assertEqual(metadata_sizes[j], recv_msize) - - def test_subscribe_deletions(self): - # Subscribe to notifications from the Plasma Store. We use - # plasma_client2 to make sure that all used objects will get evicted - # properly. - self.plasma_client2.subscribe() - for i in [1, 10, 100, 1000, 10000, 100000]: - object_ids = [random_object_id() for _ in range(i)] - # Add 1 to the sizes to make sure we have nonzero object sizes. - metadata_sizes = [np.random.randint(1000) + 1 for _ in range(i)] - data_sizes = [np.random.randint(1000) + 1 for _ in range(i)] - for j in range(i): - x = self.plasma_client2.create( - object_ids[j], size=data_sizes[j], - metadata=bytearray(np.random.bytes(metadata_sizes[j]))) - self.plasma_client2.seal(object_ids[j]) - del x - # Check that we received notifications for creating all of the - # objects. - for j in range(i): - notification_info = self.plasma_client2.get_next_notification() - recv_objid, recv_dsize, recv_msize = notification_info - self.assertEqual(object_ids[j], recv_objid) - self.assertEqual(data_sizes[j], recv_dsize) - self.assertEqual(metadata_sizes[j], recv_msize) - - # Check that we receive notifications for deleting all objects, as - # we evict them. - for j in range(i): - self.assertEqual(self.plasma_client2.evict(1), - data_sizes[j] + metadata_sizes[j]) - notification_info = self.plasma_client2.get_next_notification() - recv_objid, recv_dsize, recv_msize = notification_info - self.assertEqual(object_ids[j], recv_objid) - self.assertEqual(-1, recv_dsize) - self.assertEqual(-1, recv_msize) - - # Test multiple deletion notifications. The first 9 object IDs have - # size 0, and the last has a nonzero size. When Plasma evicts 1 byte, - # it will evict all objects, so we should receive deletion - # notifications for each. - num_object_ids = 10 - object_ids = [random_object_id() for _ in range(num_object_ids)] - metadata_sizes = [0] * (num_object_ids - 1) - data_sizes = [0] * (num_object_ids - 1) - metadata_sizes.append(np.random.randint(1000)) - data_sizes.append(np.random.randint(1000)) - for i in range(num_object_ids): - x = self.plasma_client2.create( - object_ids[i], size=data_sizes[i], - metadata=bytearray(np.random.bytes(metadata_sizes[i]))) - self.plasma_client2.seal(object_ids[i]) - del x - for i in range(num_object_ids): - notification_info = self.plasma_client2.get_next_notification() - recv_objid, recv_dsize, recv_msize = notification_info - self.assertEqual(object_ids[i], recv_objid) - self.assertEqual(data_sizes[i], recv_dsize) - self.assertEqual(metadata_sizes[i], recv_msize) - self.assertEqual(self.plasma_client2.evict(1), - data_sizes[-1] + metadata_sizes[-1]) - for i in range(num_object_ids): - notification_info = self.plasma_client2.get_next_notification() - recv_objid, recv_dsize, recv_msize = notification_info - self.assertEqual(object_ids[i], recv_objid) - self.assertEqual(-1, recv_dsize) - self.assertEqual(-1, recv_msize) +# Plasma client tests were moved into arrow class TestPlasmaManager(unittest.TestCase): def setUp(self): # Start two PlasmaStores. - store_name1, self.p2 = plasma.start_plasma_store( + store_name1, self.p2 = start_plasma_store( use_valgrind=USE_VALGRIND) - store_name2, self.p3 = plasma.start_plasma_store( + store_name2, self.p3 = start_plasma_store( use_valgrind=USE_VALGRIND) # Start a Redis server. redis_address, _ = services.start_redis("127.0.0.1") # Start two PlasmaManagers. - manager_name1, self.p4, self.port1 = plasma.start_plasma_manager( + manager_name1, self.p4, self.port1 = ray.plasma.start_plasma_manager( store_name1, redis_address, use_valgrind=USE_VALGRIND) - manager_name2, self.p5, self.port2 = plasma.start_plasma_manager( + manager_name2, self.p5, self.port2 = ray.plasma.start_plasma_manager( store_name2, redis_address, use_valgrind=USE_VALGRIND) # Connect two PlasmaClients. - self.client1 = plasma.PlasmaClient(store_name1, manager_name1) - self.client2 = plasma.PlasmaClient(store_name2, manager_name2) + self.client1 = plasma.connect(store_name1, manager_name1, 64) + self.client2 = plasma.connect(store_name2, manager_name2, 64) # Store the processes that will be explicitly killed during tearDown so # that a test case can remove ones that will be killed during the test. @@ -719,7 +342,8 @@ def finish(): # Make sure that wait returns when the requested number of object IDs # are available and does not wait for all object IDs to be available. - object_ids = [random_object_id() for _ in range(9)] + [20 * b'\x00'] + object_ids = [random_object_id() for _ in range(9)] + \ + [plasma.ObjectID(20 * b'\x00')] object_ids_perm = object_ids[:] random.shuffle(object_ids_perm) for i in range(10): @@ -812,17 +436,17 @@ class TestPlasmaManagerRecovery(unittest.TestCase): def setUp(self): # Start a Plasma store. - self.store_name, self.p2 = plasma.start_plasma_store( + self.store_name, self.p2 = start_plasma_store( use_valgrind=USE_VALGRIND) # Start a Redis server. self.redis_address, _ = services.start_redis("127.0.0.1") # Start a PlasmaManagers. - manager_name, self.p3, self.port1 = plasma.start_plasma_manager( + manager_name, self.p3, self.port1 = ray.plasma.start_plasma_manager( self.store_name, self.redis_address, use_valgrind=USE_VALGRIND) # Connect a PlasmaClient. - self.client = plasma.PlasmaClient(self.store_name, manager_name) + self.client = plasma.connect(self.store_name, manager_name, 64) # Store the processes that will be explicitly killed during tearDown so # that a test case can remove ones that will be killed during the test. @@ -865,12 +489,12 @@ def test_delayed_start(self): self.assertEqual(waiting, []) # Start a second plasma manager attached to the same store. - manager_name, self.p5, self.port2 = plasma.start_plasma_manager( + manager_name, self.p5, self.port2 = ray.plasma.start_plasma_manager( self.store_name, self.redis_address, use_valgrind=USE_VALGRIND) self.processes_to_kill = [self.p5] + self.processes_to_kill # Check that the second manager knows about existing objects. - client2 = plasma.PlasmaClient(self.store_name, manager_name) + client2 = plasma.connect(self.store_name, manager_name, 64) ready, waiting = [], object_ids while True: ready, waiting = client2.wait(object_ids, num_returns=num_objects, diff --git a/python/ray/plasma/utils.py b/python/ray/plasma/utils.py index f4141a1f19ce..d34b293180f1 100644 --- a/python/ray/plasma/utils.py +++ b/python/ray/plasma/utils.py @@ -5,9 +5,11 @@ import numpy as np import random +import pyarrow.plasma as plasma + def random_object_id(): - return np.random.bytes(20) + return plasma.ObjectID(np.random.bytes(20)) def generate_metadata(length): @@ -22,11 +24,12 @@ def generate_metadata(length): def write_to_data_buffer(buff, length): + array = np.frombuffer(buff, dtype="uint8") if length > 0: - buff[0] = chr(random.randint(0, 255)) - buff[-1] = chr(random.randint(0, 255)) + array[0] = random.randint(0, 255) + array[-1] = random.randint(0, 255) for _ in range(100): - buff[random.randint(0, length - 1)] = chr(random.randint(0, 255)) + array[random.randint(0, length - 1)] = random.randint(0, 255) def create_object_with_id(client, object_id, data_size, metadata_size, diff --git a/python/ray/worker.py b/python/ray/worker.py index 9a1b8036e828..52ebcd23411d 100644 --- a/python/ray/worker.py +++ b/python/ray/worker.py @@ -20,6 +20,7 @@ import traceback # Ray modules +import pyarrow.plasma as plasma import ray.experimental.state as state import ray.serialization as serialization import ray.services as services @@ -300,7 +301,8 @@ def store_and_register(self, object_id, value, depth=100): "type {}.".format(type(value))) counter += 1 try: - ray.numbuf.store_list(object_id.id(), self.plasma_client.conn, + ray.numbuf.store_list(object_id.id(), + self.plasma_client.to_capsule(), [value]) break except serialization.RaySerializationException as e: @@ -375,7 +377,7 @@ def retrieve_and_deserialize(self, object_ids, timeout, error_timeout=10): for i in range(0, len(object_ids), get_request_size): results += ray.numbuf.retrieve_list( object_ids[i:(i + get_request_size)], - self.plasma_client.conn, + self.plasma_client.to_capsule(), timeout) return results except serialization.RayDeserializationException as e: @@ -420,7 +422,8 @@ def get_object(self, object_ids): # smaller fetches so as to not block the manager for a prolonged period # of time in a single call. fetch_request_size = 10000 - plain_object_ids = [object_id.id() for object_id in object_ids] + plain_object_ids = [plasma.ObjectID(object_id.id()) + for object_id in object_ids] for i in range(0, len(object_ids), fetch_request_size): self.plasma_client.fetch( plain_object_ids[i:(i + fetch_request_size)]) @@ -443,7 +446,8 @@ def get_object(self, object_ids): # in case they were evicted since the last fetch. We divide the # fetch into smaller fetches so as to not block the manager for a # prolonged period of time in a single call. - object_ids_to_fetch = list(unready_ids.keys()) + object_ids_to_fetch = list(map( + plasma.ObjectID, unready_ids.keys())) for i in range(0, len(object_ids_to_fetch), fetch_request_size): self.plasma_client.fetch( object_ids_to_fetch[i:(i + fetch_request_size)]) @@ -1026,7 +1030,7 @@ def cleanup(worker=global_worker): if hasattr(worker, "local_scheduler_client"): del worker.local_scheduler_client if hasattr(worker, "plasma_client"): - worker.plasma_client.shutdown() + worker.plasma_client.disconnect() if worker.mode in [SCRIPT_MODE, SILENT_MODE]: # If this is a driver, push the finish time to Redis and clean up any @@ -1371,8 +1375,9 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker, raise Exception("This code should be unreachable.") # Create an object store client. - worker.plasma_client = ray.plasma.PlasmaClient(info["store_socket_name"], - info["manager_socket_name"]) + worker.plasma_client = plasma.connect(info["store_socket_name"], + info["manager_socket_name"], + 64) # Create the local scheduler client. if worker.actor_id != NIL_ACTOR_ID: num_gpus = int(worker.redis_client.hget(b"Actor:" + actor_id, @@ -1713,14 +1718,15 @@ def wait(object_ids, num_returns=1, timeout=None, worker=global_worker): check_connected(worker) with log_span("ray:wait", worker=worker): check_main_thread() - object_id_strs = [object_id.id() for object_id in object_ids] + object_id_strs = [plasma.ObjectID(object_id.id()) + for object_id in object_ids] timeout = timeout if timeout is not None else 2 ** 30 ready_ids, remaining_ids = worker.plasma_client.wait(object_id_strs, timeout, num_returns) - ready_ids = [ray.local_scheduler.ObjectID(object_id) + ready_ids = [ray.local_scheduler.ObjectID(object_id.binary()) for object_id in ready_ids] - remaining_ids = [ray.local_scheduler.ObjectID(object_id) + remaining_ids = [ray.local_scheduler.ObjectID(object_id.binary()) for object_id in remaining_ids] return ready_ids, remaining_ids diff --git a/python/setup.py b/python/setup.py index 40ba855014b3..04bd430b91b0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -5,14 +5,35 @@ import os import shutil import subprocess +import sys from setuptools import setup, find_packages, Distribution import setuptools.command.build_ext as _build_ext +# This used to be the first line of the run method in the build_ext class. +# However, we moved it here because the previous approach seemed to fail in +# Docker. Inside of the build.sh script, we install the pyarrow Python module. +# Something about calling "python setup.py install" inside of the build_ext +# run method doesn't work (this is easily reproducible in Docker with just a +# couple files to simulate two Python modules). The problem is that the pyarrow +# module doesn't get added to the easy-install.pth file, so it never gets added +# to the Python path even though the package is built and copied to the right +# location. An alternative fix would be to manually modify the easy-install.pth +# file. TODO(rkn): Fix all of this. +# +# Note: We are passing in sys.executable so that we use the same version of +# Python to build pyarrow inside the build.sh script. Note that certain flags +# will not be passed along such as --user or sudo. TODO(rkn): Fix this. +subprocess.check_call(["../build.sh", sys.executable]) + + class build_ext(_build_ext.build_ext): def run(self): - subprocess.check_call(["../build.sh"]) + # The line below has been moved outside of the build_ext class. See the + # explanation there. + # subprocess.check_call(["../build.sh"]) + # Ideally, we could include these files by putting them in a # MANIFEST.in or using the package_data argument to setup, but the # MANIFEST.in gets applied at the very beginning when setup.py runs @@ -46,7 +67,6 @@ def move_file(self, filename): "ray/core/src/common/redis_module/libray_redis_module.so", "ray/core/src/plasma/plasma_store", "ray/core/src/plasma/plasma_manager", - "ray/core/src/plasma/libplasma.so", "ray/core/src/local_scheduler/local_scheduler", "ray/core/src/local_scheduler/liblocal_scheduler_library.so", "ray/core/src/numbuf/libnumbuf.so", diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 27cb1a969c8e..79333a98c537 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -78,7 +78,7 @@ target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhi function(define_test test_name library) add_executable(${test_name} test/${test_name}.cc ${ARGN}) add_dependencies(${test_name} hiredis flatbuffers_ep) - target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${library}) + target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a ${library} -lpthread) target_compile_options(${test_name} PUBLIC "-DPLASMA_TEST -DLOCAL_SCHEDULER_TEST -DCOMMON_TEST -DRAY_COMMON_LOG_LEVEL=4") endfunction() diff --git a/src/common/cmake/Common.cmake b/src/common/cmake/Common.cmake index 1fd25e484041..17c498eff700 100644 --- a/src/common/cmake/Common.cmake +++ b/src/common/cmake/Common.cmake @@ -5,7 +5,7 @@ include(CMakeParseArguments) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") -set(FLATBUFFERS_VERSION "1.6.0") +set(FLATBUFFERS_VERSION "1.7.1") set(FLATBUFFERS_PREFIX "${CMAKE_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install") if (NOT TARGET flatbuffers_ep) diff --git a/src/common/common.cc b/src/common/common.cc index d22a873ce7ce..cffe82dd7762 100644 --- a/src/common/common.cc +++ b/src/common/common.cc @@ -15,8 +15,7 @@ /* This is used to define the array of object IDs. */ const UT_icd object_id_icd = {sizeof(ObjectID), NULL, NULL, NULL}; -const UniqueID NIL_ID = {{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}}; +const UniqueID NIL_ID = UniqueID::nil(); const unsigned char NIL_DIGEST[DIGEST_SIZE] = {0}; diff --git a/src/common/common.h b/src/common/common.h index 436f5fb44ece..7c494861f349 100644 --- a/src/common/common.h +++ b/src/common/common.h @@ -22,6 +22,9 @@ extern "C" { } #endif +#include "plasma/common.h" +#include "arrow/util/macros.h" + /** The duration between heartbeats. These are sent by the plasma manager and * local scheduler. */ #define HEARTBEAT_TIMEOUT_MILLISECONDS 100 @@ -46,12 +49,6 @@ extern "C" { #define RAY_COMMON_LOG_LEVEL RAY_COMMON_INFO #endif -/* Arrow defines the same macro, only define it if it has not already been - * defined. */ -#ifndef UNUSED -#define UNUSED(x) ((void) (x)) -#endif - /** * Macros to enable each level of Ray logging statements depending on the * current logging level. */ @@ -113,19 +110,7 @@ extern "C" { #define CHECK(COND) CHECKM(COND, "") -/* This should be defined if we want to check calls to DCHECK. */ -#define RAY_DCHECK - -/* Arrow also defines the DCHECK macro, so undo that definition. */ -#ifdef DCHECK -#undef DCHECK -#endif - -#ifdef RAY_DCHECK -#define DCHECK(COND) CHECK(COND) -#else -#define DCHECK(COND) -#endif +#define RAY_DCHECK(COND) CHECK(COND) /* These are exit codes for common errors that can occur in Ray components. */ #define EXIT_COULD_NOT_BIND_PORT -2 @@ -141,7 +126,23 @@ extern "C" { #define IS_NIL_ID(id) UNIQUE_ID_EQ(id, NIL_ID) -typedef struct { unsigned char id[UNIQUE_ID_SIZE]; } UniqueID; +struct UniqueID { + unsigned char id[UNIQUE_ID_SIZE]; + UniqueID(const plasma::UniqueID &from) { + memcpy(&id[0], from.data(), UNIQUE_ID_SIZE); + } + UniqueID() {} + static const UniqueID nil() { + UniqueID result; + std::fill_n(result.id, UNIQUE_ID_SIZE, 255); + return result; + } + plasma::UniqueID to_plasma_id() { + plasma::UniqueID result; + memcpy(result.mutable_data(), &id[0], UNIQUE_ID_SIZE); + return result; + } +}; extern const UniqueID NIL_ID; diff --git a/src/common/task.cc b/src/common/task.cc index dcf4a7520b59..7dafa915459e 100644 --- a/src/common/task.cc +++ b/src/common/task.cc @@ -11,7 +11,7 @@ extern "C" { ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) { /* Here, return_indices need to be >= 0, so we can use negative * indices for put. */ - DCHECK(return_index >= 0); + RAY_DCHECK(return_index >= 0); /* TODO(rkn): This line requires object and task IDs to be the same size. */ ObjectID return_id = task_id; int64_t *first_bytes = (int64_t *) &return_id; @@ -22,7 +22,7 @@ ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) { } ObjectID task_compute_put_id(TaskID task_id, int64_t put_index) { - DCHECK(put_index >= 0); + RAY_DCHECK(put_index >= 0); /* TODO(pcm): This line requires object and task IDs to be the same size. */ ObjectID put_id = task_id; int64_t *first_bytes = (int64_t *) &put_id; diff --git a/src/local_scheduler/CMakeLists.txt b/src/local_scheduler/CMakeLists.txt index 7d170f6ca228..fc1f935cedab 100644 --- a/src/local_scheduler/CMakeLists.txt +++ b/src/local_scheduler/CMakeLists.txt @@ -5,6 +5,12 @@ project(local_scheduler) # Recursively include common include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) +# Include plasma +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules) + +find_package(Plasma) +include_directories(SYSTEM ${PLASMA_INCLUDE_DIR}) + add_definitions(-fPIC) if(APPLE) @@ -21,7 +27,9 @@ endif() include_directories("${CMAKE_CURRENT_LIST_DIR}/") include_directories("${CMAKE_CURRENT_LIST_DIR}/../") +# TODO(pcm): get rid of this: include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/") +include_directories("${ARROW_DIR}/cpp/src/") include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/") # Compile flatbuffers @@ -60,10 +68,10 @@ endif(APPLE) add_dependencies(local_scheduler_library gen_local_scheduler_fbs) add_executable(local_scheduler local_scheduler.cc local_scheduler_algorithm.cc) -target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} plasma_lib) +target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a -lpthread) add_executable(local_scheduler_tests test/local_scheduler_tests.cc local_scheduler.cc local_scheduler_algorithm.cc) -target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} plasma_lib) +target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a -lpthread) target_compile_options(local_scheduler_tests PUBLIC "-DLOCAL_SCHEDULER_TEST") install(TARGETS local_scheduler_library DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler) diff --git a/src/local_scheduler/local_scheduler.cc b/src/local_scheduler/local_scheduler.cc index cc1187140fff..6aa174a039e8 100644 --- a/src/local_scheduler/local_scheduler.cc +++ b/src/local_scheduler/local_scheduler.cc @@ -359,7 +359,7 @@ LocalSchedulerState *LocalSchedulerState_init( state->db = NULL; } /* Connect to Plasma. This method will retry if Plasma hasn't started yet. */ - state->plasma_conn = new PlasmaClient(); + state->plasma_conn = new plasma::PlasmaClient(); if (plasma_manager_socket_name != NULL) { ARROW_CHECK_OK(state->plasma_conn->Connect(plasma_store_socket_name, plasma_manager_socket_name, @@ -370,7 +370,7 @@ LocalSchedulerState *LocalSchedulerState_init( } /* Subscribe to notifications about sealed objects. */ int plasma_fd; - ARROW_CHECK_OK(state->plasma_conn->Subscribe(plasma_fd)); + ARROW_CHECK_OK(state->plasma_conn->Subscribe(&plasma_fd)); /* Add the callback that processes the notification to the event loop. */ event_loop_add_file(loop, plasma_fd, EVENT_LOOP_READ, process_plasma_notification, state); diff --git a/src/local_scheduler/local_scheduler_algorithm.cc b/src/local_scheduler/local_scheduler_algorithm.cc index 46bd31525912..09b9b7a4dc09 100644 --- a/src/local_scheduler/local_scheduler_algorithm.cc +++ b/src/local_scheduler/local_scheduler_algorithm.cc @@ -456,11 +456,11 @@ void add_task_to_actor_queue(LocalSchedulerState *state, void fetch_missing_dependency(LocalSchedulerState *state, SchedulingAlgorithmState *algorithm_state, std::list::iterator task_entry_it, - ObjectID obj_id) { + plasma::ObjectID obj_id) { if (algorithm_state->remote_objects.count(obj_id) == 0) { /* We weren't actively fetching this object. Try the fetch once * immediately. */ - if (plasma_manager_is_connected(state->plasma_conn)) { + if (state->plasma_conn->get_manager_fd() != -1) { ARROW_CHECK_OK(state->plasma_conn->Fetch(1, &obj_id)); } /* Create an entry and add it to the list of active fetch requests to @@ -497,7 +497,8 @@ void fetch_missing_dependencies( ObjectID obj_id = TaskSpec_arg_id(task, i); if (algorithm_state->local_objects.count(obj_id) == 0) { /* If the entry is not yet available locally, record the dependency. */ - fetch_missing_dependency(state, algorithm_state, task_entry_it, obj_id); + fetch_missing_dependency(state, algorithm_state, task_entry_it, + obj_id.to_plasma_id()); ++num_missing_dependencies; } } @@ -536,7 +537,7 @@ int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context) { LocalSchedulerState *state = (LocalSchedulerState *) context; /* Only try the fetches if we are connected to the object store manager. */ - if (!plasma_manager_is_connected(state->plasma_conn)) { + if (state->plasma_conn->get_manager_fd() == -1) { LOG_INFO("Local scheduler is not connected to a object store manager"); return kLocalSchedulerFetchTimeoutMilliseconds; } @@ -555,8 +556,9 @@ int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context) { for (int64_t j = 0; j < num_object_ids; j += fetch_request_size) { int num_objects_in_request = std::min(num_object_ids, j + fetch_request_size) - j; - ARROW_CHECK_OK( - state->plasma_conn->Fetch(num_objects_in_request, &object_ids[j])); + ARROW_CHECK_OK(state->plasma_conn->Fetch( + num_objects_in_request, + reinterpret_cast(&object_ids[j]))); } /* Print a warning if this method took too long. */ @@ -1237,7 +1239,7 @@ void handle_object_removed(LocalSchedulerState *state, ObjectID arg_id = TaskSpec_arg_id(it->spec, i); if (ObjectID_equal(arg_id, removed_object_id)) { fetch_missing_dependency(state, algorithm_state, it, - removed_object_id); + removed_object_id.to_plasma_id()); } } } diff --git a/src/local_scheduler/local_scheduler_client.cc b/src/local_scheduler/local_scheduler_client.cc index a83a3d6d1431..81ca06769af4 100644 --- a/src/local_scheduler/local_scheduler_client.cc +++ b/src/local_scheduler/local_scheduler_client.cc @@ -6,6 +6,8 @@ #include "common/io.h" #include "common/task.h" #include +#include +#include LocalSchedulerConnection *LocalSchedulerConnection_init( const char *local_scheduler_socket, diff --git a/src/local_scheduler/local_scheduler_shared.h b/src/local_scheduler/local_scheduler_shared.h index df3372061928..6cd358cd89e9 100644 --- a/src/local_scheduler/local_scheduler_shared.h +++ b/src/local_scheduler/local_scheduler_shared.h @@ -4,7 +4,7 @@ #include "common/task.h" #include "common/state/table.h" #include "common/state/db.h" -#include "plasma_client.h" +#include "plasma/client.h" #include #include @@ -57,7 +57,7 @@ struct LocalSchedulerState { /** The handle to the database. */ DBHandle *db; /** The Plasma client. */ - PlasmaClient *plasma_conn; + plasma::PlasmaClient *plasma_conn; /** State for the scheduling algorithm. */ SchedulingAlgorithmState *algorithm_state; /** Input buffer, used for reading input in process_message to avoid diff --git a/src/numbuf/CMakeLists.txt b/src/numbuf/CMakeLists.txt index 0511480bcaf6..96573cbdde26 100644 --- a/src/numbuf/CMakeLists.txt +++ b/src/numbuf/CMakeLists.txt @@ -6,6 +6,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) +# Include plasma +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules) + +find_package(Plasma) +include_directories(SYSTEM ${PLASMA_INCLUDE_DIR}) + option(HAS_PLASMA "Are we linking with the plasma object store? Recommended if numbuf is used as part of ray." ON) @@ -29,24 +35,15 @@ if(UNIX AND NOT APPLE) link_libraries(rt) endif() -set(ARROW_DIR "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/" - CACHE STRING "Path of the arrow source directory") - -set(ARROW_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a" +set(ARROW_LIB "${ARROW_DIR}/cpp/build/release/libarrow.a" CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") -set(ARROW_PYTHON_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow_python.a" +set(ARROW_PYTHON_LIB "${ARROW_DIR}/cpp/build/release/libarrow_python.a" CACHE STRING "Path to libarrow_python.a (needs to be changed if arrow is build in debug mode)") -include_directories("${ARROW_DIR}/cpp/src/") +# include_directories("${ARROW_DIR}/cpp/src/") include_directories("cpp/src/") include_directories("python/src/") -if(HAS_PLASMA) - include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma") - include_directories("${CMAKE_CURRENT_LIST_DIR}/../common") - include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty") -endif() - add_definitions(-fPIC) add_library(numbuf SHARED @@ -60,11 +57,11 @@ add_library(numbuf SHARED if(APPLE) target_link_libraries(numbuf "-undefined dynamic_lookup" ${ARROW_LIB} ${ARROW_PYTHON_LIB} -lpthread) else() - target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_PYTHON_LIB} -lpthread) + target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_PYTHON_LIB} -lpthread -lboost_system -lboost_filesystem) endif() if(HAS_PLASMA) - target_link_libraries(numbuf plasma_lib common) + target_link_libraries(numbuf ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a common) endif() install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/) diff --git a/src/numbuf/cpp/src/numbuf/sequence.cc b/src/numbuf/cpp/src/numbuf/sequence.cc index 56019a3a706a..69488737498a 100644 --- a/src/numbuf/cpp/src/numbuf/sequence.cc +++ b/src/numbuf/cpp/src/numbuf/sequence.cc @@ -104,17 +104,18 @@ Status SequenceBuilder::AppendDict(int32_t size) { #define ADD_SUBSEQUENCE(DATA, OFFSETS, BUILDER, TAG, NAME) \ if (DATA) { \ DCHECK(DATA->length() == OFFSETS.back()); \ - auto list_builder = std::make_shared(pool_, DATA); \ - auto field = std::make_shared(NAME, list_builder->type()); \ + std::shared_ptr offset_array; \ + Int32Builder builder(pool_, std::make_shared()); \ + RETURN_NOT_OK(builder.Append(OFFSETS.data(), OFFSETS.size())); \ + RETURN_NOT_OK(builder.Finish(&offset_array)); \ + std::shared_ptr list_array; \ + ListArray::FromArrays(*offset_array, *DATA, pool_, &list_array); \ + auto field = std::make_shared(NAME, list_array->type()); \ auto type = std::make_shared(std::vector({field})); \ - auto lists = std::vector>({list_builder}); \ - StructBuilder builder(pool_, type, lists); \ - OFFSETS.pop_back(); \ - RETURN_NOT_OK(list_builder->Append(OFFSETS.data(), OFFSETS.size())); \ - for (int i = 0; i < list_builder->length(); ++i) { \ - RETURN_NOT_OK(builder.Append()); \ - } \ - ADD_ELEMENT(builder, TAG); \ + types[TAG] = std::make_shared("", type); \ + children[TAG] = std::shared_ptr( \ + new StructArray(type, list_array->length(), {list_array})); \ + RETURN_NOT_OK(nones_.AppendToBitmap(true)); \ type_ids.push_back(TAG); \ } else { \ DCHECK(OFFSETS.size() == 1); \ diff --git a/src/numbuf/python/src/pynumbuf/numbuf.cc b/src/numbuf/python/src/pynumbuf/numbuf.cc index 6fd270d9bc34..63b947d82b3b 100644 --- a/src/numbuf/python/src/pynumbuf/numbuf.cc +++ b/src/numbuf/python/src/pynumbuf/numbuf.cc @@ -12,24 +12,23 @@ // plasma_protocol, because that file is used both with the store and the // manager, the store uses it the ObjectID from plasma_common.h and the // manager uses it with the ObjectID from common.h. -#include "plasma_common.h" +#include "plasma/common.h" -#include "plasma_client.h" -#include "plasma_protocol.h" +#include "plasma/client.h" +#include "plasma/protocol.h" extern "C" { PyObject* NumbufPlasmaOutOfMemoryError; PyObject* NumbufPlasmaObjectExistsError; } -#include "plasma_extension.h" +using namespace plasma; #endif #include #include #include -#include #include #include @@ -85,7 +84,7 @@ Status read_batch_and_tensors(uint8_t* data, int64_t size, auto source = std::make_shared( LENGTH_PREFIX_SIZE + data, size - LENGTH_PREFIX_SIZE); RETURN_NOT_OK(arrow::ipc::FileReader::Open(source, batch_size, &reader)); - RETURN_NOT_OK(reader->GetRecordBatch(0, batch_out)); + RETURN_NOT_OK(reader->ReadRecordBatch(0, batch_out)); int64_t offset = batch_size; while (true) { std::shared_ptr tensor; @@ -129,6 +128,26 @@ static void ArrowCapsule_Destructor(PyObject* capsule) { delete reinterpret_cast(PyCapsule_GetPointer(capsule, "arrow")); } +static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) { + if (PyCapsule_IsValid(object, "plasma")) { + *client = reinterpret_cast(PyCapsule_GetPointer(object, "plasma")); + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule"); + return 0; + } +} + +int PyStringToUniqueID(PyObject* object, ObjectID* object_id) { + if (PyBytes_Check(object)) { + memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID)); + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "must be a 20 character string"); + return 0; + } +} + /* Documented in doc/numbuf.rst in ray-core */ static PyObject* serialize_list(PyObject* self, PyObject* args) { PyObject* value; @@ -152,7 +171,7 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) { object->batch = make_batch(array); int64_t data_size, total_size; - auto mock = std::make_shared(); + auto mock = std::make_shared(); write_batch_and_tensors( mock.get(), object->batch, object->arrays, &data_size, &total_size); @@ -253,14 +272,15 @@ static PyObject* register_callbacks(PyObject* self, PyObject* args) { * @return Void. */ static void BufferCapsule_Destructor(PyObject* capsule) { - ObjectID* id = reinterpret_cast(PyCapsule_GetPointer(capsule, "buffer")); + plasma::ObjectID* id = + reinterpret_cast(PyCapsule_GetPointer(capsule, "buffer")); auto context = reinterpret_cast(PyCapsule_GetContext(capsule)); /* We use the context of the connection capsule to indicate if the connection * is still active (if the context is NULL) or if it is closed (if the context * is (void*) 0x1). This is neccessary because the primary pointer of the * capsule cannot be NULL. */ if (PyCapsule_GetContext(context) == NULL) { - PlasmaClient* client; + plasma::PlasmaClient* client; ARROW_CHECK(PyObjectToPlasmaClient(context, &client)); ARROW_CHECK_OK(client->Release(*id)); } @@ -282,7 +302,7 @@ static void BufferCapsule_Destructor(PyObject* capsule) { */ static PyObject* store_list(PyObject* self, PyObject* args) { ObjectID obj_id; - PlasmaClient* client; + plasma::PlasmaClient* client; PyObject* value; if (!PyArg_ParseTuple(args, "O&O&O", PyStringToUniqueID, &obj_id, PyObjectToPlasmaClient, &client, &value)) { @@ -300,7 +320,7 @@ static PyObject* store_list(PyObject* self, PyObject* args) { std::shared_ptr batch = make_batch(array); int64_t data_size, total_size; - auto mock = std::make_shared(); + auto mock = std::make_shared(); write_batch_and_tensors(mock.get(), batch, tensors, &data_size, &total_size); uint8_t* data; @@ -363,7 +383,7 @@ static PyObject* retrieve_list(PyObject* self, PyObject* args) { if (!PyArg_ParseTuple(args, "OOL", &object_id_list, &plasma_client, &timeout_ms)) { return NULL; } - PlasmaClient* client; + plasma::PlasmaClient* client; if (!PyObjectToPlasmaClient(plasma_client, &client)) { return NULL; } Py_ssize_t num_object_ids = PyList_Size(object_id_list); diff --git a/src/numbuf/thirdparty/build_thirdparty.sh b/src/numbuf/thirdparty/build_thirdparty.sh deleted file mode 100755 index b037c383d0e7..000000000000 --- a/src/numbuf/thirdparty/build_thirdparty.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -x - -# Cause the script to exit if a single command fails. -set -e - -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -PREFIX=$TP_DIR/installed - -# Determine how many parallel jobs to use for make based on the number of cores -unamestr="$(uname)" -if [[ "$unamestr" == "Linux" ]]; then - PARALLEL=$(nproc) -elif [[ "$unamestr" == "Darwin" ]]; then - PARALLEL=$(sysctl -n hw.ncpu) - echo "Platform is macosx." -else - echo "Unrecognized platform." - exit 1 -fi - -echo "building arrow" -cd $TP_DIR/arrow/cpp -mkdir -p $TP_DIR/arrow/cpp/build -cd $TP_DIR/arrow/cpp/build -cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g -O3" -DCMAKE_CXX_FLAGS="-g -O3" -DARROW_BUILD_TESTS=OFF -DARROW_HDFS=OFF -DARROW_PYTHON=on .. -make VERBOSE=1 -j$PARALLEL diff --git a/src/plasma/CMakeLists.txt b/src/plasma/CMakeLists.txt index 6b949c1c7eab..4b6b54d4b23d 100644 --- a/src/plasma/CMakeLists.txt +++ b/src/plasma/CMakeLists.txt @@ -5,102 +5,30 @@ project(plasma) # Recursively include common include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) -if(APPLE) - SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif(APPLE) +# Include plasma +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules) -include_directories("${PYTHON_INCLUDE_DIRS}" thirdparty) +find_package(Plasma) +include_directories(SYSTEM ${PLASMA_INCLUDE_DIR}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++11 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -O3") -# Compile flatbuffers - -set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs") -set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/) - -set(PLASMA_FBS_OUTPUT_FILES - "${OUTPUT_DIR}/plasma_generated.h") - -add_custom_command( - OUTPUT ${PLASMA_FBS_OUTPUT_FILES} - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC} - DEPENDS ${PLASMA_FBS_SRC} - COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}" - VERBATIM) - -add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES}) - -add_dependencies(gen_plasma_fbs flatbuffers_ep) - if(UNIX AND NOT APPLE) link_libraries(rt) endif() -include_directories("${CMAKE_CURRENT_LIST_DIR}/") -include_directories("${CMAKE_CURRENT_LIST_DIR}/../") - -add_library(plasma SHARED - plasma.cc - plasma_extension.cc - plasma_protocol.cc - plasma_client.cc - thirdparty/xxhash.c - fling.c) - -add_dependencies(plasma gen_plasma_fbs) - -if(APPLE) - target_link_libraries(plasma plasma_lib "-undefined dynamic_lookup" -Wl,-force_load,${FLATBUFFERS_STATIC_LIB} ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB} -lpthread) -else(APPLE) - target_link_libraries(plasma plasma_lib -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB} -lpthread) -endif(APPLE) +include_directories("${ARROW_DIR}/cpp/src/") +# include_directories("${CMAKE_CURRENT_LIST_DIR}/../") include_directories("${FLATBUFFERS_INCLUDE_DIR}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") -set_source_files_properties(thirdparty/dlmalloc.c PROPERTIES COMPILE_FLAGS -Wno-all) - -add_library(plasma_lib STATIC - plasma_client.cc - plasma.cc - plasma_common.cc - plasma_io.cc - plasma_protocol.cc - status.cc - fling.c - thirdparty/xxhash.c) - -target_link_libraries(plasma_lib ${FLATBUFFERS_STATIC_LIB} -lpthread) -add_dependencies(plasma_lib gen_plasma_fbs) - -add_executable(plasma_store - plasma_store.cc - thirdparty/ae/ae.c - plasma.cc - plasma_events.cc - plasma_protocol.cc - eviction_policy.cc - fling.c - malloc.cc) - -add_dependencies(plasma_store hiredis gen_plasma_fbs) - -target_link_libraries(plasma_store plasma_lib ${FLATBUFFERS_STATIC_LIB}) - -add_dependencies(plasma protocol_fbs) - add_executable(plasma_manager plasma_manager.cc) -target_link_libraries(plasma_manager common plasma_lib ${FLATBUFFERS_STATIC_LIB}) - -add_library(plasma_client SHARED plasma_client.cc) -target_link_libraries(plasma_client ${FLATBUFFERS_STATIC_LIB}) - -target_link_libraries(plasma_client plasma_lib ${FLATBUFFERS_STATIC_LIB}) +target_link_libraries(plasma_manager common ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a -lpthread) -define_test(client_tests plasma_lib) -define_test(manager_tests plasma_lib plasma_manager.cc) -define_test(serialization_tests plasma_lib) +define_test(client_tests ${PLASMA_STATIC_LIB}) +define_test(manager_tests ${PLASMA_STATIC_LIB} plasma_manager.cc) diff --git a/src/plasma/eviction_policy.cc b/src/plasma/eviction_policy.cc deleted file mode 100644 index 135c63ad0d40..000000000000 --- a/src/plasma/eviction_policy.cc +++ /dev/null @@ -1,95 +0,0 @@ -#include "eviction_policy.h" - -void LRUCache::add(const ObjectID &key, int64_t size) { - auto it = item_map_.find(key); - ARROW_CHECK(it == item_map_.end()); - /* Note that it is important to use a list so the iterators stay valid. */ - item_list_.emplace_front(key, size); - item_map_.emplace(key, item_list_.begin()); -} - -void LRUCache::remove(const ObjectID &key) { - auto it = item_map_.find(key); - ARROW_CHECK(it != item_map_.end()); - item_list_.erase(it->second); - item_map_.erase(it); -} - -int64_t LRUCache::choose_objects_to_evict( - int64_t num_bytes_required, - std::vector &objects_to_evict) { - int64_t bytes_evicted = 0; - auto it = item_list_.end(); - while (bytes_evicted < num_bytes_required && it != item_list_.begin()) { - it--; - objects_to_evict.push_back(it->first); - bytes_evicted += it->second; - } - return bytes_evicted; -} - -EvictionPolicy::EvictionPolicy(PlasmaStoreInfo *store_info) - : memory_used_(0), store_info_(store_info) {} - -int64_t EvictionPolicy::choose_objects_to_evict( - int64_t num_bytes_required, - std::vector &objects_to_evict) { - int64_t bytes_evicted = - cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict); - /* Update the LRU cache. */ - for (auto &object_id : objects_to_evict) { - cache_.remove(object_id); - } - /* Update the number of bytes used. */ - memory_used_ -= bytes_evicted; - return bytes_evicted; -} - -void EvictionPolicy::object_created(ObjectID object_id) { - auto entry = store_info_->objects[object_id].get(); - cache_.add(object_id, entry->info.data_size + entry->info.metadata_size); -} - -bool EvictionPolicy::require_space(int64_t size, - std::vector &objects_to_evict) { - /* Check if there is enough space to create the object. */ - int64_t required_space = memory_used_ + size - store_info_->memory_capacity; - int64_t num_bytes_evicted; - if (required_space > 0) { - /* Try to free up at least as much space as we need right now but ideally - * up to 20% of the total capacity. */ - int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5); - ARROW_LOG(DEBUG) - << "not enough space to create this object, so evicting objects"; - /* Choose some objects to evict, and update the return pointers. */ - num_bytes_evicted = - choose_objects_to_evict(space_to_free, objects_to_evict); - ARROW_LOG(INFO) - << "There is not enough space to create this object, so evicting " - << objects_to_evict.size() << " objects to free up " - << num_bytes_evicted << " bytes."; - } else { - num_bytes_evicted = 0; - } - if (num_bytes_evicted >= required_space) { - /* We only increment the space used if there is enough space to create the - * object. */ - memory_used_ += size; - } - return num_bytes_evicted >= required_space; -} - -void EvictionPolicy::begin_object_access( - ObjectID object_id, - std::vector &objects_to_evict) { - /* If the object is in the LRU cache, remove it. */ - cache_.remove(object_id); -} - -void EvictionPolicy::end_object_access( - ObjectID object_id, - std::vector &objects_to_evict) { - auto entry = store_info_->objects[object_id].get(); - /* Add the object to the LRU cache.*/ - cache_.add(object_id, entry->info.data_size + entry->info.metadata_size); -} diff --git a/src/plasma/eviction_policy.h b/src/plasma/eviction_policy.h deleted file mode 100644 index fd3861db4677..000000000000 --- a/src/plasma/eviction_policy.h +++ /dev/null @@ -1,128 +0,0 @@ -#ifndef EVICTION_POLICY_H -#define EVICTION_POLICY_H - -#include -#include - -#include "plasma_common.h" -#include "plasma.h" - -/* ==== The eviction policy ==== - * - * This file contains declaration for all functions and data structures that - * need to be provided if you want to implement a new eviction algorithm for the - * Plasma store. - */ - -class LRUCache { - private: - /** A doubly-linked list containing the items in the cache and - * their sizes in LRU order. */ - typedef std::list> ItemList; - ItemList item_list_; - /** A hash table mapping the object ID of an object in the cache to its - * location in the doubly linked list item_list_. */ - std::unordered_map item_map_; - - public: - LRUCache(){}; - - void add(const ObjectID &key, int64_t size); - - void remove(const ObjectID &key); - - int64_t choose_objects_to_evict(int64_t num_bytes_required, - std::vector &objects_to_evict); -}; - -/** The eviction policy. */ -class EvictionPolicy { - public: - /** - * Construct an eviction policy. - * - * @param store_info Information about the Plasma store that is exposed - * to the eviction policy. - */ - EvictionPolicy(PlasmaStoreInfo *store_info); - - /** - * This method will be called whenever an object is first created in order to - * add it to the LRU cache. This is done so that the first time, the Plasma - * store calls begin_object_access, we can remove the object from the LRU - * cache. - * - * @param object_id The object ID of the object that was created. - * @return Void. - */ - void object_created(ObjectID object_id); - - /** - * This method will be called when the Plasma store needs more space, perhaps - * to create a new object. If the required amount of space cannot be freed up, - * then a fatal error will be thrown. When this method is called, the eviction - * policy will assume that the objects chosen to be evicted will in fact be - * evicted from the Plasma store by the caller. - * - * @param size The size in bytes of the new object, including both data and - * metadata. - * @param objects_to_evict The object IDs that were chosen for eviction will - * be stored into this vector. - * @return True if enough space can be freed and false otherwise. - */ - bool require_space(int64_t size, std::vector &objects_to_evict); - - /** - * This method will be called whenever an unused object in the Plasma store - * starts to be used. When this method is called, the eviction policy will - * assume that the objects chosen to be evicted will in fact be evicted from - * the Plasma store by the caller. - * - * @param object_id The ID of the object that is now being used. - * @param objects_to_evict The object IDs that were chosen for eviction will - * be stored into this vector. - * @return Void. - */ - void begin_object_access(ObjectID object_id, - std::vector &objects_to_evict); - - /** - * This method will be called whenever an object in the Plasma store that was - * being used is no longer being used. When this method is called, the - * eviction policy will assume that the objects chosen to be evicted will in - * fact be evicted from the Plasma store by the caller. - * - * @param object_id The ID of the object that is no longer being used. - * @param objects_to_evict The object IDs that were chosen for eviction will - * be stored into this vector. - * @return Void. - */ - void end_object_access(ObjectID object_id, - std::vector &objects_to_evict); - - /** - * Choose some objects to evict from the Plasma store. When this method is - * called, the eviction policy will assume that the objects chosen to be - * evicted will in fact be evicted from the Plasma store by the caller. - * - * @note This method is not part of the API. It is exposed in the header file - * only for testing. - * - * @param num_bytes_required The number of bytes of space to try to free up. - * @param objects_to_evict The object IDs that were chosen for eviction will - * be stored into this vector. - * @return The total number of bytes of space chosen to be evicted. - */ - int64_t choose_objects_to_evict(int64_t num_bytes_required, - std::vector &objects_to_evict); - - private: - /** Pointer to the plasma store info. */ - PlasmaStoreInfo *store_info_; - /** The amount of memory (in bytes) currently being used. */ - int64_t memory_used_; - /** Datastructure for the LRU cache. */ - LRUCache cache_; -}; - -#endif /* EVICTION_POLICY_H */ diff --git a/src/plasma/fling.c b/src/plasma/fling.c deleted file mode 100644 index 379b896b543e..000000000000 --- a/src/plasma/fling.c +++ /dev/null @@ -1,76 +0,0 @@ -#include "fling.h" - -#include - -void init_msg(struct msghdr *msg, - struct iovec *iov, - char *buf, - size_t buf_len) { - iov->iov_base = buf; - iov->iov_len = 1; - - msg->msg_iov = iov; - msg->msg_iovlen = 1; - msg->msg_control = buf; - msg->msg_controllen = buf_len; - msg->msg_name = NULL; - msg->msg_namelen = 0; -} - -int send_fd(int conn, int fd) { - struct msghdr msg; - struct iovec iov; - char buf[CMSG_SPACE(sizeof(int))]; - memset(&buf, 0, CMSG_SPACE(sizeof(int))); - - init_msg(&msg, &iov, buf, sizeof(buf)); - - struct cmsghdr *header = CMSG_FIRSTHDR(&msg); - header->cmsg_level = SOL_SOCKET; - header->cmsg_type = SCM_RIGHTS; - header->cmsg_len = CMSG_LEN(sizeof(int)); - *(int *) CMSG_DATA(header) = fd; - - /* Send file descriptor. */ - return sendmsg(conn, &msg, 0); -} - -int recv_fd(int conn) { - struct msghdr msg; - struct iovec iov; - char buf[CMSG_SPACE(sizeof(int))]; - init_msg(&msg, &iov, buf, sizeof(buf)); - - if (recvmsg(conn, &msg, 0) == -1) - return -1; - - int found_fd = -1; - int oh_noes = 0; - for (struct cmsghdr *header = CMSG_FIRSTHDR(&msg); header != NULL; - header = CMSG_NXTHDR(&msg, header)) - if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) { - int count = - (header->cmsg_len - (CMSG_DATA(header) - (unsigned char *) header)) / - sizeof(int); - for (int i = 0; i < count; ++i) { - int fd = ((int *) CMSG_DATA(header))[i]; - if (found_fd == -1) { - found_fd = fd; - } else { - close(fd); - oh_noes = 1; - } - } - } - - /* The sender sent us more than one file descriptor. We've closed - * them all to prevent fd leaks but notify the caller that we got - * a bad message. */ - if (oh_noes) { - close(found_fd); - errno = EBADMSG; - return -1; - } - - return found_fd; -} diff --git a/src/plasma/fling.h b/src/plasma/fling.h deleted file mode 100644 index efc41d801e89..000000000000 --- a/src/plasma/fling.h +++ /dev/null @@ -1,43 +0,0 @@ -/* FLING: Exchanging file descriptors over sockets - * - * This is a little library for sending file descriptors over a socket - * between processes. The reason for doing that (as opposed to using - * filenames to share the files) is so (a) no files remain in the - * filesystem after all the processes terminate, (b) to make sure that - * there are no name collisions and (c) to be able to control who has - * access to the data. - * - * Most of the code is from https://github.com/sharvil/flingfd */ - -#include -#include -#include -#include -#include - -/* This is neccessary for Mac OS X, see http://www.apuebook.com/faqs2e.html - * (10). */ -#if !defined(CMSG_SPACE) && !defined(CMSG_LEN) -#define CMSG_SPACE(len) \ - (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + __DARWIN_ALIGN32(len)) -#define CMSG_LEN(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + (len)) -#endif - -void init_msg(struct msghdr *msg, struct iovec *iov, char *buf, size_t buf_len); - -/** - * Send a file descriptor over a unix domain socket. - * - * @param conn Unix domain socket to send the file descriptor over. - * @param fd File descriptor to send over. - * @return Status code which is < 0 on failure. - */ -int send_fd(int conn, int fd); - -/** - * Receive a file descriptor over a unix domain socket. - * - * @param conn Unix domain socket to receive the file descriptor from. - * @return File descriptor or a value < 0 on failure. - */ -int recv_fd(int conn); diff --git a/src/plasma/logging.h b/src/plasma/logging.h deleted file mode 100644 index 917d18140ddb..000000000000 --- a/src/plasma/logging.h +++ /dev/null @@ -1,147 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef ARROW_UTIL_LOGGING_H -#define ARROW_UTIL_LOGGING_H - -#include -#include - -namespace arrow { - -// Stubbed versions of macros defined in glog/logging.h, intended for -// environments where glog headers aren't available. -// -// Add more as needed. - -// Log levels. LOG ignores them, so their values are abitrary. - -#define ARROW_DEBUG (-1) -#define ARROW_INFO 0 -#define ARROW_WARNING 1 -#define ARROW_ERROR 2 -#define ARROW_FATAL 3 - -#define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level) -#define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level) - -#define ARROW_CHECK(condition) \ - (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \ - << __FILE__ << __LINE__ \ - << " Check failed: " #condition " " - -#ifdef NDEBUG -#define ARROW_DFATAL ARROW_WARNING - -#define DCHECK(condition) \ - while (false) \ - ::arrow::internal::NullLog() -#define DCHECK_EQ(val1, val2) \ - while (false) \ - ::arrow::internal::NullLog() -#define DCHECK_NE(val1, val2) \ - while (false) \ - ::arrow::internal::NullLog() -#define DCHECK_LE(val1, val2) \ - while (false) \ - ::arrow::internal::NullLog() -#define DCHECK_LT(val1, val2) \ - while (false) \ - ::arrow::internal::NullLog() -#define DCHECK_GE(val1, val2) \ - while (false) \ - ::arrow::internal::NullLog() -#define DCHECK_GT(val1, val2) \ - while (false) \ - ::arrow::internal::NullLog() - -#else -#define ARROW_DFATAL ARROW_FATAL - -#define DCHECK(condition) ARROW_CHECK(condition) -#define DCHECK_EQ(val1, val2) ARROW_CHECK((val1) == (val2)) -#define DCHECK_NE(val1, val2) ARROW_CHECK((val1) != (val2)) -#define DCHECK_LE(val1, val2) ARROW_CHECK((val1) <= (val2)) -#define DCHECK_LT(val1, val2) ARROW_CHECK((val1) < (val2)) -#define DCHECK_GE(val1, val2) ARROW_CHECK((val1) >= (val2)) -#define DCHECK_GT(val1, val2) ARROW_CHECK((val1) > (val2)) - -#endif // NDEBUG - -namespace internal { - -class NullLog { - public: - template - NullLog &operator<<(const T &t) { - return *this; - } -}; - -class CerrLog { - public: - CerrLog(int severity) // NOLINT(runtime/explicit) - : severity_(severity), - has_logged_(false) {} - - virtual ~CerrLog() { - if (has_logged_) { - std::cerr << std::endl; - } - if (severity_ == ARROW_FATAL) { - std::exit(1); - } - } - - template - CerrLog &operator<<(const T &t) { - // TODO(pcm): Print this if in debug mode, but not if in valgrind - // mode - if (severity_ == ARROW_DEBUG) { - return *this; - } - - has_logged_ = true; - std::cerr << t; - return *this; - } - - protected: - const int severity_; - bool has_logged_; -}; - -// Clang-tidy isn't smart enough to determine that DCHECK using CerrLog doesn't -// return so we create a new class to give it a hint. -class FatalLog : public CerrLog { - public: - explicit FatalLog(int /* severity */) // NOLINT - : CerrLog(ARROW_FATAL){} // NOLINT - - [[noreturn]] ~FatalLog() { - if (has_logged_) { - std::cerr << std::endl; - } - std::exit(1); - } -}; - -} // namespace internal - -} // namespace arrow - -#endif // ARROW_UTIL_LOGGING_H diff --git a/src/plasma/malloc.cc b/src/plasma/malloc.cc deleted file mode 100644 index e8399649a98d..000000000000 --- a/src/plasma/malloc.cc +++ /dev/null @@ -1,168 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "common.h" - -extern "C" { -void *fake_mmap(size_t); -int fake_munmap(void *, size_t); - -#define MMAP(s) fake_mmap(s) -#define MUNMAP(a, s) fake_munmap(a, s) -#define DIRECT_MMAP(s) fake_mmap(s) -#define DIRECT_MUNMAP(a, s) fake_munmap(a, s) -#define USE_DL_PREFIX -#define HAVE_MORECORE 0 -#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T -#define DEFAULT_GRANULARITY ((size_t) 128U * 1024U) - -#include "thirdparty/dlmalloc.c" - -#undef MMAP -#undef MUNMAP -#undef DIRECT_MMAP -#undef DIRECT_MUNMAP -#undef USE_DL_PREFIX -#undef HAVE_MORECORE -#undef DEFAULT_GRANULARITY -} - -struct mmap_record { - int fd; - int64_t size; -}; - -namespace { - -/** Hashtable that contains one entry per segment that we got from the OS - * via mmap. Associates the address of that segment with its file descriptor - * and size. */ -std::unordered_map mmap_records; - -} /* namespace */ - -constexpr int GRANULARITY_MULTIPLIER = 2; - -static void *pointer_advance(void *p, ptrdiff_t n) { - return (unsigned char *) p + n; -} - -static void *pointer_retreat(void *p, ptrdiff_t n) { - return (unsigned char *) p - n; -} - -static ptrdiff_t pointer_distance(void const *pfrom, void const *pto) { - return (unsigned char const *) pto - (unsigned char const *) pfrom; -} - -/* Create a buffer. This is creating a temporary file and then - * immediately unlinking it so we do not leave traces in the system. */ -int create_buffer(int64_t size) { - int fd; -#ifdef _WIN32 - if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, - (DWORD)((uint64_t) size >> (CHAR_BIT * sizeof(DWORD))), - (DWORD)(uint64_t) size, NULL)) { - fd = -1; - } -#else -#ifdef __linux__ - constexpr char file_template[] = "/dev/shm/plasmaXXXXXX"; -#else - constexpr char file_template[] = "/tmp/plasmaXXXXXX"; -#endif - char file_name[32]; - strncpy(file_name, file_template, 32); - fd = mkstemp(file_name); - if (fd < 0) - return -1; - FILE *file = fdopen(fd, "a+"); - if (!file) { - close(fd); - return -1; - } - if (unlink(file_name) != 0) { - LOG_ERROR("unlink error"); - return -1; - } - if (ftruncate(fd, (off_t) size) != 0) { - LOG_ERROR("ftruncate error"); - return -1; - } -#endif - return fd; -} - -void *fake_mmap(size_t size) { - /* Add sizeof(size_t) so that the returned pointer is deliberately not - * page-aligned. This ensures that the segments of memory returned by - * fake_mmap are never contiguous. */ - size += sizeof(size_t); - - int fd = create_buffer(size); - CHECKM(fd >= 0, "Failed to create buffer during mmap"); - void *pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (pointer == MAP_FAILED) { - return pointer; - } - - /* Increase dlmalloc's allocation granularity directly. */ - mparams.granularity *= GRANULARITY_MULTIPLIER; - - mmap_record &record = mmap_records[pointer]; - record.fd = fd; - record.size = size; - - /* We lie to dlmalloc about where mapped memory actually lives. */ - pointer = pointer_advance(pointer, sizeof(size_t)); - LOG_DEBUG("%p = fake_mmap(%lu)", pointer, size); - return pointer; -} - -int fake_munmap(void *addr, size_t size) { - LOG_DEBUG("fake_munmap(%p, %lu)", addr, size); - addr = pointer_retreat(addr, sizeof(size_t)); - size += sizeof(size_t); - - auto entry = mmap_records.find(addr); - - if (entry == mmap_records.end() || entry->second.size != size) { - /* Reject requests to munmap that don't directly match previous - * calls to mmap, to prevent dlmalloc from trimming. */ - return -1; - } - - int r = munmap(addr, size); - if (r == 0) { - close(entry->second.fd); - } - - mmap_records.erase(entry); - return r; -} - -void get_malloc_mapinfo(void *addr, - int *fd, - int64_t *map_size, - ptrdiff_t *offset) { - /* TODO(rshin): Implement a more efficient search through mmap_records. */ - for (const auto &entry : mmap_records) { - if (addr >= entry.first && - addr < pointer_advance(entry.first, entry.second.size)) { - *fd = entry.second.fd; - *map_size = entry.second.size; - *offset = pointer_distance(entry.first, addr); - return; - } - } - *fd = -1; - *map_size = 0; - *offset = 0; -} diff --git a/src/plasma/malloc.h b/src/plasma/malloc.h deleted file mode 100644 index 9fc1f48bb9e6..000000000000 --- a/src/plasma/malloc.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef MALLOC_H -#define MALLOC_H - -void get_malloc_mapinfo(void *addr, - int *fd, - int64_t *map_length, - ptrdiff_t *offset); - -#endif /* MALLOC_H */ diff --git a/src/plasma/plasma.cc b/src/plasma/plasma.cc deleted file mode 100644 index 6273d3bbc8c4..000000000000 --- a/src/plasma/plasma.cc +++ /dev/null @@ -1,53 +0,0 @@ -#include "plasma_common.h" -#include "plasma.h" - -#include "io.h" -#include -#include -#include - -#include "plasma_protocol.h" - -int warn_if_sigpipe(int status, int client_sock) { - if (status >= 0) { - return 0; - } - if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) { - ARROW_LOG(WARNING) - << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " - "sending a message to client on fd " - << client_sock << ". The client on the other end may " - "have hung up."; - return errno; - } - ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock - << "."; -} - -/** - * This will create a new ObjectInfo buffer. The first sizeof(int64_t) bytes - * of this buffer are the length of the remaining message and the - * remaining message is a serialized version of the object info. - * - * @param object_info The object info to be serialized - * @return The object info buffer. It is the caller's responsibility to free - * this buffer with "free" after it has been used. - */ -uint8_t *create_object_info_buffer(ObjectInfoT *object_info) { - flatbuffers::FlatBufferBuilder fbb; - auto message = CreateObjectInfo(fbb, object_info); - fbb.Finish(message); - uint8_t *notification = (uint8_t *) malloc(sizeof(int64_t) + fbb.GetSize()); - *((int64_t *) notification) = fbb.GetSize(); - memcpy(notification + sizeof(int64_t), fbb.GetBufferPointer(), fbb.GetSize()); - return notification; -} - -ObjectTableEntry *get_object_table_entry(PlasmaStoreInfo *store_info, - ObjectID object_id) { - auto it = store_info->objects.find(object_id); - if (it == store_info->objects.end()) { - return NULL; - } - return it->second.get(); -} diff --git a/src/plasma/plasma.h b/src/plasma/plasma.h deleted file mode 100644 index c3b2d89c786b..000000000000 --- a/src/plasma/plasma.h +++ /dev/null @@ -1,186 +0,0 @@ -#ifndef PLASMA_H -#define PLASMA_H - -#include -#include -#include -#include -#include -#include /* pid_t */ - -extern "C" { -#include "sha256.h" -} - -#include -#include - -#include "format/common_generated.h" -#include "logging.h" -#include "status.h" - -#include - -#define HANDLE_SIGPIPE(s, fd_) \ - do { \ - Status _s = (s); \ - if (!_s.ok()) { \ - if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) { \ - ARROW_LOG(WARNING) \ - << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \ - "sending a message to client on fd " \ - << fd_ << ". " \ - "The client on the other end may have hung up."; \ - } else { \ - return _s; \ - } \ - } \ - } while (0); - -/** Allocation granularity used in plasma for object allocation. */ -#define BLOCK_SIZE 64 - -// Size of object hash digests. -constexpr int64_t kDigestSize = SHA256_BLOCK_SIZE; - -struct Client; - -/** - * Object request data structure. Used in the plasma_wait_for_objects() - * argument. - */ -typedef struct { - /** The ID of the requested object. If ID_NIL request any object. */ - ObjectID object_id; - /** Request associated to the object. It can take one of the following values: - * - PLASMA_QUERY_LOCAL: return if or when the object is available in the - * local Plasma Store. - * - PLASMA_QUERY_ANYWHERE: return if or when the object is available in - * the system (i.e., either in the local or a remote Plasma Store). */ - int type; - /** Object status. Same as the status returned by plasma_status() function - * call. This is filled in by plasma_wait_for_objects1(): - * - ObjectStatus_Local: object is ready at the local Plasma Store. - * - ObjectStatus_Remote: object is ready at a remote Plasma Store. - * - ObjectStatus_Nonexistent: object does not exist in the system. - * - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled - * for being transferred or it is transferring. */ - int status; -} ObjectRequest; - -/** Mapping from object IDs to type and status of the request. */ -typedef std::unordered_map - ObjectRequestMap; - -/* Handle to access memory mapped file and map it into client address space. */ -typedef struct { - /** The file descriptor of the memory mapped file in the store. It is used as - * a unique identifier of the file in the client to look up the corresponding - * file descriptor on the client's side. */ - int store_fd; - /** The size in bytes of the memory mapped file. */ - int64_t mmap_size; -} object_handle; - -typedef struct { - /** Handle for memory mapped file the object is stored in. */ - object_handle handle; - /** The offset in bytes in the memory mapped file of the data. */ - ptrdiff_t data_offset; - /** The offset in bytes in the memory mapped file of the metadata. */ - ptrdiff_t metadata_offset; - /** The size in bytes of the data. */ - int64_t data_size; - /** The size in bytes of the metadata. */ - int64_t metadata_size; -} PlasmaObject; - -typedef enum { - /** Object was created but not sealed in the local Plasma Store. */ - PLASMA_CREATED = 1, - /** Object is sealed and stored in the local Plasma Store. */ - PLASMA_SEALED -} object_state; - -typedef enum { - /** The object was not found. */ - OBJECT_NOT_FOUND = 0, - /** The object was found. */ - OBJECT_FOUND = 1 -} object_status; - -typedef enum { - /** Query for object in the local plasma store. */ - PLASMA_QUERY_LOCAL = 1, - /** Query for object in the local plasma store or in a remote plasma store. */ - PLASMA_QUERY_ANYWHERE -} object_request_type; - -/** This type is used by the Plasma store. It is here because it is exposed to - * the eviction policy. */ -struct ObjectTableEntry { - /** Object id of this object. */ - ObjectID object_id; - /** Object info like size, creation time and owner. */ - ObjectInfoT info; - /** Memory mapped file containing the object. */ - int fd; - /** Size of the underlying map. */ - int64_t map_size; - /** Offset from the base of the mmap. */ - ptrdiff_t offset; - /** Pointer to the object data. Needed to free the object. */ - uint8_t *pointer; - /** Set of clients currently using this object. */ - std::unordered_set clients; - /** The state of the object, e.g., whether it is open or sealed. */ - object_state state; - /** The digest of the object. Used to see if two objects are the same. */ - unsigned char digest[kDigestSize]; -}; - -/** The plasma store information that is exposed to the eviction policy. */ -struct PlasmaStoreInfo { - /** Objects that are in the Plasma store. */ - std::unordered_map, - UniqueIDHasher> - objects; - /** The amount of memory (in bytes) that we allow to be allocated in the - * store. */ - int64_t memory_capacity; -}; - -/** - * Get an entry from the object table and return NULL if the object_id - * is not present. - * - * @param store_info The PlasmaStoreInfo that contains the object table. - * @param object_id The object_id of the entry we are looking for. - * @return The entry associated with the object_id or NULL if the object_id - * is not present. - */ -ObjectTableEntry *get_object_table_entry(PlasmaStoreInfo *store_info, - ObjectID object_id); - -/** - * Print a warning if the status is less than zero. This should be used to check - * the success of messages sent to plasma clients. We print a warning instead of - * failing because the plasma clients are allowed to die. This is used to handle - * situations where the store writes to a client file descriptor, and the client - * may already have disconnected. If we have processed the disconnection and - * closed the file descriptor, we should get a BAD FILE DESCRIPTOR error. If we - * have not, then we should get a SIGPIPE. If we write to a TCP socket that - * isn't connected yet, then we should get an ECONNRESET. - * - * @param status The status to check. If it is less less than zero, we will - * print a warning. - * @param client_sock The client socket. This is just used to print some extra - * information. - * @return The errno set. - */ -int warn_if_sigpipe(int status, int client_sock); - -uint8_t *create_object_info_buffer(ObjectInfoT *object_info); - -#endif /* PLASMA_H */ diff --git a/src/plasma/plasma_client.cc b/src/plasma/plasma_client.cc deleted file mode 100644 index c811b418b6fa..000000000000 --- a/src/plasma/plasma_client.cc +++ /dev/null @@ -1,624 +0,0 @@ -// PLASMA CLIENT: Client library for using the plasma store and manager - -#ifdef _WIN32 -#include -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "plasma_common.h" -#include "plasma.h" -#include "plasma_io.h" -#include "plasma_protocol.h" -#include "plasma_client.h" - -#include -#include - -extern "C" { -#include "sha256.h" -#include "fling.h" - -#define XXH_STATIC_LINKING_ONLY -#include "xxhash.h" - -#define XXH64_DEFAULT_SEED 0 -} - -// Number of threads used for memcopy and hash computations. -constexpr int64_t kThreadPoolSize = 8; -constexpr int64_t kBytesInMB = 1 << 20; -static std::vector threadpool_(kThreadPoolSize); - -struct ClientMmapTableEntry { - /// The result of mmap for this file descriptor. - uint8_t *pointer; - /// The length of the memory-mapped file. - size_t length; - /// The number of objects in this memory-mapped file that are currently being - /// used by the client. When this count reaches zeros, we unmap the file. - int count; -}; - -struct ObjectInUseEntry { - /// A count of the number of times this client has called PlasmaClient::Create - /// or - /// PlasmaClient::Get on this object ID minus the number of calls to - /// PlasmaClient::Release. - /// When this count reaches zero, we remove the entry from the ObjectsInUse - /// and decrement a count in the relevant ClientMmapTableEntry. - int count; - /// Cached information to read the object. - PlasmaObject object; - /// A flag representing whether the object has been sealed. - bool is_sealed; -}; - -// If the file descriptor fd has been mmapped in this client process before, -// return the pointer that was returned by mmap, otherwise mmap it and store the -// pointer in a hash table. -uint8_t *lookup_or_mmap(PlasmaClient *conn, - int fd, - int store_fd_val, - int64_t map_size) { - auto entry = conn->mmap_table.find(store_fd_val); - if (entry != conn->mmap_table.end()) { - close(fd); - return entry->second->pointer; - } else { - uint8_t *result = (uint8_t *) mmap(NULL, map_size, PROT_READ | PROT_WRITE, - MAP_SHARED, fd, 0); - if (result == MAP_FAILED) { - ARROW_LOG(FATAL) << "mmap failed"; - } - close(fd); - ClientMmapTableEntry *entry = new ClientMmapTableEntry(); - entry->pointer = result; - entry->length = map_size; - entry->count = 0; - conn->mmap_table[store_fd_val] = entry; - return result; - } -} - -// Get a pointer to a file that we know has been memory mapped in this client -// process before. -uint8_t *lookup_mmapped_file(PlasmaClient *conn, int store_fd_val) { - auto entry = conn->mmap_table.find(store_fd_val); - ARROW_CHECK(entry != conn->mmap_table.end()); - return entry->second->pointer; -} - -void increment_object_count(PlasmaClient *conn, - ObjectID object_id, - PlasmaObject *object, - bool is_sealed) { - // Increment the count of the object to track the fact that it is being used. - // The corresponding decrement should happen in PlasmaClient::Release. - auto elem = conn->objects_in_use.find(object_id); - ObjectInUseEntry *object_entry; - if (elem == conn->objects_in_use.end()) { - // Add this object ID to the hash table of object IDs in use. The - // corresponding call to free happens in PlasmaClient::Release. - object_entry = new ObjectInUseEntry(); - object_entry->object = *object; - object_entry->count = 0; - object_entry->is_sealed = is_sealed; - conn->objects_in_use[object_id] = object_entry; - // Increment the count of the number of objects in the memory-mapped file - // that are being used. The corresponding decrement should happen in - // PlasmaClient::Release. - auto entry = conn->mmap_table.find(object->handle.store_fd); - ARROW_CHECK(entry != conn->mmap_table.end()); - ARROW_CHECK(entry->second->count >= 0); - // Update the in_use_object_bytes. - conn->in_use_object_bytes += - (object_entry->object.data_size + object_entry->object.metadata_size); - entry->second->count += 1; - } else { - object_entry = elem->second; - ARROW_CHECK(object_entry->count > 0); - } - // Increment the count of the number of instances of this object that are - // being used by this client. The corresponding decrement should happen in - // PlasmaClient::Release. - object_entry->count += 1; -} - -Status PlasmaClient::Create(ObjectID object_id, - int64_t data_size, - uint8_t *metadata, - int64_t metadata_size, - uint8_t **data) { - ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn - << " with size " << data_size << " and metadata size " - << metadata_size; - RETURN_NOT_OK( - SendCreateRequest(store_conn, object_id, data_size, metadata_size)); - std::vector buffer; - RETURN_NOT_OK( - PlasmaReceive(store_conn, MessageType_PlasmaCreateReply, buffer)); - ObjectID id; - PlasmaObject object; - RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object)); - // If the CreateReply included an error, then the store will not send a file - // descriptor. - int fd = recv_fd(store_conn); - ARROW_CHECK(fd >= 0) << "recv not successful"; - ARROW_CHECK(object.data_size == data_size); - ARROW_CHECK(object.metadata_size == metadata_size); - // The metadata should come right after the data. - ARROW_CHECK(object.metadata_offset == object.data_offset + data_size); - *data = lookup_or_mmap(this, fd, object.handle.store_fd, - object.handle.mmap_size) + - object.data_offset; - // If plasma_create is being called from a transfer, then we will not copy the - // metadata here. The metadata will be written along with the data streamed - // from the transfer. - if (metadata != NULL) { - // Copy the metadata to the buffer. - memcpy(*data + object.data_size, metadata, metadata_size); - } - // Increment the count of the number of instances of this object that this - // client is using. A call to PlasmaClient::Release is required to decrement - // this - // count. Cache the reference to the object. - increment_object_count(this, object_id, &object, false); - // We increment the count a second time (and the corresponding decrement will - // happen in a PlasmaClient::Release call in plasma_seal) so even if the - // buffer - // returned by PlasmaClient::Dreate goes out of scope, the object does not get - // released before the call to PlasmaClient::Seal happens. - increment_object_count(this, object_id, &object, false); - return Status::OK(); -} - -Status PlasmaClient::Get(ObjectID object_ids[], - int64_t num_objects, - int64_t timeout_ms, - ObjectBuffer object_buffers[]) { - // Fill out the info for the objects that are already in use locally. - bool all_present = true; - for (int i = 0; i < num_objects; ++i) { - auto object_entry = objects_in_use.find(object_ids[i]); - if (object_entry == objects_in_use.end()) { - // This object is not currently in use by this client, so we need to send - // a request to the store. - all_present = false; - // Make a note to ourselves that the object is not present. - object_buffers[i].data_size = -1; - } else { - // NOTE: If the object is still unsealed, we will deadlock, since we must - // have been the one who created it. - ARROW_CHECK(object_entry->second->is_sealed) - << "Plasma client called get on an unsealed object that it created"; - PlasmaObject *object = &object_entry->second->object; - object_buffers[i].data = - lookup_mmapped_file(this, object->handle.store_fd); - object_buffers[i].data = object_buffers[i].data + object->data_offset; - object_buffers[i].data_size = object->data_size; - object_buffers[i].metadata = object_buffers[i].data + object->data_size; - object_buffers[i].metadata_size = object->metadata_size; - // Increment the count of the number of instances of this object that this - // client is using. A call to PlasmaClient::Release is required to - // decrement this - // count. Cache the reference to the object. - increment_object_count(this, object_ids[i], object, true); - } - } - - if (all_present) { - return Status::OK(); - } - - // If we get here, then the objects aren't all currently in use by this - // client, so we need to send a request to the plasma store. - RETURN_NOT_OK( - SendGetRequest(store_conn, object_ids, num_objects, timeout_ms)); - std::vector buffer; - RETURN_NOT_OK(PlasmaReceive(store_conn, MessageType_PlasmaGetReply, buffer)); - std::vector received_object_ids(num_objects); - std::vector object_data(num_objects); - PlasmaObject *object; - RETURN_NOT_OK(ReadGetReply(buffer.data(), received_object_ids.data(), - object_data.data(), num_objects)); - - for (int i = 0; i < num_objects; ++i) { - DCHECK(received_object_ids[i] == object_ids[i]); - object = &object_data[i]; - if (object_buffers[i].data_size != -1) { - // If the object was already in use by the client, then the store should - // have returned it. - DCHECK(object->data_size != -1); - // We won't use this file descriptor, but the store sent us one, so we - // need to receive it and then close it right away so we don't leak file - // descriptors. - int fd = recv_fd(store_conn); - close(fd); - ARROW_CHECK(fd >= 0); - // We've already filled out the information for this object, so we can - // just continue. - continue; - } - // If we are here, the object was not currently in use, so we need to - // process the reply from the object store. - if (object->data_size != -1) { - // The object was retrieved. The user will be responsible for releasing - // this object. - int fd = recv_fd(store_conn); - ARROW_CHECK(fd >= 0); - object_buffers[i].data = lookup_or_mmap(this, fd, object->handle.store_fd, - object->handle.mmap_size); - // Finish filling out the return values. - object_buffers[i].data = object_buffers[i].data + object->data_offset; - object_buffers[i].data_size = object->data_size; - object_buffers[i].metadata = object_buffers[i].data + object->data_size; - object_buffers[i].metadata_size = object->metadata_size; - // Increment the count of the number of instances of this object that this - // client is using. A call to PlasmaClient::Release is required to - // decrement this - // count. Cache the reference to the object. - increment_object_count(this, received_object_ids[i], object, true); - } else { - // The object was not retrieved. Make sure we already put a -1 here to - // indicate that the object was not retrieved. The caller is not - // responsible for releasing this object. - DCHECK(object_buffers[i].data_size == -1); - object_buffers[i].data_size = -1; - } - } - return Status::OK(); -} - -/// This is a helper method for implementing plasma_release. We maintain a -/// buffer -/// of release calls and only perform them once the buffer becomes full (as -/// judged by the aggregate sizes of the objects). There may be multiple release -/// calls for the same object ID in the buffer. In this case, the first release -/// calls will not do anything. The client will only send a message to the store -/// releasing the object when the client is truly done with the object. -/// -/// @param conn The plasma connection. -/// @param object_id The object ID to attempt to release. -Status PlasmaClient::PerformRelease(ObjectID object_id) { - // Decrement the count of the number of instances of this object that are - // being used by this client. The corresponding increment should have happened - // in PlasmaClient::Get. - auto object_entry = objects_in_use.find(object_id); - ARROW_CHECK(object_entry != objects_in_use.end()); - object_entry->second->count -= 1; - ARROW_CHECK(object_entry->second->count >= 0); - // Check if the client is no longer using this object. - if (object_entry->second->count == 0) { - // Decrement the count of the number of objects in this memory-mapped file - // that the client is using. The corresponding increment should have - // happened in plasma_get. - int fd = object_entry->second->object.handle.store_fd; - auto entry = mmap_table.find(fd); - ARROW_CHECK(entry != mmap_table.end()); - entry->second->count -= 1; - ARROW_CHECK(entry->second->count >= 0); - // If none are being used then unmap the file. - if (entry->second->count == 0) { - munmap(entry->second->pointer, entry->second->length); - // Remove the corresponding entry from the hash table. - delete entry->second; - mmap_table.erase(fd); - } - // Tell the store that the client no longer needs the object. - RETURN_NOT_OK(SendReleaseRequest(store_conn, object_id)); - // Update the in_use_object_bytes. - in_use_object_bytes -= (object_entry->second->object.data_size + - object_entry->second->object.metadata_size); - DCHECK(in_use_object_bytes >= 0); - // Remove the entry from the hash table of objects currently in use. - delete object_entry->second; - objects_in_use.erase(object_id); - } - return Status::OK(); -} - -Status PlasmaClient::Release(ObjectID object_id) { - // Add the new object to the release history. - release_history.push_front(object_id); - // If there are too many bytes in use by the client or if there are too many - // pending release calls, and there are at least some pending release calls in - // the release_history list, then release some objects. - while ((in_use_object_bytes > - std::min(kL3CacheSizeBytes, store_capacity / 100) || - release_history.size() > config.release_delay) && - release_history.size() > 0) { - // Perform a release for the object ID for the first pending release. - RETURN_NOT_OK(PerformRelease(release_history.back())); - // Remove the last entry from the release history. - release_history.pop_back(); - } - return Status::OK(); -} - -// This method is used to query whether the plasma store contains an object. -Status PlasmaClient::Contains(ObjectID object_id, int *has_object) { - // Check if we already have a reference to the object. - if (objects_in_use.count(object_id) > 0) { - *has_object = 1; - } else { - // If we don't already have a reference to the object, check with the store - // to see if we have the object. - RETURN_NOT_OK(SendContainsRequest(store_conn, object_id)); - std::vector buffer; - RETURN_NOT_OK( - PlasmaReceive(store_conn, MessageType_PlasmaContainsReply, buffer)); - ObjectID object_id2; - RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object)); - } - return Status::OK(); -} - -static void compute_block_hash(const unsigned char *data, - int64_t nbytes, - uint64_t *hash) { - XXH64_state_t hash_state; - XXH64_reset(&hash_state, XXH64_DEFAULT_SEED); - XXH64_update(&hash_state, data, nbytes); - *hash = XXH64_digest(&hash_state); -} - -static inline bool compute_object_hash_parallel(XXH64_state_t *hash_state, - const unsigned char *data, - int64_t nbytes) { - // Note that this function will likely be faster if the address of data is - // aligned on a 64-byte boundary. - const uint64_t num_threads = kThreadPoolSize; - uint64_t threadhash[num_threads + 1]; - const uint64_t data_address = reinterpret_cast(data); - const uint64_t num_blocks = nbytes / BLOCK_SIZE; - const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE; - const uint64_t right_address = data_address + chunk_size * num_threads; - const uint64_t suffix = (data_address + nbytes) - right_address; - // Now the data layout is | k * num_threads * block_size | suffix | == - // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size. - // Each thread gets a "chunk" of k blocks, except the suffix thread. - - for (int i = 0; i < num_threads; i++) { - threadpool_[i] = - std::thread(compute_block_hash, - reinterpret_cast(data_address) + i * chunk_size, - chunk_size, &threadhash[i]); - } - compute_block_hash(reinterpret_cast(right_address), suffix, - &threadhash[num_threads]); - - // Join the threads. - for (auto &t : threadpool_) { - if (t.joinable()) { - t.join(); - } - } - - XXH64_update(hash_state, (unsigned char *) threadhash, sizeof(threadhash)); - return true; -} - -static uint64_t compute_object_hash(const ObjectBuffer &obj_buffer) { - XXH64_state_t hash_state; - XXH64_reset(&hash_state, XXH64_DEFAULT_SEED); - if (obj_buffer.data_size >= kBytesInMB) { - compute_object_hash_parallel(&hash_state, (unsigned char *) obj_buffer.data, - obj_buffer.data_size); - } else { - XXH64_update(&hash_state, (unsigned char *) obj_buffer.data, - obj_buffer.data_size); - } - XXH64_update(&hash_state, (unsigned char *) obj_buffer.metadata, - obj_buffer.metadata_size); - return XXH64_digest(&hash_state); -} - -bool plasma_compute_object_hash(PlasmaClient *conn, - ObjectID obj_id, - unsigned char *digest) { - // Get the plasma object data. We pass in a timeout of 0 to indicate that - // the operation should timeout immediately. - ObjectBuffer obj_buffer; - ObjectID obj_id_array[1] = {obj_id}; - uint64_t hash; - - ARROW_CHECK_OK(conn->Get(obj_id_array, 1, 0, &obj_buffer)); - // If the object was not retrieved, return false. - if (obj_buffer.data_size == -1) { - return false; - } - // Compute the hash. - hash = compute_object_hash(obj_buffer); - memcpy(digest, &hash, sizeof(hash)); - // Release the plasma object. - ARROW_CHECK_OK(conn->Release(obj_id)); - return true; -} - -Status PlasmaClient::Seal(ObjectID object_id) { - // Make sure this client has a reference to the object before sending the - // request to Plasma. - auto object_entry = objects_in_use.find(object_id); - ARROW_CHECK(object_entry != objects_in_use.end()) - << "Plasma client called seal an object without a reference to it"; - ARROW_CHECK(!object_entry->second->is_sealed) - << "Plasma client called seal an already sealed object"; - object_entry->second->is_sealed = true; - /// Send the seal request to Plasma. - static unsigned char digest[kDigestSize]; - ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0])); - RETURN_NOT_OK(SendSealRequest(store_conn, object_id, &digest[0])); - // We call PlasmaClient::Release to decrement the number of instances of this - // object - // that are currently being used by this client. The corresponding increment - // happened in plasma_create and was used to ensure that the object was not - // released before the call to PlasmaClient::Seal. - return Release(object_id); -} - -Status PlasmaClient::Delete(ObjectID object_id) { - // TODO(rkn): In the future, we can use this method to give hints to the - // eviction policy about when an object will no longer be needed. - return Status::NotImplemented("PlasmaClient::Delete is not implemented."); -} - -Status PlasmaClient::Evict(int64_t num_bytes, int64_t &num_bytes_evicted) { - // Send a request to the store to evict objects. - RETURN_NOT_OK(SendEvictRequest(store_conn, num_bytes)); - // Wait for a response with the number of bytes actually evicted. - std::vector buffer; - int64_t type; - RETURN_NOT_OK(ReadMessage(store_conn, &type, buffer)); - return ReadEvictReply(buffer.data(), num_bytes_evicted); -} - -Status PlasmaClient::Subscribe(int &fd) { - int sock[2]; - // Create a non-blocking socket pair. This will only be used to send - // notifications from the Plasma store to the client. - socketpair(AF_UNIX, SOCK_STREAM, 0, sock); - // Make the socket non-blocking. - int flags = fcntl(sock[1], F_GETFL, 0); - ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0); - // Tell the Plasma store about the subscription. - RETURN_NOT_OK(SendSubscribeRequest(store_conn)); - // Send the file descriptor that the Plasma store should use to push - // notifications about sealed objects to this client. - ARROW_CHECK(send_fd(store_conn, sock[1]) >= 0); - close(sock[1]); - // Return the file descriptor that the client should use to read notifications - // about sealed objects. - fd = sock[0]; - return Status::OK(); -} - -Status PlasmaClient::Connect(const std::string &store_socket_name, - const std::string &manager_socket_name, - int release_delay) { - store_conn = connect_ipc_sock_retry(store_socket_name, -1, -1); - if (manager_socket_name != "") { - manager_conn = connect_ipc_sock_retry(manager_socket_name, -1, -1); - } else { - manager_conn = -1; - } - config.release_delay = release_delay; - in_use_object_bytes = 0; - // Send a ConnectRequest to the store to get its memory capacity. - RETURN_NOT_OK(SendConnectRequest(store_conn)); - std::vector buffer; - RETURN_NOT_OK( - PlasmaReceive(store_conn, MessageType_PlasmaConnectReply, buffer)); - RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity)); - return Status::OK(); -} - -Status PlasmaClient::Disconnect() { - // NOTE: We purposefully do not finish sending release calls for objects in - // use, so that we don't duplicate PlasmaClient::Release calls (when handling - // a - // SIGTERM, for example). - for (auto &entry : objects_in_use) { - delete entry.second; - } - for (auto &entry : mmap_table) { - delete entry.second; - } - // Close the connections to Plasma. The Plasma store will release the objects - // that were in use by us when handling the SIGPIPE. - close(store_conn); - if (manager_conn >= 0) { - close(manager_conn); - } - return Status::OK(); -} - -bool plasma_manager_is_connected(PlasmaClient *conn) { - return conn->manager_conn >= 0; -} - -#define h_addr h_addr_list[0] - -Status PlasmaClient::Transfer(const char *address, - int port, - ObjectID object_id) { - return SendDataRequest(manager_conn, object_id, address, port); -} - -Status PlasmaClient::Fetch(int num_object_ids, ObjectID object_ids[]) { - ARROW_CHECK(manager_conn >= 0); - return SendFetchRequest(manager_conn, object_ids, num_object_ids); -} - -int get_manager_fd(PlasmaClient *conn) { - return conn->manager_conn; -} - -Status PlasmaClient::Info(ObjectID object_id, int *object_status) { - ARROW_CHECK(manager_conn >= 0); - - RETURN_NOT_OK(SendStatusRequest(manager_conn, &object_id, 1)); - std::vector buffer; - RETURN_NOT_OK( - PlasmaReceive(manager_conn, MessageType_PlasmaStatusReply, buffer)); - return ReadStatusReply(buffer.data(), &object_id, object_status, 1); -} - -Status PlasmaClient::Wait(int num_object_requests, - ObjectRequest object_requests[], - int num_ready_objects, - uint64_t timeout_ms, - int &num_objects_ready) { - ARROW_CHECK(manager_conn >= 0); - ARROW_CHECK(num_object_requests > 0); - ARROW_CHECK(num_ready_objects > 0); - ARROW_CHECK(num_ready_objects <= num_object_requests); - - for (int i = 0; i < num_object_requests; ++i) { - ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL || - object_requests[i].type == PLASMA_QUERY_ANYWHERE); - } - - RETURN_NOT_OK(SendWaitRequest(manager_conn, object_requests, - num_object_requests, num_ready_objects, - timeout_ms)); - std::vector buffer; - RETURN_NOT_OK( - PlasmaReceive(manager_conn, MessageType_PlasmaWaitReply, buffer)); - RETURN_NOT_OK( - ReadWaitReply(buffer.data(), object_requests, &num_ready_objects)); - - num_objects_ready = 0; - for (int i = 0; i < num_object_requests; ++i) { - int type = object_requests[i].type; - int status = object_requests[i].status; - switch (type) { - case PLASMA_QUERY_LOCAL: - if (status == ObjectStatus_Local) { - num_objects_ready += 1; - } - break; - case PLASMA_QUERY_ANYWHERE: - if (status == ObjectStatus_Local || status == ObjectStatus_Remote) { - num_objects_ready += 1; - } else { - ARROW_CHECK(status == ObjectStatus_Nonexistent); - } - break; - default: - ARROW_LOG(FATAL) << "This code should be unreachable."; - } - } - return Status::OK(); -} diff --git a/src/plasma/plasma_client.h b/src/plasma/plasma_client.h deleted file mode 100644 index f8abbd43e3be..000000000000 --- a/src/plasma/plasma_client.h +++ /dev/null @@ -1,334 +0,0 @@ -#ifndef PLASMA_CLIENT_H -#define PLASMA_CLIENT_H - -#include - -#include - -#include "plasma.h" - -using arrow::Status; - -#define PLASMA_DEFAULT_RELEASE_DELAY 64 - -// Use 100MB as an overestimate of the L3 cache size. -constexpr int64_t kL3CacheSizeBytes = 100000000; - -/// Object buffer data structure. -struct ObjectBuffer { - /// The size in bytes of the data object. - int64_t data_size; - /// The address of the data object. - uint8_t *data; - /// The metadata size in bytes. - int64_t metadata_size; - /// The address of the metadata. - uint8_t *metadata; -}; - -/// Configuration options for the plasma client. -struct PlasmaClientConfig { - /// Number of release calls we wait until the object is actually released. - /// This allows us to avoid invalidating the cpu cache on workers if objects - /// are reused accross tasks. - int release_delay; -}; - -struct ClientMmapTableEntry; -struct ObjectInUseEntry; - -class PlasmaClient { - public: - /// Connect to the local plasma store and plasma manager. Return - /// the resulting connection. - /// - /// @param store_socket_name The name of the UNIX domain socket to use to - /// connect to the Plasma store. - /// @param manager_socket_name The name of the UNIX domain socket to use to - /// connect to the local Plasma manager. If this is NULL, then this - /// function will not connect to a manager. - /// @param release_delay Number of released objects that are kept around - /// and not evicted to avoid too many munmaps. - /// @return The return status. - Status Connect(const std::string &store_socket_name, - const std::string &manager_socket_name, - int release_delay); - - /// Create an object in the Plasma Store. Any metadata for this object must be - /// be passed in when the object is created. - /// - /// @param object_id The ID to use for the newly created object. - /// @param data_size The size in bytes of the space to be allocated for this - /// object's - /// data (this does not include space used for metadata). - /// @param metadata The object's metadata. If there is no metadata, this - /// pointer - /// should be NULL. - /// @param metadata_size The size in bytes of the metadata. If there is no - /// metadata, this should be 0. - /// @param data The address of the newly created object will be written here. - /// @return The return status. - Status Create(ObjectID object_id, - int64_t data_size, - uint8_t *metadata, - int64_t metadata_size, - uint8_t **data); - - /// Get some objects from the Plasma Store. This function will block until the - /// objects have all been created and sealed in the Plasma Store or the - /// timeout - /// expires. The caller is responsible for releasing any retrieved objects, - /// but - /// the caller should not release objects that were not retrieved. - /// - /// @param object_ids The IDs of the objects to get. - /// @param num_object_ids The number of object IDs to get. - /// @param timeout_ms The amount of time in milliseconds to wait before this - /// request times out. If this value is -1, then no timeout is set. - /// @param object_buffers An array where the results will be stored. If the - /// data - /// size field is -1, then the object was not retrieved. - /// @return The return status. - Status Get(ObjectID object_ids[], - int64_t num_objects, - int64_t timeout_ms, - ObjectBuffer object_buffers[]); - - /// Tell Plasma that the client no longer needs the object. This should be - /// called - /// after Get when the client is done with the object. After this call, - /// the address returned by Get is no longer valid. This should be called - /// once for each call to Get (with the same object ID). - /// - /// @param object_id The ID of the object that is no longer needed. - /// @return The return status. - Status Release(ObjectID object_id); - - /// Check if the object store contains a particular object and the object has - /// been sealed. The result will be stored in has_object. - /// - /// @todo: We may want to indicate if the object has been created but not - /// sealed. - /// - /// @param object_id The ID of the object whose presence we are checking. - /// @param has_object The function will write 1 at this address if the object - /// is - /// present and 0 if it is not present. - /// @return The return status. - Status Contains(ObjectID object_id, int *has_object); - - /// Seal an object in the object store. The object will be immutable after - /// this - /// call. - /// - /// @param object_id The ID of the object to seal. - /// @return The return status. - Status Seal(ObjectID object_id); - - /// Delete an object from the object store. This currently assumes that the - /// object is present and has been sealed. - /// - /// @todo We may want to allow the deletion of objects that are not present or - /// haven't been sealed. - /// - /// @param object_id The ID of the object to delete. - /// @return The return status. - Status Delete(ObjectID object_id); - - /// Delete objects until we have freed up num_bytes bytes or there are no more - /// released objects that can be deleted. - /// - /// @param num_bytes The number of bytes to try to free up. - /// @param num_bytes_evicted Out parameter for total number of bytes of space - /// retrieved. - /// @return The return status. - Status Evict(int64_t num_bytes, int64_t &num_bytes_evicted); - - /// Subscribe to notifications when objects are sealed in the object store. - /// Whenever an object is sealed, a message will be written to the client - /// socket - /// that is returned by this method. - /// - /// @param fd Out parameter for the file descriptor the client should use to - /// read notifications - /// from the object store about sealed objects. - /// @return The return status. - Status Subscribe(int &fd); - - /// Disconnect from the local plasma instance, including the local store and - /// manager. - /// - /// @return The return status. - Status Disconnect(); - - /// Attempt to initiate the transfer of some objects from remote Plasma - /// Stores. - /// This method does not guarantee that the fetched objects will arrive - /// locally. - /// - /// For an object that is available in the local Plasma Store, this method - /// will - /// not do anything. For an object that is not available locally, it will - /// check - /// if the object are already being fetched. If so, it will not do anything. - /// If - /// not, it will query the object table for a list of Plasma Managers that - /// have - /// the object. The object table will return a non-empty list, and this Plasma - /// Manager will attempt to initiate transfers from one of those Plasma - /// Managers. - /// - /// This function is non-blocking. - /// - /// This method is idempotent in the sense that it is ok to call it multiple - /// times. - /// - /// @param num_object_ids The number of object IDs fetch is being called on. - /// @param object_ids The IDs of the objects that fetch is being called on. - /// @return The return status. - Status Fetch(int num_object_ids, ObjectID object_ids[]); - - /// Wait for (1) a specified number of objects to be available (sealed) in the - /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to - /// expire. This is a blocking call. - /// - /// @param num_object_requests Size of the object_requests array. - /// @param object_requests Object event array. Each element contains a request - /// for a particular object_id. The type of request is specified in the - /// "type" field. - /// - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes - /// available in the local Plasma Store. In this case, this function - /// sets the "status" field to ObjectStatus_Local. Note, if the - /// status - /// is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent, - /// but it may exist elsewhere in the system. - /// - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id - /// becomes - /// available either at the local Plasma Store or on a remote Plasma - /// Store. In this case, the functions sets the "status" field to - /// ObjectStatus_Local or ObjectStatus_Remote. - /// @param num_ready_objects The number of requests in object_requests array - /// that - /// must be satisfied before the function returns, unless it timeouts. - /// The num_ready_objects should be no larger than num_object_requests. - /// @param timeout_ms Timeout value in milliseconds. If this timeout expires - /// before min_num_ready_objects of requests are satisfied, the - /// function - /// returns. - /// @param num_objects_ready Out parameter for number of satisfied requests in - /// the object_requests list. If the returned number is less than - /// min_num_ready_objects this means that timeout expired. - /// @return The return status. - Status Wait(int num_object_requests, - ObjectRequest object_requests[], - int num_ready_objects, - uint64_t timeout_ms, - int &num_objects_ready); - - /// Transfer local object to a different plasma manager. - /// - /// @param conn The object containing the connection state. - /// @param addr IP address of the plasma manager we are transfering to. - /// @param port Port of the plasma manager we are transfering to. - /// @object_id ObjectID of the object we are transfering. - /// @return The return status. - Status Transfer(const char *addr, int port, ObjectID object_id); - - /// Return the status of a given object. This method may query the object - /// table. - /// - /// @param conn The object containing the connection state. - /// @param object_id The ID of the object whose status we query. - /// @param object_status Out parameter for object status. Can take the - /// following values. - /// - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma - /// Store. - /// has been already scheduled by the Plasma Manager. - /// - PLASMA_CLIENT_TRANSFER, if the object is either currently being - /// transferred or just scheduled. - /// - PLASMA_CLIENT_REMOTE, if the object is stored at a remote - /// Plasma Store. - /// - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the - /// system. - /// @return The return status. - Status Info(ObjectID object_id, int *object_status); - - // private: - - Status PerformRelease(ObjectID object_id); - - /// File descriptor of the Unix domain socket that connects to the store. - int store_conn; - /// File descriptor of the Unix domain socket that connects to the manager. - int manager_conn; - /// File descriptor of the Unix domain socket on which client receives event - /// notifications for the objects it subscribes for when these objects are - /// sealed either locally or remotely. - int manager_conn_subscribe; - /// Table of dlmalloc buffer files that have been memory mapped so far. This - /// is a hash table mapping a file descriptor to a struct containing the - /// address of the corresponding memory-mapped file. - std::unordered_map mmap_table; - /// A hash table of the object IDs that are currently being used by this - /// client. - std::unordered_map - objects_in_use; - /// Object IDs of the last few release calls. This is a deque and - /// is used to delay releasing objects to see if they can be reused by - /// subsequent tasks so we do not unneccessarily invalidate cpu caches. - /// TODO(pcm): replace this with a proper lru cache using the size of the L3 - /// cache. - std::deque release_history; - /// The number of bytes in the combined objects that are held in the release - /// history doubly-linked list. If this is too large then the client starts - /// releasing objects. - int64_t in_use_object_bytes; - /// Configuration options for the plasma client. - PlasmaClientConfig config; - /// The amount of memory available to the Plasma store. The client needs this - /// information to make sure that it does not delay in releasing so much - /// memory that the store is unable to evict enough objects to free up space. - int64_t store_capacity; -}; - -/// Return true if the plasma manager is connected. -/// -/// @param conn The connection to the local plasma store and plasma manager. -/// @return True if the plasma manager is connected and false otherwise. -bool plasma_manager_is_connected(PlasmaClient *conn); - -/// Compute the hash of an object in the object store. -/// -/// @param conn The object containing the connection state. -/// @param object_id The ID of the object we want to hash. -/// @param digest A pointer at which to return the hash digest of the object. -/// The pointer must have at least DIGEST_SIZE bytes allocated. -/// @return A boolean representing whether the hash operation succeeded. -bool plasma_compute_object_hash(PlasmaClient *conn, - ObjectID object_id, - unsigned char *digest); - -/** - * Get the file descriptor for the socket connection to the plasma manager. - * - * @param conn The plasma connection. - * @return The file descriptor for the manager connection. If there is no - * connection to the manager, this is -1. - */ -int get_manager_fd(PlasmaClient *conn); - -/** - * Return the information associated to a given object. - * - * @param conn The object containing the connection state. - * @param object_id The ID of the object whose info the client queries. - * @param object_info The object's infirmation. - * @return PLASMA_CLIENT_LOCAL, if the object is in the local Plasma Store. - * PLASMA_CLIENT_NOT_LOCAL, if not. In this case, the caller needs to - * ignore data, metadata_size, and metadata fields. - */ -// int plasma_info(PlasmaConnection *conn, -// ObjectID object_id, -// ObjectInfo *object_info); - -#endif /* PLASMA_CLIENT_H */ diff --git a/src/plasma/plasma_common.cc b/src/plasma/plasma_common.cc deleted file mode 100644 index d09be2d3516d..000000000000 --- a/src/plasma/plasma_common.cc +++ /dev/null @@ -1,67 +0,0 @@ -#include "plasma_common.h" - -#include - -#include "format/plasma_generated.h" - -using arrow::Status; - -UniqueID UniqueID::from_random() { - UniqueID id; - uint8_t *data = id.mutable_data(); - std::random_device engine; - for (int i = 0; i < kUniqueIDSize; i++) { - data[i] = engine(); - } - return id; -} - -UniqueID UniqueID::from_binary(const std::string &binary) { - UniqueID id; - std::memcpy(&id, binary.data(), sizeof(id)); - return id; -} - -const uint8_t *UniqueID::data() const { - return id_; -} - -uint8_t *UniqueID::mutable_data() { - return id_; -} - -std::string UniqueID::binary() const { - return std::string(reinterpret_cast(id_), kUniqueIDSize); -} - -std::string UniqueID::hex() const { - constexpr char hex[] = "0123456789abcdef"; - std::string result; - for (int i = 0; i < sizeof(UniqueID); i++) { - unsigned int val = id_[i]; - result.push_back(hex[val >> 4]); - result.push_back(hex[val & 0xf]); - } - return result; -} - -bool UniqueID::operator==(const UniqueID &rhs) const { - return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0; -} - -Status plasma_error_status(int plasma_error) { - switch (plasma_error) { - case PlasmaError_OK: - return Status::OK(); - case PlasmaError_ObjectExists: - return Status::PlasmaObjectExists( - "object already exists in the plasma store"); - case PlasmaError_ObjectNonexistent: - return Status::PlasmaObjectNonexistent( - "object does not exist in the plasma store"); - case PlasmaError_OutOfMemory: - return Status::PlasmaStoreFull("object does not fit in the plasma store"); - default: - ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error; - } -} diff --git a/src/plasma/plasma_common.h b/src/plasma/plasma_common.h deleted file mode 100644 index a46940d3c679..000000000000 --- a/src/plasma/plasma_common.h +++ /dev/null @@ -1,46 +0,0 @@ -#ifndef PLASMA_COMMON_H -#define PLASMA_COMMON_H - -#include -#include -// TODO(pcm): Convert getopt and sscanf in the store to use more idiomatic C++ -// and get rid of the next three lines: -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS -#endif - -#include "logging.h" -#include "status.h" - -constexpr int64_t kUniqueIDSize = 20; - -class UniqueID { - public: - static UniqueID from_random(); - static UniqueID from_binary(const std::string &binary); - bool operator==(const UniqueID &rhs) const; - const uint8_t *data() const; - uint8_t *mutable_data(); - std::string binary() const; - std::string hex() const; - - private: - uint8_t id_[kUniqueIDSize]; -}; - -static_assert(std::is_pod::value, "UniqueID must be plain old data"); - -struct UniqueIDHasher { - /* ObjectID hashing function. */ - size_t operator()(const UniqueID &id) const { - size_t result; - std::memcpy(&result, id.data(), sizeof(size_t)); - return result; - } -}; - -typedef UniqueID ObjectID; - -arrow::Status plasma_error_status(int plasma_error); - -#endif // PLASMA_COMMON_H diff --git a/src/plasma/plasma_events.cc b/src/plasma/plasma_events.cc deleted file mode 100644 index 883b745d5308..000000000000 --- a/src/plasma/plasma_events.cc +++ /dev/null @@ -1,74 +0,0 @@ -#include "plasma_events.h" - -#include - -void EventLoop::file_event_callback(aeEventLoop *loop, - int fd, - void *context, - int events) { - FileCallback *callback = reinterpret_cast(context); - (*callback)(events); -} - -int EventLoop::timer_event_callback(aeEventLoop *loop, - long long timer_id, - void *context) { - TimerCallback *callback = reinterpret_cast(context); - return (*callback)(timer_id); -} - -constexpr int kInitialEventLoopSize = 1024; - -EventLoop::EventLoop() { - loop_ = aeCreateEventLoop(kInitialEventLoopSize); -} - -bool EventLoop::add_file_event(int fd, int events, FileCallback callback) { - if (file_callbacks_.find(fd) != file_callbacks_.end()) { - return false; - } - auto data = std::unique_ptr(new FileCallback(callback)); - void *context = reinterpret_cast(data.get()); - // Try to add the file descriptor. - int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, - context); - // If it cannot be added, increase the size of the event loop. - if (err == AE_ERR && errno == ERANGE) { - err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2); - if (err != AE_OK) { - return false; - } - err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback, - context); - } - // In any case, test if there were errors. - if (err == AE_OK) { - file_callbacks_.emplace(fd, std::move(data)); - return true; - } - return false; -} - -void EventLoop::remove_file_event(int fd) { - aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE); - file_callbacks_.erase(fd); -} - -void EventLoop::run() { - aeMain(loop_); -} - -int64_t EventLoop::add_timer(int64_t timeout, TimerCallback callback) { - auto data = std::unique_ptr(new TimerCallback(callback)); - void *context = reinterpret_cast(data.get()); - int64_t timer_id = aeCreateTimeEvent( - loop_, timeout, EventLoop::timer_event_callback, context, NULL); - timer_callbacks_.emplace(timer_id, std::move(data)); - return timer_id; -} - -int EventLoop::remove_timer(int64_t timer_id) { - int err = aeDeleteTimeEvent(loop_, timer_id); - timer_callbacks_.erase(timer_id); - return err; -} diff --git a/src/plasma/plasma_events.h b/src/plasma/plasma_events.h deleted file mode 100644 index c94025f96d88..000000000000 --- a/src/plasma/plasma_events.h +++ /dev/null @@ -1,85 +0,0 @@ -#ifndef PLASMA_EVENTS -#define PLASMA_EVENTS - -#include -#include -#include - -extern "C" { -#include "ae/ae.h" -} - -/// Constant specifying that the timer is done and it will be removed. -constexpr int kEventLoopTimerDone = AE_NOMORE; - -/// Read event on the file descriptor. -constexpr int kEventLoopRead = AE_READABLE; - -/// Write event on the file descriptor. -constexpr int kEventLoopWrite = AE_WRITABLE; - -class EventLoop { - public: - // Signature of the handler that will be called when there is a new event - // on the file descriptor that this handler has been registered for. - // - // The arguments are the event flags (read or write). - typedef std::function FileCallback; - - // This handler will be called when a timer times out. The timer id is - // passed as an argument. The return is the number of milliseconds the timer - // shall be reset to or kEventLoopTimerDone if the timer shall not be - // triggered again. - typedef std::function TimerCallback; - - EventLoop(); - - /// Add a new file event handler to the event loop. - /// - /// @param fd The file descriptor we are listening to. - /// @param events The flags for events we are listening to (read or write). - /// @param callback The callback that will be called when the event happens. - /// @return Returns true if the event handler was added successfully. - bool add_file_event(int fd, int events, FileCallback callback); - - /// Remove a file event handler from the event loop. - /// - /// @param fd The file descriptor of the event handler. - /// @return Void. - void remove_file_event(int fd); - - /// Register a handler that will be called after a time slice of - /// "timeout" milliseconds. - /// - /// @param timeout The timeout in milliseconds. - /// @param callback The callback for the timeout. - /// @return The ID of the newly created timer. - int64_t add_timer(int64_t timeout, TimerCallback callback); - - /// Remove a timer handler from the event loop. - /// - /// @param timer_id The ID of the timer that is to be removed. - /// @return The ae.c error code. TODO(pcm): needs to be standardized - int remove_timer(int64_t timer_id); - - /// Run the event loop. - /// - /// @return Void. - void run(); - - private: - static void file_event_callback(aeEventLoop *loop, - int fd, - void *context, - int events); - - static int timer_event_callback(aeEventLoop *loop, - long long timer_id, - void *context); - - aeEventLoop *loop_; - std::unordered_map> file_callbacks_; - std::unordered_map> timer_callbacks_; -}; - -#endif // PLASMA_EVENTS diff --git a/src/plasma/plasma_extension.cc b/src/plasma/plasma_extension.cc deleted file mode 100644 index 58d45ec3cb6e..000000000000 --- a/src/plasma/plasma_extension.cc +++ /dev/null @@ -1,463 +0,0 @@ -#include -#include "bytesobject.h" - -#include "plasma_io.h" -#include "plasma_common.h" -#include "plasma_protocol.h" -#include "plasma_client.h" - -PyObject *PlasmaOutOfMemoryError; -PyObject *PlasmaObjectExistsError; - -#include "plasma_extension.h" - -PyObject *PyPlasma_connect(PyObject *self, PyObject *args) { - const char *store_socket_name; - const char *manager_socket_name; - int release_delay; - if (!PyArg_ParseTuple(args, "ssi", &store_socket_name, &manager_socket_name, - &release_delay)) { - return NULL; - } - PlasmaClient *client = new PlasmaClient(); - ARROW_CHECK_OK( - client->Connect(store_socket_name, manager_socket_name, release_delay)); - - return PyCapsule_New(client, "plasma", NULL); -} - -PyObject *PyPlasma_disconnect(PyObject *self, PyObject *args) { - PyObject *client_capsule; - if (!PyArg_ParseTuple(args, "O", &client_capsule)) { - return NULL; - } - PlasmaClient *client; - ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client)); - ARROW_CHECK_OK(client->Disconnect()); - /* We use the context of the connection capsule to indicate if the connection - * is still active (if the context is NULL) or if it is closed (if the context - * is (void*) 0x1). This is neccessary because the primary pointer of the - * capsule cannot be NULL. */ - PyCapsule_SetContext(client_capsule, (void *) 0x1); - Py_RETURN_NONE; -} - -PyObject *PyPlasma_create(PyObject *self, PyObject *args) { - PlasmaClient *client; - ObjectID object_id; - long long size; - PyObject *metadata; - if (!PyArg_ParseTuple(args, "O&O&LO", PyObjectToPlasmaClient, &client, - PyStringToUniqueID, &object_id, &size, &metadata)) { - return NULL; - } - if (!PyByteArray_Check(metadata)) { - PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray"); - return NULL; - } - uint8_t *data; - Status s = client->Create(object_id, size, - (uint8_t *) PyByteArray_AsString(metadata), - PyByteArray_Size(metadata), &data); - if (s.IsPlasmaObjectExists()) { - PyErr_SetString(PlasmaObjectExistsError, - "An object with this ID already exists in the plasma " - "store."); - return NULL; - } - if (s.IsPlasmaStoreFull()) { - PyErr_SetString(PlasmaOutOfMemoryError, - "The plasma store ran out of memory and could not create " - "this object."); - return NULL; - } - ARROW_CHECK(s.ok()); - -#if PY_MAJOR_VERSION >= 3 - return PyMemoryView_FromMemory((char *) data, (Py_ssize_t) size, PyBUF_WRITE); -#else - return PyBuffer_FromReadWriteMemory((void *) data, (Py_ssize_t) size); -#endif -} - -PyObject *PyPlasma_hash(PyObject *self, PyObject *args) { - PlasmaClient *client; - ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, - PyStringToUniqueID, &object_id)) { - return NULL; - } - unsigned char digest[kDigestSize]; - bool success = plasma_compute_object_hash(client, object_id, digest); - if (success) { - PyObject *digest_string = - PyBytes_FromStringAndSize((char *) digest, kDigestSize); - return digest_string; - } else { - Py_RETURN_NONE; - } -} - -PyObject *PyPlasma_seal(PyObject *self, PyObject *args) { - PlasmaClient *client; - ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, - PyStringToUniqueID, &object_id)) { - return NULL; - } - ARROW_CHECK_OK(client->Seal(object_id)); - Py_RETURN_NONE; -} - -PyObject *PyPlasma_release(PyObject *self, PyObject *args) { - PlasmaClient *client; - ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, - PyStringToUniqueID, &object_id)) { - return NULL; - } - ARROW_CHECK_OK(client->Release(object_id)); - Py_RETURN_NONE; -} - -PyObject *PyPlasma_get(PyObject *self, PyObject *args) { - PlasmaClient *client; - PyObject *object_id_list; - long long timeout_ms; - if (!PyArg_ParseTuple(args, "O&OL", PyObjectToPlasmaClient, &client, - &object_id_list, &timeout_ms)) { - return NULL; - } - - Py_ssize_t num_object_ids = PyList_Size(object_id_list); - ObjectID *object_ids = (ObjectID *) malloc(sizeof(ObjectID) * num_object_ids); - ObjectBuffer *object_buffers = - (ObjectBuffer *) malloc(sizeof(ObjectBuffer) * num_object_ids); - - for (int i = 0; i < num_object_ids; ++i) { - PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]); - } - - Py_BEGIN_ALLOW_THREADS; - ARROW_CHECK_OK( - client->Get(object_ids, num_object_ids, timeout_ms, object_buffers)); - Py_END_ALLOW_THREADS; - free(object_ids); - - PyObject *returns = PyList_New(num_object_ids); - for (int i = 0; i < num_object_ids; ++i) { - if (object_buffers[i].data_size != -1) { - /* The object was retrieved, so return the object. */ - PyObject *t = PyTuple_New(2); -#if PY_MAJOR_VERSION >= 3 - PyTuple_SetItem( - t, 0, PyMemoryView_FromMemory( - (char *) object_buffers[i].data, - (Py_ssize_t) object_buffers[i].data_size, PyBUF_READ)); - PyTuple_SetItem( - t, 1, PyMemoryView_FromMemory( - (char *) object_buffers[i].metadata, - (Py_ssize_t) object_buffers[i].metadata_size, PyBUF_READ)); -#else - PyTuple_SetItem( - t, 0, PyBuffer_FromMemory((void *) object_buffers[i].data, - (Py_ssize_t) object_buffers[i].data_size)); - PyTuple_SetItem(t, 1, PyBuffer_FromMemory( - (void *) object_buffers[i].metadata, - (Py_ssize_t) object_buffers[i].metadata_size)); -#endif - PyList_SetItem(returns, i, t); - } else { - /* The object was not retrieved, so just add None to the list of return - * values. */ - Py_XINCREF(Py_None); - PyList_SetItem(returns, i, Py_None); - } - } - free(object_buffers); - return returns; -} - -PyObject *PyPlasma_contains(PyObject *self, PyObject *args) { - PlasmaClient *client; - ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, - PyStringToUniqueID, &object_id)) { - return NULL; - } - int has_object; - ARROW_CHECK_OK(client->Contains(object_id, &has_object)); - - if (has_object) - Py_RETURN_TRUE; - else - Py_RETURN_FALSE; -} - -PyObject *PyPlasma_fetch(PyObject *self, PyObject *args) { - PlasmaClient *client; - PyObject *object_id_list; - if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client, - &object_id_list)) { - return NULL; - } - if (!plasma_manager_is_connected(client)) { - PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager"); - return NULL; - } - Py_ssize_t n = PyList_Size(object_id_list); - ObjectID *object_ids = (ObjectID *) malloc(sizeof(ObjectID) * n); - for (int i = 0; i < n; ++i) { - PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]); - } - ARROW_CHECK_OK(client->Fetch((int) n, object_ids)); - free(object_ids); - Py_RETURN_NONE; -} - -PyObject *PyPlasma_wait(PyObject *self, PyObject *args) { - PlasmaClient *client; - PyObject *object_id_list; - long long timeout; - int num_returns; - if (!PyArg_ParseTuple(args, "O&OLi", PyObjectToPlasmaClient, &client, - &object_id_list, &timeout, &num_returns)) { - return NULL; - } - Py_ssize_t n = PyList_Size(object_id_list); - - if (!plasma_manager_is_connected(client)) { - PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager"); - return NULL; - } - if (num_returns < 0) { - PyErr_SetString(PyExc_RuntimeError, - "The argument num_returns cannot be less than zero."); - return NULL; - } - if (num_returns > n) { - PyErr_SetString( - PyExc_RuntimeError, - "The argument num_returns cannot be greater than len(object_ids)"); - return NULL; - } - int64_t threshold = 1 << 30; - if (timeout > threshold) { - PyErr_SetString(PyExc_RuntimeError, - "The argument timeout cannot be greater than 2 ** 30."); - return NULL; - } - - ObjectRequest *object_requests = - (ObjectRequest *) malloc(sizeof(ObjectRequest) * n); - for (int i = 0; i < n; ++i) { - ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i), - &object_requests[i].object_id) == 1); - object_requests[i].type = PLASMA_QUERY_ANYWHERE; - } - /* Drop the global interpreter lock while we are waiting, so other threads can - * run. */ - int num_return_objects; - Py_BEGIN_ALLOW_THREADS; - ARROW_CHECK_OK(client->Wait((int) n, object_requests, num_returns, - (uint64_t) timeout, num_return_objects)); - Py_END_ALLOW_THREADS; - - int num_to_return = std::min(num_return_objects, num_returns); - PyObject *ready_ids = PyList_New(num_to_return); - PyObject *waiting_ids = PySet_New(object_id_list); - int num_returned = 0; - for (int i = 0; i < n; ++i) { - if (num_returned == num_to_return) { - break; - } - if (object_requests[i].status == ObjectStatus_Local || - object_requests[i].status == ObjectStatus_Remote) { - PyObject *ready = - PyBytes_FromStringAndSize((char *) &object_requests[i].object_id, - sizeof(object_requests[i].object_id)); - PyList_SetItem(ready_ids, num_returned, ready); - PySet_Discard(waiting_ids, ready); - num_returned += 1; - } else { - ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent); - } - } - ARROW_CHECK(num_returned == num_to_return); - /* Return both the ready IDs and the remaining IDs. */ - PyObject *t = PyTuple_New(2); - PyTuple_SetItem(t, 0, ready_ids); - PyTuple_SetItem(t, 1, waiting_ids); - return t; -} - -PyObject *PyPlasma_evict(PyObject *self, PyObject *args) { - PlasmaClient *client; - long long num_bytes; - if (!PyArg_ParseTuple(args, "O&L", PyObjectToPlasmaClient, &client, - &num_bytes)) { - return NULL; - } - int64_t evicted_bytes; - ARROW_CHECK_OK(client->Evict((int64_t) num_bytes, evicted_bytes)); - return PyLong_FromLong((long) evicted_bytes); -} - -PyObject *PyPlasma_delete(PyObject *self, PyObject *args) { - PlasmaClient *client; - ObjectID object_id; - if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client, - PyStringToUniqueID, &object_id)) { - return NULL; - } - ARROW_CHECK_OK(client->Delete(object_id)); - Py_RETURN_NONE; -} - -PyObject *PyPlasma_transfer(PyObject *self, PyObject *args) { - PlasmaClient *client; - ObjectID object_id; - const char *addr; - int port; - if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client, - PyStringToUniqueID, &object_id, &addr, &port)) { - return NULL; - } - - if (!plasma_manager_is_connected(client)) { - PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager"); - return NULL; - } - - ARROW_CHECK_OK(client->Transfer(addr, port, object_id)); - Py_RETURN_NONE; -} - -PyObject *PyPlasma_subscribe(PyObject *self, PyObject *args) { - PlasmaClient *client; - if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) { - return NULL; - } - - int sock; - ARROW_CHECK_OK(client->Subscribe(sock)); - return PyLong_FromLong(sock); -} - -PyObject *PyPlasma_receive_notification(PyObject *self, PyObject *args) { - int plasma_sock; - - if (!PyArg_ParseTuple(args, "i", &plasma_sock)) { - return NULL; - } - /* Receive object notification from the plasma connection socket. If the - * object was added, return a tuple of its fields: ObjectID, data_size, - * metadata_size. If the object was deleted, data_size and metadata_size will - * be set to -1. */ - uint8_t *notification = read_message_async(plasma_sock); - if (notification == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "Failed to read object notification from Plasma socket"); - return NULL; - } - auto object_info = flatbuffers::GetRoot(notification); - /* Construct a tuple from object_info and return. */ - PyObject *t = PyTuple_New(3); - PyTuple_SetItem(t, 0, - PyBytes_FromStringAndSize(object_info->object_id()->data(), - object_info->object_id()->size())); - if (object_info->is_deletion()) { - PyTuple_SetItem(t, 1, PyLong_FromLong(-1)); - PyTuple_SetItem(t, 2, PyLong_FromLong(-1)); - } else { - PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size())); - PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size())); - } - - free(notification); - return t; -} - -static PyMethodDef plasma_methods[] = { - {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."}, - {"disconnect", PyPlasma_disconnect, METH_VARARGS, - "Disconnect from plasma."}, - {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."}, - {"hash", PyPlasma_hash, METH_VARARGS, - "Compute the hash of a plasma object."}, - {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."}, - {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."}, - {"contains", PyPlasma_contains, METH_VARARGS, - "Does the plasma store contain this plasma object?"}, - {"fetch", PyPlasma_fetch, METH_VARARGS, - "Fetch the object from another plasma manager instance."}, - {"wait", PyPlasma_wait, METH_VARARGS, - "Wait until num_returns objects in object_ids are ready."}, - {"evict", PyPlasma_evict, METH_VARARGS, - "Evict some objects until we recover some number of bytes."}, - {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."}, - {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."}, - {"transfer", PyPlasma_transfer, METH_VARARGS, - "Transfer object to another plasma manager."}, - {"subscribe", PyPlasma_subscribe, METH_VARARGS, - "Subscribe to the plasma notification socket."}, - {"receive_notification", PyPlasma_receive_notification, METH_VARARGS, - "Receive next notification from plasma notification socket."}, - {NULL} /* Sentinel */ -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "libplasma", /* m_name */ - "A Python client library for plasma.", /* m_doc */ - 0, /* m_size */ - plasma_methods, /* m_methods */ - NULL, /* m_reload */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL, /* m_free */ -}; -#endif - -#if PY_MAJOR_VERSION >= 3 -#define INITERROR return NULL -#else -#define INITERROR return -#endif - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif - -#if PY_MAJOR_VERSION >= 3 -#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) -#else -#define MOD_INIT(name) PyMODINIT_FUNC init##name(void) -#endif - -MOD_INIT(libplasma) { -#if PY_MAJOR_VERSION >= 3 - PyObject *m = PyModule_Create(&moduledef); -#else - PyObject *m = Py_InitModule3("libplasma", plasma_methods, - "A Python client library for plasma."); -#endif - - /* Create a custom exception for when an object ID is reused. */ - char plasma_object_exists_error[] = "plasma_object_exists.error"; - PlasmaObjectExistsError = - PyErr_NewException(plasma_object_exists_error, NULL, NULL); - Py_INCREF(PlasmaObjectExistsError); - PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError); - /* Create a custom exception for when the plasma store is out of memory. */ - char plasma_out_of_memory_error[] = "plasma_out_of_memory.error"; - PlasmaOutOfMemoryError = - PyErr_NewException(plasma_out_of_memory_error, NULL, NULL); - Py_INCREF(PlasmaOutOfMemoryError); - PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError); - -#if PY_MAJOR_VERSION >= 3 - return m; -#endif -} diff --git a/src/plasma/plasma_extension.h b/src/plasma/plasma_extension.h deleted file mode 100644 index 6c7bf5954560..000000000000 --- a/src/plasma/plasma_extension.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef PLASMA_EXTENSION_H -#define PLASMA_EXTENSION_H - -static int PyObjectToPlasmaClient(PyObject *object, PlasmaClient **client) { - if (PyCapsule_IsValid(object, "plasma")) { - *client = (PlasmaClient *) PyCapsule_GetPointer(object, "plasma"); - return 1; - } else { - PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule"); - return 0; - } -} - -int PyStringToUniqueID(PyObject *object, ObjectID *object_id) { - if (PyBytes_Check(object)) { - memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID)); - return 1; - } else { - PyErr_SetString(PyExc_TypeError, "must be a 20 character string"); - return 0; - } -} - -#endif /* PLASMA_EXTENSION_H */ diff --git a/src/plasma/plasma_io.cc b/src/plasma/plasma_io.cc deleted file mode 100644 index 323e703be056..000000000000 --- a/src/plasma/plasma_io.cc +++ /dev/null @@ -1,220 +0,0 @@ -#include "plasma_io.h" -#include "plasma_common.h" - -using arrow::Status; - -/* Number of times we try binding to a socket. */ -#define NUM_BIND_ATTEMPTS 5 -#define BIND_TIMEOUT_MS 100 - -/* Number of times we try connecting to a socket. */ -#define NUM_CONNECT_ATTEMPTS 50 -#define CONNECT_TIMEOUT_MS 100 - -Status WriteBytes(int fd, uint8_t *cursor, size_t length) { - ssize_t nbytes = 0; - size_t bytesleft = length; - size_t offset = 0; - while (bytesleft > 0) { - /* While we haven't written the whole message, write to the file descriptor, - * advance the cursor, and decrease the amount left to write. */ - nbytes = write(fd, cursor + offset, bytesleft); - if (nbytes < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { - continue; - } - return Status::IOError(std::string(strerror(errno))); - } else if (nbytes == 0) { - return Status::IOError("Encountered unexpected EOF"); - } - ARROW_CHECK(nbytes > 0); - bytesleft -= nbytes; - offset += nbytes; - } - - return Status::OK(); -} - -Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t *bytes) { - int64_t version = PLASMA_PROTOCOL_VERSION; - RETURN_NOT_OK( - WriteBytes(fd, reinterpret_cast(&version), sizeof(version))); - RETURN_NOT_OK( - WriteBytes(fd, reinterpret_cast(&type), sizeof(type))); - RETURN_NOT_OK( - WriteBytes(fd, reinterpret_cast(&length), sizeof(length))); - return WriteBytes(fd, bytes, length * sizeof(char)); -} - -Status ReadBytes(int fd, uint8_t *cursor, size_t length) { - ssize_t nbytes = 0; - /* Termination condition: EOF or read 'length' bytes total. */ - size_t bytesleft = length; - size_t offset = 0; - while (bytesleft > 0) { - nbytes = read(fd, cursor + offset, bytesleft); - if (nbytes < 0) { - if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { - continue; - } - return Status::IOError(std::string(strerror(errno))); - } else if (0 == nbytes) { - return Status::IOError("Encountered unexpected EOF"); - } - ARROW_CHECK(nbytes > 0); - bytesleft -= nbytes; - offset += nbytes; - } - - return Status::OK(); -} - -Status ReadMessage(int fd, int64_t *type, std::vector &buffer) { - int64_t version; - RETURN_NOT_OK_ELSE( - ReadBytes(fd, reinterpret_cast(&version), sizeof(version)), - *type = DISCONNECT_CLIENT); - ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version; - int64_t length; - RETURN_NOT_OK_ELSE( - ReadBytes(fd, reinterpret_cast(type), sizeof(*type)), - *type = DISCONNECT_CLIENT); - RETURN_NOT_OK_ELSE( - ReadBytes(fd, reinterpret_cast(&length), sizeof(length)), - *type = DISCONNECT_CLIENT); - if (length > buffer.size()) { - buffer.resize(length); - } - RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer.data(), length), - *type = DISCONNECT_CLIENT); - return Status::OK(); -} - -int bind_ipc_sock(const std::string &pathname, bool shall_listen) { - struct sockaddr_un socket_address; - int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (socket_fd < 0) { - ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname; - return -1; - } - /* Tell the system to allow the port to be reused. */ - int on = 1; - if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, - sizeof(on)) < 0) { - ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname; - close(socket_fd); - return -1; - } - - unlink(pathname.c_str()); - memset(&socket_address, 0, sizeof(socket_address)); - socket_address.sun_family = AF_UNIX; - if (pathname.size() + 1 > sizeof(socket_address.sun_path)) { - ARROW_LOG(ERROR) << "Socket pathname is too long."; - close(socket_fd); - return -1; - } - strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1); - - if (bind(socket_fd, (struct sockaddr *) &socket_address, - sizeof(socket_address)) != 0) { - ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname; - close(socket_fd); - return -1; - } - if (shall_listen && listen(socket_fd, 128) == -1) { - ARROW_LOG(ERROR) << "Could not listen to socket " << pathname; - close(socket_fd); - return -1; - } - return socket_fd; -} - -int connect_ipc_sock_retry(const std::string &pathname, - int num_retries, - int64_t timeout) { - /* Pick the default values if the user did not specify. */ - if (num_retries < 0) { - num_retries = NUM_CONNECT_ATTEMPTS; - } - if (timeout < 0) { - timeout = CONNECT_TIMEOUT_MS; - } - - int fd = -1; - for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) { - fd = connect_ipc_sock(pathname); - if (fd >= 0) { - break; - } - if (num_attempts == 0) { - ARROW_LOG(ERROR) << "Connection to socket failed for pathname " - << pathname; - } - /* Sleep for timeout milliseconds. */ - usleep(timeout * 1000); - } - /* If we could not connect to the socket, exit. */ - if (fd == -1) { - ARROW_LOG(FATAL) << "Could not connect to socket " << pathname; - } - return fd; -} - -int connect_ipc_sock(const std::string &pathname) { - struct sockaddr_un socket_address; - int socket_fd; - - socket_fd = socket(AF_UNIX, SOCK_STREAM, 0); - if (socket_fd < 0) { - ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname; - return -1; - } - - memset(&socket_address, 0, sizeof(socket_address)); - socket_address.sun_family = AF_UNIX; - if (pathname.size() + 1 > sizeof(socket_address.sun_path)) { - ARROW_LOG(ERROR) << "Socket pathname is too long."; - return -1; - } - strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1); - - if (connect(socket_fd, (struct sockaddr *) &socket_address, - sizeof(socket_address)) != 0) { - close(socket_fd); - return -1; - } - - return socket_fd; -} - -int AcceptClient(int socket_fd) { - int client_fd = accept(socket_fd, NULL, NULL); - if (client_fd < 0) { - ARROW_LOG(ERROR) << "Error reading from socket."; - return -1; - } - return client_fd; -} - -uint8_t *read_message_async(int sock) { - int64_t size; - Status s = ReadBytes(sock, (uint8_t *) &size, sizeof(int64_t)); - if (!s.ok()) { - /* The other side has closed the socket. */ - ARROW_LOG(DEBUG) - << "Socket has been closed, or some other error has occurred."; - close(sock); - return NULL; - } - uint8_t *message = (uint8_t *) malloc(size); - s = ReadBytes(sock, message, size); - if (!s.ok()) { - /* The other side has closed the socket. */ - ARROW_LOG(DEBUG) - << "Socket has been closed, or some other error has occurred."; - close(sock); - return NULL; - } - return message; -} diff --git a/src/plasma/plasma_io.h b/src/plasma/plasma_io.h deleted file mode 100644 index 5127633947a9..000000000000 --- a/src/plasma/plasma_io.h +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include -#include - -#include -#include - -#include "status.h" - -// TODO(pcm): Replace our own custom message header (message type, -// message length, plasma protocol verion) with one that is serialized -// using flatbuffers. -#define PLASMA_PROTOCOL_VERSION 0x0000000000000000 -#define DISCONNECT_CLIENT 0 - -arrow::Status WriteBytes(int fd, uint8_t *cursor, size_t length); - -arrow::Status WriteMessage(int fd, - int64_t type, - int64_t length, - uint8_t *bytes); - -arrow::Status ReadBytes(int fd, uint8_t *cursor, size_t length); - -arrow::Status ReadMessage(int fd, int64_t *type, std::vector &buffer); - -int bind_ipc_sock(const std::string &pathname, bool shall_listen); - -int connect_ipc_sock(const std::string &pathname); - -int connect_ipc_sock_retry(const std::string &pathname, - int num_retries, - int64_t timeout); - -int AcceptClient(int socket_fd); - -uint8_t *read_message_async(int sock); diff --git a/src/plasma/plasma_manager.cc b/src/plasma/plasma_manager.cc index 7a96c9788dd5..a0ba1dba4f7f 100644 --- a/src/plasma/plasma_manager.cc +++ b/src/plasma/plasma_manager.cc @@ -28,13 +28,14 @@ #include "utarray.h" #include "utstring.h" #include "common_protocol.h" -#include "common.h" #include "io.h" #include "net.h" #include "event_loop.h" -#include "plasma.h" -#include "plasma_protocol.h" -#include "plasma_client.h" +#include "common.h" +#include "plasma/plasma.h" +#include "plasma/events.h" +#include "plasma/protocol.h" +#include "plasma/client.h" #include "plasma_manager.h" #include "state/db.h" #include "state/object_table.h" @@ -168,7 +169,7 @@ struct WaitRequest { WaitRequest(ClientConnection *client_conn, int64_t timer, int64_t num_object_requests, - ObjectRequestMap &&object_requests, + plasma::ObjectRequestMap &&object_requests, int64_t num_objects_to_wait_for, int64_t num_satisfied) : client_conn(client_conn), @@ -188,7 +189,7 @@ struct WaitRequest { /** The object requests for this wait request. Each object request has a * status field which is either PLASMA_QUERY_LOCAL or PLASMA_QUERY_ANYWHERE. */ - ObjectRequestMap object_requests; + plasma::ObjectRequestMap object_requests; /** The minimum number of objects to wait for in this request. */ int64_t num_objects_to_wait_for; /** The number of object requests in this wait request that are already @@ -214,7 +215,7 @@ struct PlasmaManagerState { /** Event loop. */ event_loop *loop; /** Connection to the local plasma store for reading or writing data. */ - PlasmaClient *plasma_conn; + plasma::PlasmaClient *plasma_conn; /** Hash table of all contexts for active connections to * other plasma managers. These are used for writing data to * other plasma stores. */ @@ -341,9 +342,9 @@ ObjectWaitRequests **object_wait_requests_table_ptr_from_type( PlasmaManagerState *manager_state, int type) { /* We use different types of hash tables for different requests. */ - if (type == PLASMA_QUERY_LOCAL) { + if (type == plasma::PLASMA_QUERY_LOCAL) { return &manager_state->object_wait_requests_local; - } else if (type == PLASMA_QUERY_ANYWHERE) { + } else if (type == plasma::PLASMA_QUERY_ANYWHERE) { return &manager_state->object_wait_requests_remote; } else { LOG_FATAL("This code should be unreachable."); @@ -415,10 +416,10 @@ void remove_wait_request(PlasmaManagerState *manager_state, void return_from_wait(PlasmaManagerState *manager_state, WaitRequest *wait_req) { /* Send the reply to the client. */ - handle_sigpipe( - SendWaitReply(wait_req->client_conn->fd, wait_req->object_requests, - wait_req->num_object_requests), - wait_req->client_conn->fd); + handle_sigpipe(plasma::SendWaitReply(wait_req->client_conn->fd, + wait_req->object_requests, + wait_req->num_object_requests), + wait_req->client_conn->fd); /* Iterate over all object IDs requested as part of this wait request. * Remove the wait request from each of the relevant object_wait_requests hash * tables if it is present there. */ @@ -455,7 +456,8 @@ void update_object_wait_requests(PlasmaManagerState *manager_state, WaitRequest *wait_req = *wait_req_ptr; wait_req->num_satisfied += 1; /* Mark the object as present in the wait request. */ - auto object_request = wait_req->object_requests.find(obj_id); + auto object_request = + wait_req->object_requests.find(obj_id.to_plasma_id()); /* Check that we found the object. */ CHECK(object_request != wait_req->object_requests.end()); /* Check that the object found was not previously known to us. */ @@ -513,7 +515,7 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name, PlasmaManagerState *state = (PlasmaManagerState *) malloc(sizeof(PlasmaManagerState)); state->loop = event_loop_create(); - state->plasma_conn = new PlasmaClient(); + state->plasma_conn = new plasma::PlasmaClient(); ARROW_CHECK_OK(state->plasma_conn->Connect(store_socket_name, "", PLASMA_DEFAULT_RELEASE_DELAY)); state->manager_connections = NULL; @@ -551,7 +553,7 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name, state->local_available_objects = NULL; /* Subscribe to notifications about sealed objects. */ int plasma_fd; - ARROW_CHECK_OK(state->plasma_conn->Subscribe(plasma_fd)); + ARROW_CHECK_OK(state->plasma_conn->Subscribe(&plasma_fd)); /* Add the callback that processes the notification to the event loop. */ event_loop_add_file(state->loop, plasma_fd, EVENT_LOOP_READ, process_object_notification, state); @@ -642,7 +644,8 @@ int write_object_chunk(ClientConnection *conn, PlasmaRequestBuffer *buf) { conn->cursor = 0; /* We are done sending the object, so release it. The corresponding call to * plasma_get occurred in process_transfer_request. */ - ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release(buf->object_id)); + ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release( + buf->object_id.to_plasma_id())); } return 0; @@ -668,7 +671,8 @@ void send_queued_request(event_loop *loop, switch (buf->type) { case MessageType_PlasmaDataRequest: err = handle_sigpipe( - SendDataRequest(conn->fd, buf->object_id, state->addr, state->port), + plasma::SendDataRequest(conn->fd, buf->object_id.to_plasma_id(), + state->addr, state->port), conn->fd); break; case MessageType_PlasmaDataReply: @@ -676,9 +680,10 @@ void send_queued_request(event_loop *loop, if (conn->cursor == 0) { /* If the cursor is zero, we haven't sent any requests for this object * yet, so send the initial data request. */ - err = handle_sigpipe(SendDataReply(conn->fd, buf->object_id, - buf->data_size, buf->metadata_size), - conn->fd); + err = handle_sigpipe( + plasma::SendDataReply(conn->fd, buf->object_id.to_plasma_id(), + buf->data_size, buf->metadata_size), + conn->fd); } if (err == 0) { err = write_object_chunk(conn, buf); @@ -759,8 +764,10 @@ void process_data_chunk(event_loop *loop, LOG_DEBUG("reading on channel %d finished", data_sock); /* The following seal also triggers notification of clients for fetch or * wait requests, see process_object_notification. */ - ARROW_CHECK_OK(conn->manager_state->plasma_conn->Seal(buf->object_id)); - ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release(buf->object_id)); + ARROW_CHECK_OK( + conn->manager_state->plasma_conn->Seal(buf->object_id.to_plasma_id())); + ARROW_CHECK_OK( + conn->manager_state->plasma_conn->Release(buf->object_id.to_plasma_id())); /* Remove the request buffer used for reading this object's data. */ DL_DELETE(conn->transfer_queue, buf); free(buf); @@ -838,10 +845,11 @@ void process_transfer_request(event_loop *loop, } /* Allocate and append the request to the transfer queue. */ - ObjectBuffer object_buffer; + plasma::ObjectBuffer object_buffer; + plasma::ObjectID object_id = obj_id.to_plasma_id(); /* We pass in 0 to indicate that the command should return immediately. */ ARROW_CHECK_OK( - conn->manager_state->plasma_conn->Get(&obj_id, 1, 0, &object_buffer)); + conn->manager_state->plasma_conn->Get(&object_id, 1, 0, &object_buffer)); if (object_buffer.data_size == -1) { /* If the object wasn't locally available, exit immediately. If the object * later appears locally, the requesting plasma manager should request the @@ -908,7 +916,7 @@ void process_data_request(event_loop *loop, /* The corresponding call to plasma_release should happen in * process_data_chunk. */ Status s = conn->manager_state->plasma_conn->Create( - object_id, data_size, NULL, metadata_size, &(buf->data)); + object_id.to_plasma_id(), data_size, NULL, metadata_size, &(buf->data)); /* If success_create == true, a new object has been created. * If success_create == false the object creation has failed, possibly * due to an object with the same ID already existing in the Plasma Store. */ @@ -1118,7 +1126,8 @@ void object_present_callback(ObjectID object_id, CHECK(manager_count >= 1); /* Update the in-progress remote wait requests. */ - update_object_wait_requests(manager_state, object_id, PLASMA_QUERY_ANYWHERE, + update_object_wait_requests(manager_state, object_id, + plasma::PLASMA_QUERY_ANYWHERE, ObjectStatus_Remote); } @@ -1143,7 +1152,7 @@ void object_table_subscribe_callback(ObjectID object_id, void process_fetch_requests(ClientConnection *client_conn, int num_object_ids, - ObjectID object_ids[]) { + plasma::ObjectID object_ids[]) { PlasmaManagerState *manager_state = client_conn->manager_state; int num_object_ids_to_request = 0; @@ -1197,7 +1206,7 @@ int wait_timeout_handler(event_loop *loop, timer_id id, void *context) { } void process_wait_request(ClientConnection *client_conn, - ObjectRequestMap &&object_requests, + plasma::ObjectRequestMap &&object_requests, uint64_t timeout_ms, int num_ready_objects) { CHECK(client_conn != NULL); @@ -1231,10 +1240,10 @@ void process_wait_request(ClientConnection *client_conn, add_wait_request_for_object(manager_state, obj_id, object_request.type, wait_req); - if (object_request.type == PLASMA_QUERY_LOCAL) { + if (object_request.type == plasma::PLASMA_QUERY_LOCAL) { /* TODO(rkn): If desired, we could issue a fetch command here to retrieve * the object. */ - } else if (object_request.type == PLASMA_QUERY_ANYWHERE) { + } else if (object_request.type == plasma::PLASMA_QUERY_ANYWHERE) { /* Add this object ID to the list of object IDs to request notifications * for from the object table. */ object_ids_to_request[num_object_ids_to_request] = obj_id; @@ -1289,8 +1298,10 @@ void request_status_done(ObjectID object_id, ClientConnection *client_conn = (ClientConnection *) context; int status = request_status(object_id, manager_count, manager_vector, context); - handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1), - client_conn->fd); + plasma::ObjectID object_id_copy = object_id.to_plasma_id(); + handle_sigpipe( + plasma::SendStatusReply(client_conn->fd, &object_id_copy, &status, 1), + client_conn->fd); } int request_status(ObjectID object_id, @@ -1318,19 +1329,22 @@ void object_table_lookup_fail_callback(ObjectID object_id, CHECK(0); } -void process_status_request(ClientConnection *client_conn, ObjectID object_id) { +void process_status_request(ClientConnection *client_conn, + plasma::ObjectID object_id) { /* Return success immediately if we already have this object. */ if (is_object_local(client_conn->manager_state, object_id)) { int status = ObjectStatus_Local; - handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1), - client_conn->fd); + handle_sigpipe( + plasma::SendStatusReply(client_conn->fd, &object_id, &status, 1), + client_conn->fd); return; } if (client_conn->manager_state->db == NULL) { int status = ObjectStatus_Nonexistent; - handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1), - client_conn->fd); + handle_sigpipe( + plasma::SendStatusReply(client_conn->fd, &object_id, &status, 1), + client_conn->fd); return; } @@ -1426,9 +1440,9 @@ void process_add_object_notification(PlasmaManagerState *state, } /* Update the in-progress local and remote wait requests. */ - update_object_wait_requests(state, object_id, PLASMA_QUERY_LOCAL, + update_object_wait_requests(state, object_id, plasma::PLASMA_QUERY_LOCAL, ObjectStatus_Local); - update_object_wait_requests(state, object_id, PLASMA_QUERY_ANYWHERE, + update_object_wait_requests(state, object_id, plasma::PLASMA_QUERY_ANYWHERE, ObjectStatus_Local); } @@ -1546,46 +1560,47 @@ void process_message(event_loop *loop, switch (type) { case MessageType_PlasmaDataRequest: { LOG_DEBUG("Processing data request"); - ObjectID object_id; + plasma::ObjectID object_id; char *address; int port; - ARROW_CHECK_OK(ReadDataRequest(data, &object_id, &address, &port)); + ARROW_CHECK_OK( + plasma::ReadDataRequest(data, length, &object_id, &address, &port)); process_transfer_request(loop, object_id, address, port, conn); free(address); } break; case MessageType_PlasmaDataReply: { LOG_DEBUG("Processing data reply"); - ObjectID object_id; + plasma::ObjectID object_id; int64_t object_size; int64_t metadata_size; - ARROW_CHECK_OK( - ReadDataReply(data, &object_id, &object_size, &metadata_size)); + ARROW_CHECK_OK(plasma::ReadDataReply(data, length, &object_id, &object_size, + &metadata_size)); process_data_request(loop, client_sock, object_id, object_size, metadata_size, conn); } break; case MessageType_PlasmaFetchRequest: { LOG_DEBUG("Processing fetch remote"); - std::vector object_ids_to_fetch; + std::vector object_ids_to_fetch; /* TODO(pcm): process_fetch_requests allocates an array of num_objects * object_ids too so these should be shared in the future. */ - ARROW_CHECK_OK(ReadFetchRequest(data, object_ids_to_fetch)); + ARROW_CHECK_OK(plasma::ReadFetchRequest(data, length, object_ids_to_fetch)); process_fetch_requests(conn, object_ids_to_fetch.size(), object_ids_to_fetch.data()); } break; case MessageType_PlasmaWaitRequest: { LOG_DEBUG("Processing wait"); - ObjectRequestMap object_requests; + plasma::ObjectRequestMap object_requests; int64_t timeout_ms; int num_ready_objects; - ARROW_CHECK_OK(ReadWaitRequest(data, object_requests, &timeout_ms, - &num_ready_objects)); + ARROW_CHECK_OK(plasma::ReadWaitRequest(data, length, object_requests, + &timeout_ms, &num_ready_objects)); process_wait_request(conn, std::move(object_requests), timeout_ms, num_ready_objects); } break; case MessageType_PlasmaStatusRequest: { LOG_DEBUG("Processing status"); - ObjectID object_id; - ARROW_CHECK_OK(ReadStatusRequest(data, &object_id, 1)); + plasma::ObjectID object_id; + ARROW_CHECK_OK(plasma::ReadStatusRequest(data, length, &object_id, 1)); process_status_request(conn, object_id); } break; case DISCONNECT_CLIENT: { diff --git a/src/plasma/plasma_protocol.h b/src/plasma/plasma_protocol.h deleted file mode 100644 index 3d525282b8b5..000000000000 --- a/src/plasma/plasma_protocol.h +++ /dev/null @@ -1,194 +0,0 @@ -#ifndef PLASMA_PROTOCOL_H -#define PLASMA_PROTOCOL_H - -#include "status.h" -#include "format/plasma_generated.h" -#include "plasma.h" - -using arrow::Status; - -/* Plasma receive message. */ - -Status PlasmaReceive(int sock, - int64_t message_type, - std::vector &buffer); - -/* Plasma Create message functions. */ - -Status SendCreateRequest(int sock, - ObjectID object_id, - int64_t data_size, - int64_t metadata_size); - -Status ReadCreateRequest(uint8_t *data, - ObjectID *object_id, - int64_t *data_size, - int64_t *metadata_size); - -Status SendCreateReply(int sock, - ObjectID object_id, - PlasmaObject *object, - int error); - -Status ReadCreateReply(uint8_t *data, - ObjectID *object_id, - PlasmaObject *object); - -/* Plasma Seal message functions. */ - -Status SendSealRequest(int sock, ObjectID object_id, unsigned char *digest); - -Status ReadSealRequest(uint8_t *data, - ObjectID *object_id, - unsigned char *digest); - -Status SendSealReply(int sock, ObjectID object_id, int error); - -Status ReadSealReply(uint8_t *data, ObjectID *object_id); - -/* Plasma Get message functions. */ - -Status SendGetRequest(int sock, - ObjectID object_ids[], - int64_t num_objects, - int64_t timeout_ms); - -Status ReadGetRequest(uint8_t *data, - std::vector &object_ids, - int64_t *timeout_ms); - -Status SendGetReply( - int sock, - ObjectID object_ids[], - std::unordered_map &plasma_objects, - int64_t num_objects); - -Status ReadGetReply(uint8_t *data, - ObjectID object_ids[], - PlasmaObject plasma_objects[], - int64_t num_objects); - -/* Plasma Release message functions. */ - -Status SendReleaseRequest(int sock, ObjectID object_id); - -Status ReadReleaseRequest(uint8_t *data, ObjectID *object_id); - -Status SendReleaseReply(int sock, ObjectID object_id, int error); - -Status ReadReleaseReply(uint8_t *data, ObjectID *object_id); - -/* Plasma Delete message functions. */ - -Status SendDeleteRequest(int sock, ObjectID object_id); - -Status ReadDeleteRequest(uint8_t *data, ObjectID *object_id); - -Status SendDeleteReply(int sock, ObjectID object_id, int error); - -Status ReadDeleteReply(uint8_t *data, ObjectID *object_id); - -/* Satus messages. */ - -Status SendStatusRequest(int sock, ObjectID object_ids[], int64_t num_objects); - -Status ReadStatusRequest(uint8_t *data, - ObjectID object_ids[], - int64_t num_objects); - -Status SendStatusReply(int sock, - ObjectID object_ids[], - int object_status[], - int64_t num_objects); - -int64_t ReadStatusReply_num_objects(uint8_t *data); - -Status ReadStatusReply(uint8_t *data, - ObjectID object_ids[], - int object_status[], - int64_t num_objects); - -/* Plasma Constains message functions. */ - -Status SendContainsRequest(int sock, ObjectID object_id); - -Status ReadContainsRequest(uint8_t *data, ObjectID *object_id); - -Status SendContainsReply(int sock, ObjectID object_id, int has_object); - -Status ReadContainsReply(uint8_t *data, ObjectID *object_id, int *has_object); - -/* Plasma Connect message functions. */ - -Status SendConnectRequest(int sock); - -Status ReadConnectRequest(uint8_t *data); - -Status SendConnectReply(int sock, int64_t memory_capacity); - -Status ReadConnectReply(uint8_t *data, int64_t *memory_capacity); - -/* Plasma Evict message functions (no reply so far). */ - -Status SendEvictRequest(int sock, int64_t num_bytes); - -Status ReadEvictRequest(uint8_t *data, int64_t *num_bytes); - -Status SendEvictReply(int sock, int64_t num_bytes); - -Status ReadEvictReply(uint8_t *data, int64_t &num_bytes); - -/* Plasma Fetch Remote message functions. */ - -Status SendFetchRequest(int sock, ObjectID object_ids[], int64_t num_objects); - -Status ReadFetchRequest(uint8_t *data, std::vector &object_ids); - -/* Plasma Wait message functions. */ - -Status SendWaitRequest(int sock, - ObjectRequest object_requests[], - int num_requests, - int num_ready_objects, - int64_t timeout_ms); - -Status ReadWaitRequest(uint8_t *data, - ObjectRequestMap &object_requests, - int64_t *timeout_ms, - int *num_ready_objects); - -Status SendWaitReply(int sock, - const ObjectRequestMap &object_requests, - int num_ready_objects); - -Status ReadWaitReply(uint8_t *data, - ObjectRequest object_requests[], - int *num_ready_objects); - -/* Plasma Subscribe message functions. */ - -Status SendSubscribeRequest(int sock); - -/* Data messages. */ - -Status SendDataRequest(int sock, - ObjectID object_id, - const char *address, - int port); - -Status ReadDataRequest(uint8_t *data, - ObjectID *object_id, - char **address, - int *port); - -Status SendDataReply(int sock, - ObjectID object_id, - int64_t object_size, - int64_t metadata_size); - -Status ReadDataReply(uint8_t *data, - ObjectID *object_id, - int64_t *object_size, - int64_t *metadata_size); - -#endif /* PLASMA_PROTOCOL */ diff --git a/src/plasma/plasma_store.cc b/src/plasma/plasma_store.cc deleted file mode 100644 index 9fa4ea446342..000000000000 --- a/src/plasma/plasma_store.cc +++ /dev/null @@ -1,689 +0,0 @@ -// PLASMA STORE: This is a simple object store server process -// -// It accepts incoming client connections on a unix domain socket -// (name passed in via the -s option of the executable) and uses a -// single thread to serve the clients. Each client establishes a -// connection and can create objects, wait for objects and seal -// objects through that connection. -// -// It keeps a hash table that maps object_ids (which are 20 byte long, -// just enough to store and SHA1 hash) to memory mapped files. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "plasma_common.h" -#include "plasma_store.h" -#include "format/common_generated.h" -#include "plasma_io.h" -#include "malloc.h" - -extern "C" { -#include "fling.h" -void *dlmalloc(size_t); -void *dlmemalign(size_t alignment, size_t bytes); -void dlfree(void *); -size_t dlmalloc_set_footprint_limit(size_t bytes); -} - -struct GetRequest { - GetRequest(Client *client, const std::vector &object_ids); - - /// The client that called get. - Client *client; - /// The ID of the timer that will time out and cause this wait to return to - /// the client if it hasn't already returned. - int64_t timer; - /// The object IDs involved in this request. This is used in the reply. - std::vector object_ids; - /// The object information for the objects in this request. This is used in - /// the reply. - std::unordered_map objects; - /// The minimum number of objects to wait for in this request. - int64_t num_objects_to_wait_for; - /// The number of object requests in this wait request that are already - /// satisfied. - int64_t num_satisfied; -}; - -GetRequest::GetRequest(Client *client, const std::vector &object_ids) - : client(client), - timer(-1), - object_ids(object_ids.begin(), object_ids.end()), - objects(object_ids.size()), - num_satisfied(0) { - std::unordered_set unique_ids(object_ids.begin(), - object_ids.end()); - num_objects_to_wait_for = unique_ids.size(); -} - -Client::Client(int fd) : fd(fd) {} - -PlasmaStore::PlasmaStore(EventLoop *loop, int64_t system_memory) - : loop_(loop), eviction_policy_(&store_info_) { - store_info_.memory_capacity = system_memory; -} - -PlasmaStore::~PlasmaStore() { - for (const auto &element : pending_notifications_) { - auto object_notifications = element.second.object_notifications; - for (int i = 0; i < object_notifications.size(); ++i) { - uint8_t *notification = (uint8_t *) object_notifications.at(i); - uint8_t *data = notification; - free(data); - } - } -} - -// If this client is not already using the object, add the client to the -// object's list of clients, otherwise do nothing. -void PlasmaStore::add_client_to_object_clients(ObjectTableEntry *entry, - Client *client) { - // Check if this client is already using the object. - if (entry->clients.find(client) != entry->clients.end()) { - return; - } - // If there are no other clients using this object, notify the eviction policy - // that the object is being used. - if (entry->clients.size() == 0) { - // Tell the eviction policy that this object is being used. - std::vector objects_to_evict; - eviction_policy_.begin_object_access(entry->object_id, objects_to_evict); - delete_objects(objects_to_evict); - } - // Add the client pointer to the list of clients using this object. - entry->clients.insert(client); -} - -// Create a new object buffer in the hash table. -int PlasmaStore::create_object(ObjectID object_id, - int64_t data_size, - int64_t metadata_size, - Client *client, - PlasmaObject *result) { - ARROW_LOG(DEBUG) << "creating object " << object_id.hex(); - if (store_info_.objects.count(object_id) != 0) { - // There is already an object with the same ID in the Plasma Store, so - // ignore this requst. - return PlasmaError_ObjectExists; - } - // Try to evict objects until there is enough space. - uint8_t *pointer; - do { - // Allocate space for the new object. We use dlmemalign instead of dlmalloc - // in order to align the allocated region to a 64-byte boundary. This is not - // strictly necessary, but it is an optimization that could speed up the - // computation of a hash of the data (see compute_object_hash_parallel in - // plasma_client.cc). Note that even though this pointer is 64-byte aligned, - // it is not guaranteed that the corresponding pointer in the client will be - // 64-byte aligned, but in practice it often will be. - pointer = (uint8_t *) dlmemalign(BLOCK_SIZE, data_size + metadata_size); - if (pointer == NULL) { - // Tell the eviction policy how much space we need to create this object. - std::vector objects_to_evict; - bool success = eviction_policy_.require_space(data_size + metadata_size, - objects_to_evict); - delete_objects(objects_to_evict); - // Return an error to the client if not enough space could be freed to - // create the object. - if (!success) { - return PlasmaError_OutOfMemory; - } - } - } while (pointer == NULL); - int fd; - int64_t map_size; - ptrdiff_t offset; - get_malloc_mapinfo(pointer, &fd, &map_size, &offset); - assert(fd != -1); - - auto entry = std::unique_ptr(new ObjectTableEntry()); - entry->object_id = object_id; - entry->info.object_id = object_id.binary(); - entry->info.data_size = data_size; - entry->info.metadata_size = metadata_size; - entry->pointer = pointer; - // TODO(pcm): Set the other fields. - entry->fd = fd; - entry->map_size = map_size; - entry->offset = offset; - entry->state = PLASMA_CREATED; - - store_info_.objects[object_id] = std::move(entry); - result->handle.store_fd = fd; - result->handle.mmap_size = map_size; - result->data_offset = offset; - result->metadata_offset = offset + data_size; - result->data_size = data_size; - result->metadata_size = metadata_size; - // Notify the eviction policy that this object was created. This must be done - // immediately before the call to add_client_to_object_clients so that the - // eviction policy does not have an opportunity to evict the object. - eviction_policy_.object_created(object_id); - // Record that this client is using this object. - add_client_to_object_clients(store_info_.objects[object_id].get(), client); - return PlasmaError_OK; -} - -void PlasmaObject_init(PlasmaObject *object, ObjectTableEntry *entry) { - DCHECK(object != NULL); - DCHECK(entry != NULL); - DCHECK(entry->state == PLASMA_SEALED); - object->handle.store_fd = entry->fd; - object->handle.mmap_size = entry->map_size; - object->data_offset = entry->offset; - object->metadata_offset = entry->offset + entry->info.data_size; - object->data_size = entry->info.data_size; - object->metadata_size = entry->info.metadata_size; -} - -void PlasmaStore::return_from_get(GetRequest *get_req) { - // Send the get reply to the client. - Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0], - get_req->objects, get_req->object_ids.size()); - warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd); - // If we successfully sent the get reply message to the client, then also send - // the file descriptors. - if (s.ok()) { - // Send all of the file descriptors for the present objects. - for (const auto &object_id : get_req->object_ids) { - PlasmaObject &object = get_req->objects[object_id]; - // We use the data size to indicate whether the object is present or not. - if (object.data_size != -1) { - int error_code = send_fd(get_req->client->fd, object.handle.store_fd); - // If we failed to send the file descriptor, loop until we have sent it - // successfully. TODO(rkn): This is problematic for two reasons. First - // of all, sending the file descriptor should just succeed without any - // errors, but sometimes I see a "Message too long" error number. - // Second, looping like this allows a client to potentially block the - // plasma store event loop which should never happen. - while (error_code < 0) { - if (errno == EMSGSIZE) { - ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying."; - error_code = send_fd(get_req->client->fd, object.handle.store_fd); - continue; - } - warn_if_sigpipe(error_code, get_req->client->fd); - break; - } - } - } - } - - // Remove the get request from each of the relevant object_get_requests hash - // tables if it is present there. It should only be present there if the get - // request timed out. - for (ObjectID &object_id : get_req->object_ids) { - auto &get_requests = object_get_requests_[object_id]; - // Erase get_req from the vector. - auto it = std::find(get_requests.begin(), get_requests.end(), get_req); - if (it != get_requests.end()) { - get_requests.erase(it); - } - } - // Remove the get request. - if (get_req->timer != -1) { - ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK); - } - delete get_req; -} - -void PlasmaStore::update_object_get_requests(ObjectID object_id) { - std::vector &get_requests = object_get_requests_[object_id]; - int index = 0; - int num_requests = get_requests.size(); - for (int i = 0; i < num_requests; ++i) { - GetRequest *get_req = get_requests[index]; - auto entry = get_object_table_entry(&store_info_, object_id); - ARROW_CHECK(entry != NULL); - - PlasmaObject_init(&get_req->objects[object_id], entry); - get_req->num_satisfied += 1; - // Record the fact that this client will be using this object and will - // be responsible for releasing this object. - add_client_to_object_clients(entry, get_req->client); - - // If this get request is done, reply to the client. - if (get_req->num_satisfied == get_req->num_objects_to_wait_for) { - return_from_get(get_req); - } else { - // The call to return_from_get will remove the current element in the - // array, so we only increment the counter in the else branch. - index += 1; - } - } - - DCHECK(index == get_requests.size()); - // Remove the array of get requests for this object, since no one should be - // waiting for this object anymore. - object_get_requests_.erase(object_id); -} - -void PlasmaStore::process_get_request(Client *client, - const std::vector &object_ids, - uint64_t timeout_ms) { - // Create a get request for this object. - GetRequest *get_req = new GetRequest(client, object_ids); - - for (auto object_id : object_ids) { - // Check if this object is already present locally. If so, record that the - // object is being used and mark it as accounted for. - auto entry = get_object_table_entry(&store_info_, object_id); - if (entry && entry->state == PLASMA_SEALED) { - // Update the get request to take into account the present object. - PlasmaObject_init(&get_req->objects[object_id], entry); - get_req->num_satisfied += 1; - // If necessary, record that this client is using this object. In the case - // where entry == NULL, this will be called from seal_object. - add_client_to_object_clients(entry, client); - } else { - // Add a placeholder plasma object to the get request to indicate that the - // object is not present. This will be parsed by the client. We set the - // data size to -1 to indicate that the object is not present. - get_req->objects[object_id].data_size = -1; - // Add the get request to the relevant data structures. - object_get_requests_[object_id].push_back(get_req); - } - } - - // If all of the objects are present already or if the timeout is 0, return to - // the client. - if (get_req->num_satisfied == get_req->num_objects_to_wait_for || - timeout_ms == 0) { - return_from_get(get_req); - } else if (timeout_ms != -1) { - // Set a timer that will cause the get request to return to the client. Note - // that a timeout of -1 is used to indicate that no timer should be set. - get_req->timer = - loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) { - return_from_get(get_req); - return kEventLoopTimerDone; - }); - } -} - -int PlasmaStore::remove_client_from_object_clients(ObjectTableEntry *entry, - Client *client) { - auto it = entry->clients.find(client); - if (it != entry->clients.end()) { - entry->clients.erase(it); - // If no more clients are using this object, notify the eviction policy - // that the object is no longer being used. - if (entry->clients.size() == 0) { - // Tell the eviction policy that this object is no longer being used. - std::vector objects_to_evict; - eviction_policy_.end_object_access(entry->object_id, objects_to_evict); - delete_objects(objects_to_evict); - } - // Return 1 to indicate that the client was removed. - return 1; - } else { - // Return 0 to indicate that the client was not removed. - return 0; - } -} - -void PlasmaStore::release_object(ObjectID object_id, Client *client) { - auto entry = get_object_table_entry(&store_info_, object_id); - ARROW_CHECK(entry != NULL); - // Remove the client from the object's array of clients. - ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1); -} - -// Check if an object is present. -int PlasmaStore::contains_object(ObjectID object_id) { - auto entry = get_object_table_entry(&store_info_, object_id); - return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND - : OBJECT_NOT_FOUND; -} - -// Seal an object that has been created in the hash table. -void PlasmaStore::seal_object(ObjectID object_id, unsigned char digest[]) { - ARROW_LOG(DEBUG) << "sealing object " << object_id.hex(); - auto entry = get_object_table_entry(&store_info_, object_id); - ARROW_CHECK(entry != NULL); - ARROW_CHECK(entry->state == PLASMA_CREATED); - // Set the state of object to SEALED. - entry->state = PLASMA_SEALED; - // Set the object digest. - entry->info.digest = std::string((char *) &digest[0], kDigestSize); - // Inform all subscribers that a new object has been sealed. - push_notification(&entry->info); - - // Update all get requests that involve this object. - update_object_get_requests(object_id); -} - -void PlasmaStore::delete_objects(const std::vector &object_ids) { - for (const auto &object_id : object_ids) { - ARROW_LOG(DEBUG) << "deleting object " << object_id.hex(); - auto entry = get_object_table_entry(&store_info_, object_id); - // TODO(rkn): This should probably not fail, but should instead throw an - // error. Maybe we should also support deleting objects that have been - // created but not sealed. - ARROW_CHECK(entry != NULL) - << "To delete an object it must be in the object table."; - ARROW_CHECK(entry->state == PLASMA_SEALED) - << "To delete an object it must have been sealed."; - ARROW_CHECK(entry->clients.size() == 0) - << "To delete an object, there must be no clients currently using it."; - dlfree(entry->pointer); - store_info_.objects.erase(object_id); - // Inform all subscribers that the object has been deleted. - ObjectInfoT notification; - notification.object_id = object_id.binary(); - notification.is_deletion = true; - push_notification(¬ification); - } -} - -void PlasmaStore::connect_client(int listener_sock) { - int client_fd = AcceptClient(listener_sock); - // This is freed in disconnect_client. - Client *client = new Client(client_fd); - // Add a callback to handle events on this socket. - // TODO(pcm): Check return value. - loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) { - process_message(client); - }); - ARROW_LOG(DEBUG) << "New connection with fd " << client_fd; -} - -void PlasmaStore::disconnect_client(Client *client) { - ARROW_CHECK(client != NULL); - ARROW_CHECK(client->fd > 0); - loop_->remove_file_event(client->fd); - // Close the socket. - close(client->fd); - ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd; - // If this client was using any objects, remove it from the appropriate - // lists. - for (const auto &entry : store_info_.objects) { - remove_client_from_object_clients(entry.second.get(), client); - } - // Note, the store may still attempt to send a message to the disconnected - // client (for example, when an object ID that the client was waiting for - // is ready). In these cases, the attempt to send the message will fail, but - // the store should just ignore the failure. - delete client; -} - -/// Send notifications about sealed objects to the subscribers. This is called -/// in seal_object. If the socket's send buffer is full, the notification will -/// be -/// buffered, and this will be called again when the send buffer has room. -/// -/// @param client The client to send the notification to. -/// @return Void. -void PlasmaStore::send_notifications(int client_fd) { - auto it = pending_notifications_.find(client_fd); - - int num_processed = 0; - bool closed = false; - // Loop over the array of pending notifications and send as many of them as - // possible. - for (int i = 0; i < it->second.object_notifications.size(); ++i) { - uint8_t *notification = (uint8_t *) it->second.object_notifications.at(i); - // Decode the length, which is the first bytes of the message. - int64_t size = *((int64_t *) notification); - - // Attempt to send a notification about this object ID. - int nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0); - if (nbytes >= 0) { - ARROW_CHECK(nbytes == sizeof(int64_t) + size); - } else if (nbytes == -1 && - (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { - ARROW_LOG(DEBUG) - << "The socket's send buffer is full, so we are caching this " - "notification and will send it later."; - // Add a callback to the event loop to send queued notifications whenever - // there is room in the socket's send buffer. Callbacks can be added - // more than once here and will be overwritten. The callback is removed - // at the end of the method. - // TODO(pcm): Introduce status codes and check in case the file descriptor - // is added twice. - loop_->add_file_event( - client_fd, kEventLoopWrite, - [this, client_fd](int events) { send_notifications(client_fd); }); - break; - } else { - ARROW_LOG(WARNING) << "Failed to send notification to client on fd " - << client_fd; - if (errno == EPIPE) { - closed = true; - break; - } - } - num_processed += 1; - // The corresponding malloc happened in create_object_info_buffer - // within push_notification. - free(notification); - } - // Remove the sent notifications from the array. - it->second.object_notifications.erase( - it->second.object_notifications.begin(), - it->second.object_notifications.begin() + num_processed); - - // Stop sending notifications if the pipe was broken. - if (closed) { - close(client_fd); - pending_notifications_.erase(client_fd); - } - - // If we have sent all notifications, remove the fd from the event loop. - if (it->second.object_notifications.empty()) { - loop_->remove_file_event(client_fd); - } -} - -void PlasmaStore::push_notification(ObjectInfoT *object_info) { - for (auto &element : pending_notifications_) { - uint8_t *notification = create_object_info_buffer(object_info); - element.second.object_notifications.push_back(notification); - send_notifications(element.first); - // The notification gets freed in send_notifications when the notification - // is sent over the socket. - } -} - -// Subscribe to notifications about sealed objects. -void PlasmaStore::subscribe_to_updates(Client *client) { - ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd; - // TODO(rkn): The store could block here if the client doesn't send a file - // descriptor. - int fd = recv_fd(client->fd); - if (fd < 0) { - // This may mean that the client died before sending the file descriptor. - ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd " - << client->fd << "."; - return; - } - - // Create a new array to buffer notifications that can't be sent to the - // subscriber yet because the socket send buffer is full. TODO(rkn): the queue - // never gets freed. - NotificationQueue &queue = pending_notifications_[fd]; - - // Push notifications to the new subscriber about existing objects. - for (const auto &entry : store_info_.objects) { - push_notification(&entry.second->info); - } - send_notifications(fd); -} - -Status PlasmaStore::process_message(Client *client) { - int64_t type; - Status s = ReadMessage(client->fd, &type, input_buffer_); - ARROW_CHECK(s.ok() || s.IsIOError()); - - uint8_t *input = input_buffer_.data(); - ObjectID object_id; - PlasmaObject object; - // TODO(pcm): Get rid of the following. - memset(&object, 0, sizeof(object)); - - // Process the different types of requests. - switch (type) { - case MessageType_PlasmaCreateRequest: { - int64_t data_size; - int64_t metadata_size; - RETURN_NOT_OK( - ReadCreateRequest(input, &object_id, &data_size, &metadata_size)); - int error_code = - create_object(object_id, data_size, metadata_size, client, &object); - HANDLE_SIGPIPE(SendCreateReply(client->fd, object_id, &object, error_code), - client->fd); - if (error_code == PlasmaError_OK) { - warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd); - } - } break; - case MessageType_PlasmaGetRequest: { - std::vector object_ids_to_get; - int64_t timeout_ms; - RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms)); - process_get_request(client, object_ids_to_get, timeout_ms); - } break; - case MessageType_PlasmaReleaseRequest: - RETURN_NOT_OK(ReadReleaseRequest(input, &object_id)); - release_object(object_id, client); - break; - case MessageType_PlasmaContainsRequest: - RETURN_NOT_OK(ReadContainsRequest(input, &object_id)); - if (contains_object(object_id) == OBJECT_FOUND) { - HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd); - } else { - HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd); - } - break; - case MessageType_PlasmaSealRequest: { - unsigned char digest[kDigestSize]; - RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0])); - seal_object(object_id, &digest[0]); - } break; - case MessageType_PlasmaEvictRequest: { - // This code path should only be used for testing. - int64_t num_bytes; - RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes)); - std::vector objects_to_evict; - int64_t num_bytes_evicted = - eviction_policy_.choose_objects_to_evict(num_bytes, objects_to_evict); - delete_objects(objects_to_evict); - HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd); - } break; - case MessageType_PlasmaSubscribeRequest: - subscribe_to_updates(client); - break; - case MessageType_PlasmaConnectRequest: { - HANDLE_SIGPIPE(SendConnectReply(client->fd, store_info_.memory_capacity), - client->fd); - } break; - case DISCONNECT_CLIENT: - ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd; - disconnect_client(client); - break; - default: - // This code should be unreachable. - ARROW_CHECK(0); - } - return Status::OK(); -} - -// Report "success" to valgrind. -void signal_handler(int signal) { - if (signal == SIGTERM) { - exit(0); - } -} - -void start_server(char *socket_name, int64_t system_memory) { - // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write - // to a client that has already died, the store could die. - signal(SIGPIPE, SIG_IGN); - // Create the event loop. - EventLoop loop; - PlasmaStore store(&loop, system_memory); - int socket = bind_ipc_sock(socket_name, true); - ARROW_CHECK(socket >= 0); - // TODO(pcm): Check return value. - loop.add_file_event(socket, kEventLoopRead, [&store, socket](int events) { - store.connect_client(socket); - }); - loop.run(); -} - -int main(int argc, char *argv[]) { - signal(SIGTERM, signal_handler); - char *socket_name = NULL; - int64_t system_memory = -1; - int c; - while ((c = getopt(argc, argv, "s:m:")) != -1) { - switch (c) { - case 's': - socket_name = optarg; - break; - case 'm': { - char extra; - int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra); - ARROW_CHECK(scanned == 1); - ARROW_LOG(INFO) << "Allowing the Plasma store to use up to " - << ((double) system_memory) / 1000000000 - << "GB of memory."; - break; - } - default: - exit(-1); - } - } - if (!socket_name) { - ARROW_LOG(FATAL) - << "please specify socket for incoming connections with -s switch"; - } - if (system_memory == -1) { - ARROW_LOG(FATAL) - << "please specify the amount of system memory with -m switch"; - } -#ifdef __linux__ - // On Linux, check that the amount of memory available in /dev/shm is large - // enough to accommodate the request. If it isn't, then fail. - int shm_fd = open("/dev/shm", O_RDONLY); - struct statvfs shm_vfs_stats; - fstatvfs(shm_fd, &shm_vfs_stats); - // The value shm_vfs_stats.f_bsize is the block size, and the value - // shm_vfs_stats.f_bavail is the number of available blocks. - int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail; - close(shm_fd); - if (system_memory > shm_mem_avail) { - ARROW_LOG(FATAL) - << "System memory request exceeds memory available in /dev/shm. The " - "request is for " - << system_memory << " bytes, and the amount available is " - << shm_mem_avail - << " bytes. You may be able to free up space by deleting files in " - "/dev/shm. If you are inside a Docker container, you may need to " - "pass " - "an argument with the flag '--shm-size' to 'docker run'."; - } -#endif - // Make it so dlmalloc fails if we try to request more memory than is - // available. - dlmalloc_set_footprint_limit((size_t) system_memory); - ARROW_LOG(DEBUG) << "starting server listening on " << socket_name; - start_server(socket_name, system_memory); -} diff --git a/src/plasma/plasma_store.h b/src/plasma/plasma_store.h deleted file mode 100644 index c63fb43b6fbd..000000000000 --- a/src/plasma/plasma_store.h +++ /dev/null @@ -1,154 +0,0 @@ -#ifndef PLASMA_STORE_H -#define PLASMA_STORE_H - -#include "eviction_policy.h" -#include "plasma.h" -#include "plasma_common.h" -#include "plasma_events.h" -#include "plasma_protocol.h" - -class GetRequest; - -struct NotificationQueue { - /// The object notifications for clients. We notify the client about the - /// objects in the order that the objects were sealed or deleted. - std::deque object_notifications; -}; - -/// Contains all information that is associated with a Plasma store client. -struct Client { - Client(int fd); - - /// The file descriptor used to communicate with the client. - int fd; -}; - -class PlasmaStore { - public: - PlasmaStore(EventLoop *loop, int64_t system_memory); - - ~PlasmaStore(); - - /// Create a new object. The client must do a call to release_object to tell - /// the store when it is done with the object. - /// - /// @param object_id Object ID of the object to be created. - /// @param data_size Size in bytes of the object to be created. - /// @param metadata_size Size in bytes of the object metadata. - /// @return One of the following error codes: - /// - PlasmaError_OK, if the object was created successfully. - /// - PlasmaError_ObjectExists, if an object with this ID is already - /// present in the store. In this case, the client should not call - /// plasma_release. - /// - PlasmaError_OutOfMemory, if the store is out of memory and - /// cannot create the object. In this case, the client should not call - /// plasma_release. - int create_object(ObjectID object_id, - int64_t data_size, - int64_t metadata_size, - Client *client, - PlasmaObject *result); - - /// Delete objects that have been created in the hash table. This should only - /// be called on objects that are returned by the eviction policy to evict. - /// - /// @param object_ids Object IDs of the objects to be deleted. - /// @return Void. - void delete_objects(const std::vector &object_ids); - - /// Process a get request from a client. This method assumes that we will - /// eventually have these objects sealed. If one of the objects has not yet - /// been sealed, the client that requested the object will be notified when it - /// is sealed. - /// - /// For each object, the client must do a call to release_object to tell the - /// store when it is done with the object. - /// - /// @param client The client making this request. - /// @param object_ids Object IDs of the objects to be gotten. - /// @param timeout_ms The timeout for the get request in milliseconds. - /// @return Void. - void process_get_request(Client *client, - const std::vector &object_ids, - uint64_t timeout_ms); - - /// Seal an object. The object is now immutable and can be accessed with get. - /// - /// @param object_id Object ID of the object to be sealed. - /// @param digest The digest of the object. This is used to tell if two - /// objects - /// with the same object ID are the same. - /// @return Void. - void seal_object(ObjectID object_id, unsigned char digest[]); - - /// Check if the plasma store contains an object: - /// - /// @param object_id Object ID that will be checked. - /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if - /// not - int contains_object(ObjectID object_id); - - /// Record the fact that a particular client is no longer using an object. - /// - /// @param object_id The object ID of the object that is being released. - /// @param client The client making this request. - /// @param Void. - void release_object(ObjectID object_id, Client *client); - - /// Subscribe a file descriptor to updates about new sealed objects. - /// - /// @param client The client making this request. - /// @return Void. - void subscribe_to_updates(Client *client); - - /// Connect a new client to the PlasmaStore. - /// - /// @param listener_sock The socket that is listening to incoming connections. - /// @return Void. - void connect_client(int listener_sock); - - /// Disconnect a client from the PlasmaStore. - /// - /// @param client The client that is disconnected. - /// @return Void. - void disconnect_client(Client *client); - - void send_notifications(int client_fd); - - Status process_message(Client *client); - - private: - void push_notification(ObjectInfoT *object_notification); - - void add_client_to_object_clients(ObjectTableEntry *entry, Client *client); - - void return_from_get(GetRequest *get_req); - - void update_object_get_requests(ObjectID object_id); - - int remove_client_from_object_clients(ObjectTableEntry *entry, - Client *client); - - /// Event loop of the plasma store. - EventLoop *loop_; - /// The plasma store information, including the object tables, that is exposed - /// to the eviction policy. - PlasmaStoreInfo store_info_; - /// The state that is managed by the eviction policy. - EvictionPolicy eviction_policy_; - /// Input buffer. This is allocated only once to avoid mallocs for every - /// call to process_message. - std::vector input_buffer_; - /// A hash table mapping object IDs to a vector of the get requests that are - /// waiting for the object to arrive. - std::unordered_map, UniqueIDHasher> - object_get_requests_; - /// The pending notifications that have not been sent to subscribers because - /// the socket send buffers were full. This is a hash table from client file - /// descriptor to an array of object_ids to send to that client. - /// TODO(pcm): Consider putting this into the Client data structure and - /// reorganize the code slightly. - std::unordered_map pending_notifications_; -}; - -#endif // PLASMA_STORE_H diff --git a/src/plasma/status.cc b/src/plasma/status.cc deleted file mode 100644 index 11082d6cd9da..000000000000 --- a/src/plasma/status.cc +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. -// -// A Status encapsulates the result of an operation. It may indicate success, -// or it may indicate an error with an associated error message. -// -// Multiple threads can invoke const methods on a Status without -// external synchronization, but if any of the threads may call a -// non-const method, all threads accessing the same Status must use -// external synchronization. - -#include "status.h" - -#include - -namespace arrow { - -Status::Status(StatusCode code, const std::string &msg, int16_t posix_code) { - assert(code != StatusCode::OK); - const uint32_t size = static_cast(msg.size()); - char *result = new char[size + 7]; - memcpy(result, &size, sizeof(size)); - result[4] = static_cast(code); - memcpy(result + 5, &posix_code, sizeof(posix_code)); - memcpy(result + 7, msg.c_str(), msg.size()); - state_ = result; -} - -const char *Status::CopyState(const char *state) { - uint32_t size; - memcpy(&size, state, sizeof(size)); - char *result = new char[size + 7]; - memcpy(result, state, size + 7); - return result; -} - -std::string Status::CodeAsString() const { - if (state_ == NULL) { - return "OK"; - } - - const char *type; - switch (code()) { - case StatusCode::OK: - type = "OK"; - break; - case StatusCode::OutOfMemory: - type = "Out of memory"; - break; - case StatusCode::KeyError: - type = "Key error"; - break; - case StatusCode::TypeError: - type = "Type error"; - break; - case StatusCode::Invalid: - type = "Invalid"; - break; - case StatusCode::IOError: - type = "IOError"; - break; - case StatusCode::UnknownError: - type = "Unknown error"; - break; - case StatusCode::NotImplemented: - type = "NotImplemented"; - break; - default: - type = "Unknown"; - break; - } - return std::string(type); -} - -std::string Status::ToString() const { - std::string result(CodeAsString()); - if (state_ == NULL) { - return result; - } - - result.append(": "); - - uint32_t length; - memcpy(&length, state_, sizeof(length)); - result.append(reinterpret_cast(state_ + 7), length); - return result; -} - -} // namespace arrow diff --git a/src/plasma/status.h b/src/plasma/status.h deleted file mode 100644 index 30d3a1e2f826..000000000000 --- a/src/plasma/status.h +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. -// -// A Status encapsulates the result of an operation. It may indicate success, -// or it may indicate an error with an associated error message. -// -// Multiple threads can invoke const methods on a Status without -// external synchronization, but if any of the threads may call a -// non-const method, all threads accessing the same Status must use -// external synchronization. - -// Adapted from Kudu github.com/cloudera/kudu - -#ifndef ARROW_STATUS_H_ -#define ARROW_STATUS_H_ - -#include -#include -#include - -// Return the given status if it is not OK. -#define ARROW_RETURN_NOT_OK(s) \ - do { \ - ::arrow::Status _s = (s); \ - if (!_s.ok()) { \ - return _s; \ - } \ - } while (0); - -// Return the given status if it is not OK, but first clone it and -// prepend the given message. -#define ARROW_RETURN_NOT_OK_PREPEND(s, msg) \ - do { \ - ::arrow::Status _s = (s); \ - if (::gutil::PREDICT_FALSE(!_s.ok())) \ - return _s.CloneAndPrepend(msg); \ - } while (0); - -// Return 'to_return' if 'to_call' returns a bad status. -// The substitution for 'to_return' may reference the variable -// 's' for the bad status. -#define ARROW_RETURN_NOT_OK_RET(to_call, to_return) \ - do { \ - ::arrow::Status s = (to_call); \ - if (::gutil::PREDICT_FALSE(!s.ok())) \ - return (to_return); \ - } while (0); - -// If 'to_call' returns a bad status, CHECK immediately with a logged message -// of 'msg' followed by the status. -#define ARROW_CHECK_OK_PREPEND(to_call, msg) \ - do { \ - ::arrow::Status _s = (to_call); \ - ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \ - } while (0); - -// If the status is bad, CHECK immediately, appending the status to the -// logged message. -#define ARROW_CHECK_OK(s) ARROW_CHECK_OK_PREPEND(s, "Bad status") - -namespace arrow { - -#define RETURN_NOT_OK(s) \ - do { \ - Status _s = (s); \ - if (!_s.ok()) { \ - return _s; \ - } \ - } while (0); - -#define RETURN_NOT_OK_ELSE(s, else_) \ - do { \ - Status _s = (s); \ - if (!_s.ok()) { \ - else_; \ - return _s; \ - } \ - } while (0); - -enum class StatusCode : char { - OK = 0, - OutOfMemory = 1, - KeyError = 2, - TypeError = 3, - Invalid = 4, - IOError = 5, - UnknownError = 9, - NotImplemented = 10, - PlasmaObjectExists = 20, - PlasmaObjectNonexistent = 21, - PlasmaStoreFull = 22 -}; - -class Status { - public: - // Create a success status. - Status() : state_(NULL) {} - ~Status() { delete[] state_; } - - Status(StatusCode code, const std::string &msg) : Status(code, msg, -1) {} - - // Copy the specified status. - Status(const Status &s); - void operator=(const Status &s); - - // Return a success status. - static Status OK() { return Status(); } - - // Return error status of an appropriate type. - static Status OutOfMemory(const std::string &msg, int16_t posix_code = -1) { - return Status(StatusCode::OutOfMemory, msg, posix_code); - } - - static Status KeyError(const std::string &msg) { - return Status(StatusCode::KeyError, msg, -1); - } - - static Status TypeError(const std::string &msg) { - return Status(StatusCode::TypeError, msg, -1); - } - - static Status UnknownError(const std::string &msg) { - return Status(StatusCode::UnknownError, msg, -1); - } - - static Status NotImplemented(const std::string &msg) { - return Status(StatusCode::NotImplemented, msg, -1); - } - - static Status Invalid(const std::string &msg) { - return Status(StatusCode::Invalid, msg, -1); - } - - static Status IOError(const std::string &msg) { - return Status(StatusCode::IOError, msg, -1); - } - - static Status PlasmaObjectExists(const std::string &msg) { - return Status(StatusCode::PlasmaObjectExists, msg, -1); - } - - static Status PlasmaObjectNonexistent(const std::string &msg) { - return Status(StatusCode::PlasmaObjectNonexistent, msg, -1); - } - - static Status PlasmaStoreFull(const std::string &msg) { - return Status(StatusCode::PlasmaStoreFull, msg, -1); - } - - // Returns true iff the status indicates success. - bool ok() const { return (state_ == NULL); } - - bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; } - bool IsKeyError() const { return code() == StatusCode::KeyError; } - bool IsInvalid() const { return code() == StatusCode::Invalid; } - bool IsIOError() const { return code() == StatusCode::IOError; } - bool IsTypeError() const { return code() == StatusCode::TypeError; } - bool IsUnknownError() const { return code() == StatusCode::UnknownError; } - bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; } - // An object with this object ID already exists in the plasma store. - bool IsPlasmaObjectExists() const { - return code() == StatusCode::PlasmaObjectExists; - } - // An object was requested that doesn't exist in the plasma store. - bool IsPlasmaObjectNonexistent() const { - return code() == StatusCode::PlasmaObjectNonexistent; - } - // An object is too large to fit into the plasma store. - bool IsPlasmaStoreFull() const { - return code() == StatusCode::PlasmaStoreFull; - } - - // Return a string representation of this status suitable for printing. - // Returns the string "OK" for success. - std::string ToString() const; - - // Return a string representation of the status code, without the message - // text or posix code information. - std::string CodeAsString() const; - - // Get the POSIX code associated with this Status, or -1 if there is none. - int16_t posix_code() const; - - StatusCode code() const { - return ((state_ == NULL) ? StatusCode::OK - : static_cast(state_[4])); - } - - std::string message() const { - uint32_t length; - memcpy(&length, state_, sizeof(length)); - std::string msg; - msg.append((state_ + 7), length); - return msg; - } - - private: - // OK status has a NULL state_. Otherwise, state_ is a new[] array - // of the following form: - // state_[0..3] == length of message - // state_[4] == code - // state_[5..6] == posix_code - // state_[7..] == message - const char *state_; - - Status(StatusCode code, const std::string &msg, int16_t posix_code); - static const char *CopyState(const char *s); -}; - -inline Status::Status(const Status &s) { - state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); -} - -inline void Status::operator=(const Status &s) { - // The following condition catches both aliasing (when this == &s), - // and the common case where both s and *this are ok. - if (state_ != s.state_) { - delete[] state_; - state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); - } -} - -} // namespace arrow - -#endif // ARROW_STATUS_H_ diff --git a/src/plasma/test/client_tests.cc b/src/plasma/test/client_tests.cc index f255734cb964..5362483592e2 100644 --- a/src/plasma/test/client_tests.cc +++ b/src/plasma/test/client_tests.cc @@ -4,10 +4,12 @@ #include #include -#include "plasma_common.h" -#include "plasma.h" -#include "plasma_protocol.h" -#include "plasma_client.h" +#include "plasma/common.h" +#include "plasma/plasma.h" +#include "plasma/protocol.h" +#include "plasma/client.h" + +using namespace plasma; SUITE(plasma_client_tests); @@ -177,7 +179,7 @@ TEST plasma_wait_for_objects_tests(void) { gettimeofday(&start, NULL); int n; ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS, n)); + WAIT_TIMEOUT_MS, &n)); ASSERT(n == 0); gettimeofday(&end, NULL); float diff_ms = (end.tv_sec - start.tv_sec); @@ -195,7 +197,7 @@ TEST plasma_wait_for_objects_tests(void) { ARROW_CHECK_OK(client1.Seal(oid1)); ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS, n)); + WAIT_TIMEOUT_MS, &n)); ASSERT(n == 1); /* Create and insert an object in client2. */ @@ -204,21 +206,21 @@ TEST plasma_wait_for_objects_tests(void) { ARROW_CHECK_OK(client2.Seal(oid2)); ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS, n)); + WAIT_TIMEOUT_MS, &n)); ASSERT(n == 2); ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS, n)); + WAIT_TIMEOUT_MS, &n)); ASSERT(n == 2); obj_requests[0].type = PLASMA_QUERY_LOCAL; obj_requests[1].type = PLASMA_QUERY_LOCAL; ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS, n)); + WAIT_TIMEOUT_MS, &n)); ASSERT(n == 1); ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST, - WAIT_TIMEOUT_MS, n)); + WAIT_TIMEOUT_MS, &n)); ASSERT(n == 1); ARROW_CHECK_OK(client1.Disconnect()); diff --git a/src/plasma/test/manager_tests.cc b/src/plasma/test/manager_tests.cc index de500c28b3b0..f390c63ee838 100644 --- a/src/plasma/test/manager_tests.cc +++ b/src/plasma/test/manager_tests.cc @@ -13,10 +13,10 @@ #include "io.h" #include "utstring.h" -#include "plasma.h" -#include "plasma_client.h" -#include "plasma_manager.h" -#include "plasma_protocol.h" +#include "plasma/plasma.h" +#include "plasma/client.h" +#include "../plasma_manager.h" +#include "plasma/protocol.h" SUITE(plasma_manager_tests); @@ -53,7 +53,7 @@ typedef struct { ClientConnection *read_conn; /* Connect a new client to the local plasma manager and mock a request to an * object. */ - PlasmaClient *plasma_client; + plasma::PlasmaClient *plasma_client; ClientConnection *client_conn; } plasma_mock; @@ -85,7 +85,7 @@ plasma_mock *init_plasma_mock(plasma_mock *remote_mock) { } /* Connect a new client to the local plasma manager and mock a request to an * object. */ - mock->plasma_client = new PlasmaClient(); + mock->plasma_client = new plasma::PlasmaClient(); ARROW_CHECK_OK(mock->plasma_client->Connect( plasma_store_socket_name, utstring_body(manager_socket_name), 0)); wait_for_pollin(mock->manager_local_fd); @@ -130,13 +130,13 @@ TEST request_transfer_test(void) { event_loop_run(local_mock->loop); int read_fd = get_client_sock(remote_mock->read_conn); std::vector request_data; - ARROW_CHECK_OK( - PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data)); - ObjectID object_id2; + ARROW_CHECK_OK(plasma::PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, + &request_data)); + plasma::ObjectID object_id2; char *address; int port; - ARROW_CHECK_OK( - ReadDataRequest(request_data.data(), &object_id2, &address, &port)); + ARROW_CHECK_OK(plasma::ReadDataRequest( + request_data.data(), request_data.size(), &object_id2, &address, &port)); ASSERT(ObjectID_equal(object_id, object_id2)); free(address); /* Clean up. */ @@ -184,13 +184,13 @@ TEST request_transfer_retry_test(void) { int read_fd = get_client_sock(remote_mock2->read_conn); std::vector request_data; - ARROW_CHECK_OK( - PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data)); - ObjectID object_id2; + ARROW_CHECK_OK(plasma::PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, + &request_data)); + plasma::ObjectID object_id2; char *address; int port; - ARROW_CHECK_OK( - ReadDataRequest(request_data.data(), &object_id2, &address, &port)); + ARROW_CHECK_OK(plasma::ReadDataRequest( + request_data.data(), request_data.size(), &object_id2, &address, &port)); free(address); ASSERT(ObjectID_equal(object_id, object_id2)); /* Clean up. */ @@ -271,24 +271,24 @@ TEST object_notifications_test(void) { ASSERT(!is_local); /* Check that the object is local after receiving an object notification. */ - uint8_t *notification = create_object_info_buffer(&info); + uint8_t *notification = plasma::create_object_info_buffer(&info); int64_t size = *((int64_t *) notification); send(fd[1], notification, sizeof(int64_t) + size, 0); process_object_notification(local_mock->loop, fd[0], local_mock->state, 0); is_local = is_object_local(local_mock->state, object_id); ASSERT(is_local); - free(notification); + delete[] notification; /* Check that the object is not local after receiving a notification about * the object deletion. */ info.is_deletion = true; - notification = create_object_info_buffer(&info); + notification = plasma::create_object_info_buffer(&info); size = *((int64_t *) notification); send(fd[1], notification, sizeof(int64_t) + size, 0); process_object_notification(local_mock->loop, fd[0], local_mock->state, 0); is_local = is_object_local(local_mock->state, object_id); ASSERT(!is_local); - free(notification); + delete[] notification; /* Clean up. */ close(fd[0]); diff --git a/src/plasma/test/run_tests.sh b/src/plasma/test/run_tests.sh index 82f8ff9944d1..01af169d9529 100644 --- a/src/plasma/test/run_tests.sh +++ b/src/plasma/test/run_tests.sh @@ -7,7 +7,6 @@ set -e sleep 1 ./src/plasma/manager_tests killall plasma_store -./src/plasma/serialization_tests # Start the Redis shards. ./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 & diff --git a/src/plasma/test/run_valgrind.sh b/src/plasma/test/run_valgrind.sh index 74531e72161d..8b7e3b0fb6f5 100644 --- a/src/plasma/test/run_valgrind.sh +++ b/src/plasma/test/run_valgrind.sh @@ -7,4 +7,3 @@ set -e sleep 1 valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests killall plasma_store -valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests diff --git a/src/plasma/test/serialization_tests.cc b/src/plasma/test/serialization_tests.cc deleted file mode 100644 index b39094535443..000000000000 --- a/src/plasma/test/serialization_tests.cc +++ /dev/null @@ -1,439 +0,0 @@ -#include "greatest.h" - -#include -#include - -#include "plasma_common.h" -#include "plasma.h" -#include "plasma_io.h" -#include "plasma_protocol.h" - -SUITE(plasma_serialization_tests); - -/** - * Create a temporary file. Needs to be closed by the caller. - * - * @return File descriptor of the file. - */ -int create_temp_file(void) { - static char temp[] = "/tmp/tempfileXXXXXX"; - char file_name[32]; - strncpy(file_name, temp, 32); - return mkstemp(file_name); -} - -/** - * Seek to the beginning of a file and read a message from it. - * - * @param fd File descriptor of the file. - * @param message type Message type that we expect in the file. - * - * @return Pointer to the content of the message. Needs to be freed by the - * caller. - */ -std::vector read_message_from_file(int fd, int message_type) { - /* Go to the beginning of the file. */ - lseek(fd, 0, SEEK_SET); - int64_t type; - std::vector data; - ARROW_CHECK_OK(ReadMessage(fd, &type, data)); - ARROW_CHECK(type == message_type); - return data; -} - -PlasmaObject random_plasma_object(void) { - int random = rand(); - PlasmaObject object; - memset(&object, 0, sizeof(object)); - object.handle.store_fd = random + 7; - object.handle.mmap_size = random + 42; - object.data_offset = random + 1; - object.metadata_offset = random + 2; - object.data_size = random + 3; - object.metadata_size = random + 4; - return object; -} - -TEST plasma_create_request_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - int64_t data_size1 = 42; - int64_t metadata_size1 = 11; - ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaCreateRequest); - ObjectID object_id2; - int64_t data_size2; - int64_t metadata_size2; - ARROW_CHECK_OK(ReadCreateRequest(data.data(), &object_id2, &data_size2, - &metadata_size2)); - ASSERT_EQ(data_size1, data_size2); - ASSERT_EQ(metadata_size1, metadata_size2); - ASSERT(object_id1 == object_id2); - close(fd); - PASS(); -} - -TEST plasma_create_reply_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - PlasmaObject object1 = random_plasma_object(); - ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaCreateReply); - ObjectID object_id2; - PlasmaObject object2; - memset(&object2, 0, sizeof(object2)); - ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2)); - ASSERT(object_id1 == object_id2); - ASSERT(memcmp(&object1, &object2, sizeof(object1)) == 0); - close(fd); - PASS(); -} - -TEST plasma_seal_request_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - unsigned char digest1[kDigestSize]; - memset(&digest1[0], 7, kDigestSize); - ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0])); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaSealRequest); - ObjectID object_id2; - unsigned char digest2[kDigestSize]; - ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0])); - ASSERT(object_id1 == object_id2); - ASSERT(memcmp(&digest1[0], &digest2[0], kDigestSize) == 0); - close(fd); - PASS(); -} - -TEST plasma_seal_reply_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaSealReply); - ObjectID object_id2; - Status s = ReadSealReply(data.data(), &object_id2); - ASSERT(object_id1 == object_id2); - ASSERT(s.IsPlasmaObjectExists()); - close(fd); - PASS(); -} - -TEST plasma_get_request_test(void) { - int fd = create_temp_file(); - ObjectID object_ids[2]; - object_ids[0] = ObjectID::from_random(); - object_ids[1] = ObjectID::from_random(); - int64_t timeout_ms = 1234; - ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaGetRequest); - std::vector object_ids_return; - int64_t timeout_ms_return; - ARROW_CHECK_OK( - ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return)); - ASSERT(object_ids[0] == object_ids_return[0]); - ASSERT(object_ids[1] == object_ids_return[1]); - ASSERT(timeout_ms == timeout_ms_return); - close(fd); - PASS(); -} - -TEST plasma_get_reply_test(void) { - int fd = create_temp_file(); - ObjectID object_ids[2]; - object_ids[0] = ObjectID::from_random(); - object_ids[1] = ObjectID::from_random(); - std::unordered_map plasma_objects; - plasma_objects[object_ids[0]] = random_plasma_object(); - plasma_objects[object_ids[1]] = random_plasma_object(); - ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaGetReply); - ObjectID object_ids_return[2]; - PlasmaObject plasma_objects_return[2]; - memset(&plasma_objects_return, 0, sizeof(plasma_objects_return)); - ARROW_CHECK_OK(ReadGetReply(data.data(), object_ids_return, - &plasma_objects_return[0], 2)); - ASSERT(object_ids[0] == object_ids_return[0]); - ASSERT(object_ids[1] == object_ids_return[1]); - ASSERT(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0], - sizeof(PlasmaObject)) == 0); - ASSERT(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1], - sizeof(PlasmaObject)) == 0); - close(fd); - PASS(); -} - -TEST plasma_release_request_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaReleaseRequest); - ObjectID object_id2; - ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2)); - ASSERT(object_id1 == object_id2); - close(fd); - PASS(); -} - -TEST plasma_release_reply_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaReleaseReply); - ObjectID object_id2; - Status s = ReadReleaseReply(data.data(), &object_id2); - ASSERT(object_id1 == object_id2); - ASSERT(s.IsPlasmaObjectExists()); - close(fd); - PASS(); -} - -TEST plasma_delete_request_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaDeleteRequest); - ObjectID object_id2; - ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2)); - ASSERT(object_id1 == object_id2); - close(fd); - PASS(); -} - -TEST plasma_delete_reply_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - int error1 = PlasmaError_ObjectExists; - ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaDeleteReply); - ObjectID object_id2; - Status s = ReadDeleteReply(data.data(), &object_id2); - ASSERT(object_id1 == object_id2); - ASSERT(s.IsPlasmaObjectExists()); - close(fd); - PASS(); -} - -TEST plasma_status_request_test(void) { - int fd = create_temp_file(); - int64_t num_objects = 2; - ObjectID object_ids[num_objects]; - object_ids[0] = ObjectID::from_random(); - object_ids[1] = ObjectID::from_random(); - ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaStatusRequest); - ObjectID object_ids_read[num_objects]; - ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects)); - ASSERT(object_ids[0] == object_ids_read[0]); - ASSERT(object_ids[1] == object_ids_read[1]); - close(fd); - PASS(); -} - -TEST plasma_status_reply_test(void) { - int fd = create_temp_file(); - ObjectID object_ids[2]; - object_ids[0] = ObjectID::from_random(); - object_ids[1] = ObjectID::from_random(); - int object_statuses[2] = {42, 43}; - ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaStatusReply); - int64_t num_objects = ReadStatusReply_num_objects(data.data()); - ObjectID object_ids_read[num_objects]; - int object_statuses_read[num_objects]; - ARROW_CHECK_OK(ReadStatusReply(data.data(), object_ids_read, - object_statuses_read, num_objects)); - ASSERT(object_ids[0] == object_ids_read[0]); - ASSERT(object_ids[1] == object_ids_read[1]); - ASSERT_EQ(object_statuses[0], object_statuses_read[0]); - ASSERT_EQ(object_statuses[1], object_statuses_read[1]); - close(fd); - PASS(); -} - -TEST plasma_evict_request_test(void) { - int fd = create_temp_file(); - int64_t num_bytes = 111; - ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaEvictRequest); - int64_t num_bytes_received; - ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received)); - ASSERT_EQ(num_bytes, num_bytes_received); - close(fd); - PASS(); -} - -TEST plasma_evict_reply_test(void) { - int fd = create_temp_file(); - int64_t num_bytes = 111; - ARROW_CHECK_OK(SendEvictReply(fd, num_bytes)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaEvictReply); - int64_t num_bytes_received; - ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received)); - ASSERT_EQ(num_bytes, num_bytes_received); - close(fd); - PASS(); -} - -TEST plasma_fetch_request_test(void) { - int fd = create_temp_file(); - ObjectID object_ids[2]; - object_ids[0] = ObjectID::from_random(); - object_ids[1] = ObjectID::from_random(); - ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2)); - std::vector data = - read_message_from_file(fd, MessageType_PlasmaFetchRequest); - std::vector object_ids_read; - ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read)); - ASSERT(object_ids[0] == object_ids_read[0]); - ASSERT(object_ids[1] == object_ids_read[1]); - close(fd); - PASS(); -} - -TEST plasma_wait_request_test(void) { - int fd = create_temp_file(); - const int num_objects_in = 2; - ObjectRequest object_requests_in[num_objects_in] = { - ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}), - ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})}; - const int num_ready_objects_in = 1; - int64_t timeout_ms = 1000; - - ARROW_CHECK_OK(SendWaitRequest(fd, &object_requests_in[0], num_objects_in, - num_ready_objects_in, timeout_ms)); - /* Read message back. */ - std::vector data = - read_message_from_file(fd, MessageType_PlasmaWaitRequest); - int num_ready_objects_out; - int64_t timeout_ms_read; - ObjectRequestMap object_requests_out; - ARROW_CHECK_OK(ReadWaitRequest(data.data(), object_requests_out, - &timeout_ms_read, &num_ready_objects_out)); - ASSERT_EQ(num_objects_in, object_requests_out.size()); - ASSERT_EQ(num_ready_objects_out, num_ready_objects_in); - for (int i = 0; i < num_objects_in; i++) { - const ObjectID &object_id = object_requests_in[i].object_id; - ASSERT_EQ(1, object_requests_out.count(object_id)); - const auto &entry = object_requests_out.find(object_id); - ASSERT(entry != object_requests_out.end()); - ASSERT(entry->second.object_id == object_requests_in[i].object_id); - ASSERT_EQ(entry->second.type, object_requests_in[i].type); - } - close(fd); - PASS(); -} - -TEST plasma_wait_reply_test(void) { - int fd = create_temp_file(); - const int num_objects_in = 2; - /* Create a map with two ObjectRequests in it. */ - ObjectRequestMap objects_in(num_objects_in); - ObjectID id1 = ObjectID::from_random(); - objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local}); - ObjectID id2 = ObjectID::from_random(); - objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent}); - - ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in)); - /* Read message back. */ - std::vector data = - read_message_from_file(fd, MessageType_PlasmaWaitReply); - ObjectRequest objects_out[2]; - int num_objects_out; - ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out)); - ASSERT(num_objects_in == num_objects_out); - for (int i = 0; i < num_objects_out; i++) { - /* Each object request must appear exactly once. */ - ASSERT(1 == objects_in.count(objects_out[i].object_id)); - const auto &entry = objects_in.find(objects_out[i].object_id); - ASSERT(entry != objects_in.end()); - ASSERT(entry->second.object_id == objects_out[i].object_id); - ASSERT(entry->second.status == objects_out[i].status); - } - close(fd); - PASS(); -} - -TEST plasma_data_request_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - const char *address1 = "address1"; - int port1 = 12345; - ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1)); - /* Reading message back. */ - std::vector data = - read_message_from_file(fd, MessageType_PlasmaDataRequest); - ObjectID object_id2; - char *address2; - int port2; - ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2)); - ASSERT(object_id1 == object_id2); - ASSERT(strcmp(address1, address2) == 0); - ASSERT(port1 == port2); - free(address2); - close(fd); - PASS(); -} - -TEST plasma_data_reply_test(void) { - int fd = create_temp_file(); - ObjectID object_id1 = ObjectID::from_random(); - int64_t object_size1 = 146; - int64_t metadata_size1 = 198; - ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1)); - /* Reading message back. */ - std::vector data = - read_message_from_file(fd, MessageType_PlasmaDataReply); - ObjectID object_id2; - int64_t object_size2; - int64_t metadata_size2; - ARROW_CHECK_OK( - ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2)); - ASSERT(object_id1 == object_id2); - ASSERT(object_size1 == object_size2); - ASSERT(metadata_size1 == metadata_size2); - PASS(); -} - -SUITE(plasma_serialization_tests) { - RUN_TEST(plasma_create_request_test); - RUN_TEST(plasma_create_reply_test); - RUN_TEST(plasma_seal_request_test); - RUN_TEST(plasma_seal_reply_test); - RUN_TEST(plasma_get_request_test); - RUN_TEST(plasma_get_reply_test); - RUN_TEST(plasma_release_request_test); - RUN_TEST(plasma_release_reply_test); - RUN_TEST(plasma_delete_request_test); - RUN_TEST(plasma_delete_reply_test); - RUN_TEST(plasma_status_request_test); - RUN_TEST(plasma_status_reply_test); - RUN_TEST(plasma_evict_request_test); - RUN_TEST(plasma_evict_reply_test); - RUN_TEST(plasma_fetch_request_test); - RUN_TEST(plasma_wait_request_test); - RUN_TEST(plasma_wait_reply_test); - RUN_TEST(plasma_data_request_test); - RUN_TEST(plasma_data_reply_test); -} - -GREATEST_MAIN_DEFS(); - -int main(int argc, char **argv) { - GREATEST_MAIN_BEGIN(); - RUN_SUITE(plasma_serialization_tests); - GREATEST_MAIN_END(); -} diff --git a/src/thirdparty/build_thirdparty.sh b/src/thirdparty/build_thirdparty.sh new file mode 100755 index 000000000000..be7095e5b376 --- /dev/null +++ b/src/thirdparty/build_thirdparty.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +set -x + +# Cause the script to exit if a single command fails. +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +if [[ -z "$1" ]]; then + PYTHON_EXECUTABLE=`which python` +else + PYTHON_EXECUTABLE=$1 +fi +echo "Using Python executable $PYTHON_EXECUTABLE." + +# Determine how many parallel jobs to use for make based on the number of cores +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) + echo "Platform is macosx." +else + echo "Unrecognized platform." + exit 1 +fi + +echo "building arrow" +cd $TP_DIR/arrow/cpp +mkdir -p $TP_DIR/arrow/cpp/build +cd $TP_DIR/arrow/cpp/build +export ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-g -O3" \ + -DCMAKE_CXX_FLAGS="-g -O3" \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DARROW_BUILD_TESTS=off \ + -DARROW_HDFS=on \ + -DARROW_PYTHON=on \ + -DARROW_PLASMA=on \ + -DPLASMA_PYTHON=on \ + -DARROW_JEMALLOC=off \ + -DARROW_WITH_BROTLI=off \ + -DARROW_WITH_LZ4=off \ + -DARROW_WITH_SNAPPY=off \ + -DARROW_WITH_ZLIB=off \ + -DARROW_WITH_ZSTD=off \ + .. +make VERBOSE=1 -j$PARALLEL +make install + +echo "installing pyarrow" +cd $TP_DIR/arrow/python +# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can +# find plasma. +ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install +PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig PYARROW_WITH_PLASMA=1 PYARROW_BUNDLE_ARROW_CPP=1 $PYTHON_EXECUTABLE setup.py install diff --git a/src/numbuf/thirdparty/download_thirdparty.sh b/src/thirdparty/download_thirdparty.sh similarity index 76% rename from src/numbuf/thirdparty/download_thirdparty.sh rename to src/thirdparty/download_thirdparty.sh index 9f44f12ddbdd..b9eead06a6df 100755 --- a/src/numbuf/thirdparty/download_thirdparty.sh +++ b/src/thirdparty/download_thirdparty.sh @@ -11,6 +11,6 @@ if [ ! -d $TP_DIR/arrow ]; then git clone https://github.com/apache/arrow/ "$TP_DIR/arrow" fi cd $TP_DIR/arrow -git pull origin master +git fetch origin master -git checkout 8a700ccdad745c250fe5d91a9104e7c2d6364c1b +git checkout dca5d96c7a029c079183e2903db425e486e2deb9 diff --git a/test/component_failures_test.py b/test/component_failures_test.py index 72ab30f470d8..e88f59334492 100644 --- a/test/component_failures_test.py +++ b/test/component_failures_test.py @@ -6,6 +6,8 @@ import time import unittest +import pyarrow as pa + class ComponentFailureTest(unittest.TestCase): @@ -38,8 +40,9 @@ def f(): # Seal the object so the store attempts to notify the worker that the # get has been fulfilled. - ray.worker.global_worker.plasma_client.create(obj_id, 100) - ray.worker.global_worker.plasma_client.seal(obj_id) + ray.worker.global_worker.plasma_client.create( + pa.plasma.ObjectID(obj_id), 100) + ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id)) time.sleep(0.1) # Make sure that nothing has died. @@ -72,8 +75,9 @@ def f(): # Seal the object so the store attempts to notify the worker that the # get has been fulfilled. - ray.worker.global_worker.plasma_client.create(obj_id, 100) - ray.worker.global_worker.plasma_client.seal(obj_id) + ray.worker.global_worker.plasma_client.create( + pa.plasma.ObjectID(obj_id), 100) + ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id)) time.sleep(0.1) # Make sure that nothing has died.