diff --git a/.travis.yml b/.travis.yml index 3b9c179c919f..e4c3d0d24f04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,11 +57,13 @@ install: - sudo python setup.py install - cd ../../../.. - - cd src/photon/lib/python + - cd src/photon - sudo python setup.py install - - cd ../../../.. + - cd ../.. - - source src/plasma/setup-env.sh + - cd lib/python + - sudo python setup.py install + - cd ../.. script: - python src/common/test/test.py diff --git a/build.sh b/build.sh index 59c1396e80b4..1b4fbb527863 100755 --- a/build.sh +++ b/build.sh @@ -13,16 +13,36 @@ else exit 1 fi -pushd "$ROOT_DIR/src/common" +COMMON_DIR="$ROOT_DIR/src/common" +PLASMA_DIR="$ROOT_DIR/src/plasma" +PHOTON_DIR="$ROOT_DIR/src/photon" + +PYTHON_DIR="$ROOT_DIR/lib/python" +PYTHON_COMMON_DIR="$PYTHON_DIR/common" +PYTHON_PLASMA_DIR="$PYTHON_DIR/plasma" +PYTHON_PHOTON_DIR="$PYTHON_DIR/photon" + +pushd "$COMMON_DIR" make make test popd +cp "$COMMON_DIR/thirdparty/redis-3.2.3/src/redis-server" "$PYTHON_COMMON_DIR/thirdparty/redis-3.2.3/src/" -pushd "$ROOT_DIR/src/plasma" +pushd "$PLASMA_DIR" make make test popd +cp "$PLASMA_DIR/build/plasma_store" "$PYTHON_PLASMA_DIR/build/" +cp "$PLASMA_DIR/build/plasma_manager" "$PYTHON_PLASMA_DIR/build/" +cp "$PLASMA_DIR/build/plasma_client.so" "$PYTHON_PLASMA_DIR/build/" +cp "$PLASMA_DIR/lib/python/plasma.py" "$PYTHON_PLASMA_DIR/lib/python/" -pushd "$ROOT_DIR/src/photon" +pushd "$PHOTON_DIR" make + pushd "$PHOTON_DIR/build" + cmake .. + make install + popd popd +cp "$PHOTON_DIR/build/photon_scheduler" "$PYTHON_PHOTON_DIR/build" +cp "$PHOTON_DIR/photon/libphoton.so" "$PYTHON_PHOTON_DIR/" diff --git a/lib/python/common/__init__.py b/lib/python/common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lib/python/common/thirdparty/redis-3.2.3/src/.gitkeep b/lib/python/common/thirdparty/redis-3.2.3/src/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lib/python/photon/__init__.py b/lib/python/photon/__init__.py new file mode 100644 index 000000000000..27f7c578fa7f --- /dev/null +++ b/lib/python/photon/__init__.py @@ -0,0 +1 @@ +from libphoton import * diff --git a/lib/python/photon/build/.gitkeep b/lib/python/photon/build/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lib/python/plasma/__init__.py b/lib/python/plasma/__init__.py new file mode 100644 index 000000000000..af5061d17824 --- /dev/null +++ b/lib/python/plasma/__init__.py @@ -0,0 +1 @@ +from lib.python.plasma import * diff --git a/lib/python/plasma/build/.gitkeep b/lib/python/plasma/build/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lib/python/plasma/lib/__init__.py b/lib/python/plasma/lib/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lib/python/plasma/lib/python/__init__.py b/lib/python/plasma/lib/python/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/lib/python/ray/__init__.py b/lib/python/ray/__init__.py index 9847505c3611..330705ad8f97 100644 --- a/lib/python/ray/__init__.py +++ b/lib/python/ray/__init__.py @@ -1,5 +1,5 @@ # Ray version string -__version__ = "0.1" +__version__ = "0.01" import ctypes # Windows only diff --git a/lib/python/setup.py b/lib/python/setup.py index 77a4238dfa0c..9cc416a3864a 100644 --- a/lib/python/setup.py +++ b/lib/python/setup.py @@ -1,17 +1,25 @@ -import sys +import subprocess -from setuptools import setup, Extension, find_packages -import setuptools +from setuptools import setup, find_packages +import setuptools.command.install as _install -# because of relative paths, this must be run from inside ray/lib/python/ +class install(_install.install): + def run(self): + subprocess.check_call(["../../build.sh"]) + # Calling _install.install.run(self) does not fetch required packages and + # instead performs an old-style install. See command/install.py in + # setuptools. So, calling do_egg_install() manually here. + self.do_egg_install() -setup( - name = "ray", - version = "0.1.dev0", - use_2to3=True, - packages=find_packages(), - package_data = { - "ray": ["libraylib.so", "scheduler", "objstore"] - }, - zip_safe=False -) +setup(name="ray", + version="0.0.1", + packages=find_packages(), + package_data={"common": ["thirdparty/redis-3.2.3/src/redis-server"], + "plasma": ["build/plasma_store", + "build/plasma_manager", + "build/plasma_client.so"], + "photon": ["build/photon_scheduler", + "libphoton.so"]}, + cmdclass={"install": install}, + include_package_data=True, + zip_safe=False) diff --git a/src/photon/CMakeLists.txt b/src/photon/CMakeLists.txt new file mode 100644 index 000000000000..988137371dd5 --- /dev/null +++ b/src/photon/CMakeLists.txt @@ -0,0 +1,73 @@ +cmake_minimum_required(VERSION 2.8) + +project(photon) + +if(NOT APPLE) + find_package(PythonInterp REQUIRED) + find_package(PythonLibs REQUIRED) + set(CUSTOM_PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE}) +else() + find_program(CUSTOM_PYTHON_EXECUTABLE python) + message("-- Found Python program: ${CUSTOM_PYTHON_EXECUTABLE}") + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "import sys; print 'python' + sys.version[0:3]" + OUTPUT_VARIABLE PYTHON_LIBRARY_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "import sys; print sys.exec_prefix" + OUTPUT_VARIABLE PYTHON_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) + FIND_LIBRARY(PYTHON_LIBRARIES + NAMES ${PYTHON_LIBRARY_NAME} + HINTS "${PYTHON_PREFIX}" + PATH_SUFFIXES "lib" "libs" + NO_DEFAULT_PATH) + execute_process(COMMAND ${CUSTOM_PYTHON_EXECUTABLE} -c + "from distutils.sysconfig import *; print get_python_inc()" + OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE) + if(PYTHON_LIBRARIES AND PYTHON_INCLUDE_DIRS) + SET(PYTHONLIBS_FOUND TRUE) + message("-- Found PythonLibs: " ${PYTHON_LIBRARIES}) + message("-- -- Used custom search path") + else() + find_package(PythonLibs REQUIRED) + message("-- -- Used find_package(PythonLibs)") + endif() +endif() + +if(APPLE) + SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif(APPLE) + +include_directories("${PYTHON_INCLUDE_DIRS}") + +set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} --std=c99 -Werror") + +if (UNIX AND NOT APPLE) + link_libraries(rt) +endif() + +set(PHOTON_CLIENT_LIB "${CMAKE_SOURCE_DIR}/build/photon_client.a" CACHE STRING + "Path to photon_client.a") + +set(COMMON_LIB "${CMAKE_SOURCE_DIR}/../common/build/libcommon.a" CACHE STRING + "Path to libcommon.a") + +include_directories("${CMAKE_SOURCE_DIR}/") +include_directories("${CMAKE_SOURCE_DIR}/../") +include_directories("${CMAKE_SOURCE_DIR}/../common/") +include_directories("${CMAKE_SOURCE_DIR}/../common/thirdparty/") +include_directories("${CMAKE_SOURCE_DIR}/../common/lib/python/") + +add_library(photon SHARED + photon_extension.c + ../common/lib/python/common_extension.c) + +get_filename_component(PYTHON_SHARED_LIBRARY ${PYTHON_LIBRARIES} NAME) +if(APPLE) + add_custom_command(TARGET photon + POST_BUILD COMMAND + ${CMAKE_INSTALL_NAME_TOOL} -change ${PYTHON_SHARED_LIBRARY} ${PYTHON_LIBRARIES} libphoton.so) +endif(APPLE) + +target_link_libraries(photon ${PHOTON_CLIENT_LIB} ${COMMON_LIB} ${PYTHON_LIBRARIES}) + +install(TARGETS photon DESTINATION ${CMAKE_SOURCE_DIR}/photon) diff --git a/src/photon/lib/python/photon_extension.c b/src/photon/lib/python/photon_extension.c deleted file mode 100644 index 4aa199887627..000000000000 --- a/src/photon/lib/python/photon_extension.c +++ /dev/null @@ -1,140 +0,0 @@ -#include - -#include "common_extension.h" -#include "photon_client.h" -#include "task.h" - -PyObject *PhotonError; - -// clang-format off -typedef struct { - PyObject_HEAD - photon_conn *photon_connection; -} PyPhotonClient; -// clang-format on - -static int PyPhotonClient_init(PyPhotonClient *self, PyObject *args, - PyObject *kwds) { - char *socket_name; - if (!PyArg_ParseTuple(args, "s", &socket_name)) { - return -1; - } - self->photon_connection = photon_connect(socket_name); - return 0; -} - -static void PyPhotonClient_dealloc(PyPhotonClient *self) { - free(((PyPhotonClient *)self)->photon_connection); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static PyObject *PyPhotonClient_submit(PyObject *self, PyObject *args) { - PyObject *py_task; - if (!PyArg_ParseTuple(args, "O", &py_task)) { - return NULL; - } - photon_submit(((PyPhotonClient *)self)->photon_connection, - ((PyTask *)py_task)->spec); - Py_RETURN_NONE; -} - -// clang-format off -static PyObject *PyPhotonClient_get_task(PyObject *self) { - task_spec *task_spec; - /* Drop the global interpreter lock while we get a task because - * photon_get_task may block for a long time. */ - Py_BEGIN_ALLOW_THREADS - task_spec = photon_get_task(((PyPhotonClient *)self)->photon_connection); - Py_END_ALLOW_THREADS - return PyTask_make(task_spec); -} -// clang-format on - -static PyMethodDef PyPhotonClient_methods[] = { - {"submit", (PyCFunction)PyPhotonClient_submit, METH_VARARGS, - "Submit a task to the local scheduler."}, - {"get_task", (PyCFunction)PyPhotonClient_get_task, METH_NOARGS, - "Get a task from the local scheduler."}, - {NULL} /* Sentinel */ -}; - -static PyTypeObject PyPhotonClientType = { - PyObject_HEAD_INIT(NULL) 0, /* ob_size */ - "photon.PhotonClient", /* tp_name */ - sizeof(PyPhotonClient), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PyPhotonClient_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "PhotonClient object", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PyPhotonClient_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PyPhotonClient_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - -static PyMethodDef photon_methods[] = { - {"check_simple_value", check_simple_value, METH_VARARGS, - "Should the object be passed by value?"}, - {NULL} /* Sentinel */ -}; - -#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ -#define PyMODINIT_FUNC void -#endif - -PyMODINIT_FUNC initphoton(void) { - PyObject *m; - - if (PyType_Ready(&PyTaskType) < 0) - return; - - if (PyType_Ready(&PyObjectIDType) < 0) - return; - - if (PyType_Ready(&PyPhotonClientType) < 0) - return; - - m = Py_InitModule3("photon", photon_methods, - "A module for the local scheduler."); - - Py_INCREF(&PyTaskType); - PyModule_AddObject(m, "Task", (PyObject *)&PyTaskType); - - Py_INCREF(&PyObjectIDType); - PyModule_AddObject(m, "ObjectID", (PyObject *)&PyObjectIDType); - - Py_INCREF(&PyPhotonClientType); - PyModule_AddObject(m, "PhotonClient", (PyObject *)&PyPhotonClientType); - - char photon_error[] = "photon.error"; - PhotonError = PyErr_NewException(photon_error, NULL, NULL); - Py_INCREF(PhotonError); - PyModule_AddObject(m, "photon_error", PhotonError); -} diff --git a/src/photon/lib/python/setup.py b/src/photon/lib/python/setup.py deleted file mode 100644 index c3d62eb956f5..000000000000 --- a/src/photon/lib/python/setup.py +++ /dev/null @@ -1,14 +0,0 @@ -from setuptools import setup, find_packages, Extension - -photon_module = Extension("photon", - sources=["photon_extension.c", "../../../common/lib/python/common_extension.c"], - include_dirs=["../../", "../../..", "../../../common/", - "../../../common/thirdparty/", - "../../../common/lib/python"], - extra_objects=["../../build/photon_client.a", "../../../common/build/libcommon.a"], - extra_compile_args=["--std=c99", "-Werror"]) - -setup(name="Photon", - version="0.1", - description="Photon library for Ray", - ext_modules=[photon_module]) diff --git a/src/photon/photon/__init__.py b/src/photon/photon/__init__.py new file mode 100644 index 000000000000..27f7c578fa7f --- /dev/null +++ b/src/photon/photon/__init__.py @@ -0,0 +1 @@ +from libphoton import * diff --git a/src/photon/photon_extension.c b/src/photon/photon_extension.c new file mode 100644 index 000000000000..e7ddded15e9b --- /dev/null +++ b/src/photon/photon_extension.c @@ -0,0 +1,141 @@ +#include + +#include "common_extension.h" +#include "photon_client.h" +#include "task.h" + +PyObject *PhotonError; + +// clang-format off +typedef struct { + PyObject_HEAD + photon_conn *photon_connection; +} PyPhotonClient; +// clang-format on + +static int PyPhotonClient_init(PyPhotonClient *self, + PyObject *args, + PyObject *kwds) { + char *socket_name; + if (!PyArg_ParseTuple(args, "s", &socket_name)) { + return -1; + } + self->photon_connection = photon_connect(socket_name); + return 0; +} + +static void PyPhotonClient_dealloc(PyPhotonClient *self) { + free(((PyPhotonClient *) self)->photon_connection); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject *PyPhotonClient_submit(PyObject *self, PyObject *args) { + PyObject *py_task; + if (!PyArg_ParseTuple(args, "O", &py_task)) { + return NULL; + } + photon_submit(((PyPhotonClient *) self)->photon_connection, + ((PyTask *) py_task)->spec); + Py_RETURN_NONE; +} + +// clang-format off +static PyObject *PyPhotonClient_get_task(PyObject *self) { + task_spec *task_spec; + /* Drop the global interpreter lock while we get a task because + * photon_get_task may block for a long time. */ + Py_BEGIN_ALLOW_THREADS + task_spec = photon_get_task(((PyPhotonClient *) self)->photon_connection); + Py_END_ALLOW_THREADS + return PyTask_make(task_spec); +} +// clang-format on + +static PyMethodDef PyPhotonClient_methods[] = { + {"submit", (PyCFunction) PyPhotonClient_submit, METH_VARARGS, + "Submit a task to the local scheduler."}, + {"get_task", (PyCFunction) PyPhotonClient_get_task, METH_NOARGS, + "Get a task from the local scheduler."}, + {NULL} /* Sentinel */ +}; + +static PyTypeObject PyPhotonClientType = { + PyObject_HEAD_INIT(NULL) 0, /* ob_size */ + "photon.PhotonClient", /* tp_name */ + sizeof(PyPhotonClient), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor) PyPhotonClient_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "PhotonClient object", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PyPhotonClient_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc) PyPhotonClient_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +static PyMethodDef photon_methods[] = { + {"check_simple_value", check_simple_value, METH_VARARGS, + "Should the object be passed by value?"}, + {NULL} /* Sentinel */ +}; + +#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ +#define PyMODINIT_FUNC void +#endif + +PyMODINIT_FUNC initlibphoton(void) { + PyObject *m; + + if (PyType_Ready(&PyTaskType) < 0) + return; + + if (PyType_Ready(&PyObjectIDType) < 0) + return; + + if (PyType_Ready(&PyPhotonClientType) < 0) + return; + + m = Py_InitModule3("libphoton", photon_methods, + "A module for the local scheduler."); + + Py_INCREF(&PyTaskType); + PyModule_AddObject(m, "Task", (PyObject *) &PyTaskType); + + Py_INCREF(&PyObjectIDType); + PyModule_AddObject(m, "ObjectID", (PyObject *) &PyObjectIDType); + + Py_INCREF(&PyPhotonClientType); + PyModule_AddObject(m, "PhotonClient", (PyObject *) &PyPhotonClientType); + + char photon_error[] = "photon.error"; + PhotonError = PyErr_NewException(photon_error, NULL, NULL); + Py_INCREF(PhotonError); + PyModule_AddObject(m, "photon_error", PhotonError); +} diff --git a/src/photon/setup.py b/src/photon/setup.py new file mode 100644 index 000000000000..863649863d3b --- /dev/null +++ b/src/photon/setup.py @@ -0,0 +1,23 @@ +import subprocess + +from setuptools import setup, find_packages +import setuptools.command.install as _install + +class install(_install.install): + def run(self): + subprocess.check_call(["make"]) + subprocess.check_call(["cmake", ".."], cwd="build") + subprocess.check_call(["make", "install"], cwd="build") + # Calling _install.install.run(self) does not fetch required packages and + # instead performs an old-style install. See command/install.py in + # setuptools. So, calling do_egg_install() manually here. + self.do_egg_install() + +setup(name="photon", + version="0.1", + description="Photon library for Ray", + packages=find_packages(), + package_data={"photon": ["libphoton.so"]}, + cmdclass={"install": install}, + include_package_data=True, + zip_safe=False)