Change Ray's build dir, as it's only for Python now

.travis.yml

@@ -57,7 +57,7 @@ matrix:
         - ./.travis/install-ray.sh
 
       script:
-        - cd python/ray/core
+        - cd build/ray/core
         - bash ../../../src/common/test/run_valgrind.sh
         - bash ../../../src/plasma/test/run_valgrind.sh
         - bash ../../../src/local_scheduler/test/run_valgrind.sh
@@ -155,7 +155,7 @@ install:
   - ./.travis/install-ray.sh
   - ./.travis/install-cython-examples.sh
 
-  - cd python/ray/core
+  - cd build/ray/core
   - bash ../../../src/ray/test/run_gcs_tests.sh
   # Raylet tests.
   - bash ../../../src/ray/test/run_object_manager_tests.sh

build.sh

@@ -28,7 +28,11 @@ else
 fi
 
 # Now we build everything.
-pushd "$ROOT_DIR/python/ray/core"
+BUILD_DIR="$ROOT_DIR/build/ray/core"
+if [ ! -d "${BUILD_DIR}" ]; then
+  mkdir -p ${BUILD_DIR}
+fi
+pushd "$BUILD_DIR"
   # We use these variables to set PKG_CONFIG_PATH, which is important so that
   # in cmake, pkg-config can find plasma.
   TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg
@@ -53,4 +57,4 @@ pushd "$ROOT_DIR/python/ray/core"
 popd
 
 # Move stuff from Arrow to Ray.
-cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $ROOT_DIR/python/ray/core/src/plasma/
+cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $ROOT_DIR/build/ray/core/src/plasma/

doc/source/development.rst

@@ -29,7 +29,7 @@ helpful.
 
    .. code-block:: shell
 
-     cd ray/python/ray/core
+     cd ray/build/ray/core
      make -j8
 
 2. **Starting processes in a debugger:** When processes are crashing, it is

python/setup.py

@@ -15,28 +15,29 @@
 # MANIFEST.in gets applied at the very beginning when setup.py runs
 # before these files have been created, so we have to move the files
 # manually.
-ray_files = [
+ray_files_generated = [
     "ray/core/src/common/thirdparty/redis/src/redis-server",
     "ray/core/src/common/redis_module/libray_redis_module.so",
     "ray/core/src/plasma/plasma_store", "ray/core/src/plasma/plasma_manager",
     "ray/core/src/local_scheduler/local_scheduler",
     "ray/core/src/local_scheduler/liblocal_scheduler_library.so",
     "ray/core/src/global_scheduler/global_scheduler",
-    "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet",
-    "ray/WebUI.ipynb"
+    "ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet"
 ]
+ray_files = ["ray/WebUI.ipynb"]
 
 optional_ray_files = []
+optional_ray_files_generated = []
 
-ray_ui_files = [
+ray_ui_files_generated = [
     "ray/core/src/catapult_files/index.html",
     "ray/core/src/catapult_files/trace_viewer_full.html"
 ]
 
 ray_autoscaler_files = ["ray/autoscaler/aws/example-full.yaml"]
 
 if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
-    ray_files += [
+    ray_files_generated += [
         "ray/core/src/credis/build/src/libmember.so",
         "ray/core/src/credis/build/src/libmaster.so",
         "ray/core/src/credis/redis/src/redis-server"
@@ -45,9 +46,9 @@
 # The UI files are mandatory if the INCLUDE_UI environment variable equals 1.
 # Otherwise, they are optional.
 if "INCLUDE_UI" in os.environ and os.environ["INCLUDE_UI"] == "1":
-    ray_files += ray_ui_files
+    ray_files_generated += ray_ui_files_generated
 else:
-    optional_ray_files += ray_ui_files
+    optional_ray_files_generated += ray_ui_files_generated
 
 optional_ray_files += ray_autoscaler_files
 
@@ -65,6 +66,17 @@ def run(self):
         # TODO(rkn): Fix this.
         subprocess.check_call(["../build.sh", sys.executable])
 
+        # Copy over the ray files generated after compilation.
+        for filename in ray_files_generated:
+            source = "../build/" + filename
+            build_lib_destination = os.path.join(self.build_lib, filename)
+            self.move_file(source, build_lib_destination)
+            self.move_file(source, filename)
+        # Copy over the existing ray files.
+        for filename in ray_files:
+            build_lib_destination = os.path.join(self.build_lib, filename)
+            self.move_file(filename, build_lib_destination)
+
         # We also need to install pyarrow along with Ray, so make sure that the
         # relevant non-Python pyarrow files get copied.
         pyarrow_files = [
@@ -73,32 +85,37 @@ def run(self):
             if not os.path.isdir(
                 os.path.join("ray/pyarrow_files/pyarrow", filename))
         ]
+        for filename in pyarrow_files:
+            build_lib_destination = os.path.join(self.build_lib, filename)
+            self.move_file(filename, build_lib_destination)
 
-        files_to_include = ray_files + pyarrow_files
-
-        for filename in files_to_include:
-            self.move_file(filename)
         # Copy over the autogenerated flatbuffer Python bindings.
         generated_python_directory = "ray/core/generated"
         for filename in os.listdir(generated_python_directory):
             if filename[-3:] == ".py":
-                self.move_file(
-                    os.path.join(generated_python_directory, filename))
+                source = os.path.join(generated_python_directory, filename)
+                build_lib_destination = os.path.join(self.build_lib, source)
+                self.move_file(source, build_lib_destination)
 
-        # Try to copy over the optional files.
-        for filename in optional_ray_files:
+        # Try to copy over the generated optional files.
+        for filename in optional_ray_files_generated:
             try:
-                self.move_file(filename)
+                source = "../build/" + filename
+                build_lib_destination = os.path.join(self.build_lib, filename)
+                self.move_file(source, build_lib_destination)
+                self.move_file(source, filename)
             except Exception as e:
                 print("Failed to copy optional file {}. This is ok."
                       .format(filename))
+        # Copy over the existing optional files.
+        for filename in optional_ray_files:
+            build_lib_destination = os.path.join(self.build_lib, filename)
+            self.move_file(filename, build_lib_destination)
 
-    def move_file(self, filename):
+    def move_file(self, source, destination):
         # TODO(rkn): This feels very brittle. It may not handle all cases. See
         # https://github.com/apache/arrow/blob/master/python/setup.py for an
         # example.
-        source = filename
-        destination = os.path.join(self.build_lib, filename)
         # Create the target directory if it doesn't already exist.
         parent_directory = os.path.dirname(destination)
         if not os.path.exists(parent_directory):

src/common/CMakeLists.txt

@@ -32,7 +32,7 @@ add_custom_command(
 add_custom_target(gen_common_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
 
 # Generate Python bindings for the flatbuffers objects.
-set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../python/ray/core/generated/)
 add_custom_command(
   TARGET gen_common_fbs
   COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}

src/ray/gcs/CMakeLists.txt

@@ -20,7 +20,7 @@ add_custom_command(
 add_custom_target(gen_gcs_fbs DEPENDS ${GCS_FBS_OUTPUT_FILES})
 
 # Generate Python bindings for the flatbuffers objects.
-set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../python/ray/core/generated/)
 add_custom_command(
   TARGET gen_gcs_fbs
   COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${GCS_FBS_SRC}

src/ray/test/run_gcs_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build/ray/core
 
 # Cause the script to exit if a single command fails.
 set -e

src/ray/test/run_object_manager_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build/ray/core
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then
   exit 1
 fi
 
-CORE_DIR="$RAY_ROOT/python/ray/core"
+CORE_DIR="$RAY_ROOT/build/ray/core"
 REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
 REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
 STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"

src/ray/test/run_object_manager_valgrind.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build/ray/core
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then
   exit 1
 fi
 
-CORE_DIR="$RAY_ROOT/python/ray/core"
+CORE_DIR="$RAY_ROOT/build/ray/core"
 REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
 REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
 STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"

src/ray/test/start_raylet.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build/ray/core
 
 # Cause the script to exit if a single command fails.
 set -e

src/ray/test/start_raylets.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build/ray/core
 
 # Cause the script to exit if a single command fails.
 set -e

thirdparty/scripts/build_credis.sh

@@ -34,11 +34,11 @@ if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
         make -j
       popd
 
-      mkdir -p $ROOT_DIR/python/ray/core/src/credis/redis/src/
-      cp redis/src/redis-server $ROOT_DIR/python/ray/core/src/credis/redis/src/redis-server
-      mkdir -p $ROOT_DIR/python/ray/core/src/credis/build/src/
-      cp build/src/libmaster.so $ROOT_DIR/python/ray/core/src/credis/build/src/libmaster.so
-      cp build/src/libmember.so $ROOT_DIR/python/ray/core/src/credis/build/src/libmember.so
+      mkdir -p $ROOT_DIR/build/ray/core/src/credis/redis/src/
+      cp redis/src/redis-server $ROOT_DIR/build/ray/core/src/credis/redis/src/redis-server
+      mkdir -p $ROOT_DIR/build/ray/core/src/credis/build/src/
+      cp build/src/libmaster.so $ROOT_DIR/build/ray/core/src/credis/build/src/libmaster.so
+      cp build/src/libmember.so $ROOT_DIR/build/ray/core/src/credis/build/src/libmember.so
     popd
   fi
 fi

thirdparty/scripts/build_ui.sh

@@ -11,7 +11,7 @@ CATAPULT_COMMIT=18cd334755701cf0c3b90b7172126c686d2eb787
 CATAPULT_HOME=$TP_DIR/pkg/catapult
 VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer
 
-CATAPULT_FILES=$TP_DIR/../python/ray/core/src/catapult_files
+CATAPULT_FILES=$TP_DIR/../build/ray/core/src/catapult_files
 
 # This is where we will copy the files that need to be packaged with the wheels.
 mkdir -p $CATAPULT_FILES