Switch from Bazel to CMake

uber · May 24, 2019 · d91f0b6 · d91f0b6
1 parent 3428ab3
commit d91f0b6
Show file tree

Hide file tree

Showing 14 changed files with 224 additions and 27 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,17 +3,7 @@ language: minimal
 services:
   - docker
 
-before_install:
-  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-  - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-  - sudo apt-get update
-  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-
-install:
-  # TODO(vip): update this image to be Neuropods specific once we open source
-  - docker pull vpanyam/dl_base:latest
-
 script:
   # This builds both the C++ and python versions and runs tests
-  - docker build -f build/neuropods.dockerfile -t neuropods .
+  - docker build -f build/neuropods_cmake.dockerfile -t neuropods .
 
diff --git a/build/neuropods_cmake.dockerfile b/build/neuropods_cmake.dockerfile
@@ -0,0 +1,57 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y gcc-4.9 g++-4.9 cmake wget unzip python-pip libeigen3-dev libboost-python-dev python-numpy
+
+# Use GCC 4.9 to build
+ENV CC=/usr/bin/gcc-4.9 CXX=/usr/bin/g++-4.9
+
+# Download and unpack libtorch and tensorflow
+RUN mkdir -p /usr/deps
+WORKDIR /usr/deps
+RUN wget https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-1.0.0.dev20190318.zip && \
+    unzip libtorch-*.zip && \
+    mkdir tensorflow && \
+    wget https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-1.12.0.tar.gz && \
+    tar -xvf libtensorflow-cpu-linux-x86_64-1.12.0.tar.gz -C tensorflow
+
+# Download and build jsoncpp
+RUN wget https://github.com/open-source-parsers/jsoncpp/archive/0.8.0.tar.gz && \
+    tar -xvf 0.8.0.tar.gz && \
+    mkdir -p /usr/deps/jsoncpp-0.8.0/build && \
+    cd /usr/deps/jsoncpp-0.8.0/build && \
+    cmake -DBUILD_SHARED_LIBS=ON .. && \
+    make && \
+    make install && \
+    ldconfig
+
+# Copy the python code into the image
+RUN mkdir -p /usr/src/source/python /usr/src/source/neuropods/python
+COPY source/python /usr/src/source/python
+COPY source/neuropods/python /usr/src/source/neuropods/python
+
+# Install deps for the python interface
+# (the -f flag tells pip where to find the torch nightly builds)
+WORKDIR /usr/src/source/python
+RUN pip install -U pip setuptools && \
+    python setup.py egg_info && \
+    cat neuropods.egg-info/requires.txt  | sed '/^\[/ d' | paste -sd " " - | xargs pip install -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+
+# Run python tests
+WORKDIR /usr/src/source/python
+RUN python -m unittest discover --verbose neuropods/tests
+
+# Build a wheel + install locally
+RUN python setup.py bdist_wheel && pip install dist/*.whl
+
+# Copy the rest of the code in
+RUN mkdir -p /usr/src
+COPY . /usr/src
+
+# Build native library and run tests
+WORKDIR /usr/src/source/neuropods
+RUN mkdir build && \
+    cd build && \
+    cmake --warn-uninitialized -DCMAKE_PREFIX_PATH="/usr/deps/libtorch;/usr/deps/tensorflow;/usr/deps/jsoncpp-0.8.0" .. && \
+    make && \
+    cd tests && \
+    ctest --output-on-failure
diff --git a/source/neuropods/CMakeLists.txt b/source/neuropods/CMakeLists.txt
@@ -0,0 +1,30 @@
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+project(neuropods)
+set (CMAKE_CXX_STANDARD 11)
+
+# The header include paths are relative to the parent directory
+# (i.e. they are of the form `neuropods/...`)
+include_directories("${CMAKE_SOURCE_DIR}/../")
+
+# Make sure all symbols are defined in all the libraries we build
+set (CMAKE_SHARED_LINKER_FLAGS "-Wl,--no-undefined")
+
+# Link libraries we specify even if there is no compile-time dependency
+# (this is needed in the tests)
+set (CMAKE_EXE_LINKER_FLAGS "-Wl,--no-as-needed")
+
+# Load libtorch. This needs to happen here so both the tests and the
+# backends can find the library
+# TODO(vip): refactor so this doesn't need to be in the root CMakeLists.txt
+find_package(Torch REQUIRED)
+
+# Add the backends
+add_subdirectory(backends)
+
+# Build the main library
+file(GLOB INTERNAL_SOURCES "internal/*.cc")
+add_library( neuropods SHARED ${INTERNAL_SOURCES} "neuropods.cc")
+target_link_libraries(neuropods dl jsoncpp) 
+
+# Add the tests
+add_subdirectory(tests)
diff --git a/source/neuropods/backends/CMakeLists.txt b/source/neuropods/backends/CMakeLists.txt
@@ -0,0 +1,7 @@
+project(neuropod_backends)
+
+# Add each of the backends
+add_subdirectory(python_bridge)
+add_subdirectory(tensorflow)
+add_subdirectory(test_backend)
+add_subdirectory(torchscript)
diff --git a/source/neuropods/backends/python_bridge/CMakeLists.txt b/source/neuropods/backends/python_bridge/CMakeLists.txt
@@ -0,0 +1,14 @@
+project(python_bridge_backend)
+
+# Find Python and Boost Python
+find_package(PythonLibs 2.7 REQUIRED)
+find_package(Boost 1.54 COMPONENTS python REQUIRED)
+
+# Add the includes
+include_directories(${PYTHON_INCLUDE_DIRS})
+include_directories(${Boost_INCLUDE_DIR})
+
+# Build the backend
+file(GLOB SOURCES "*.cc")
+add_library( neuropod_pythonbridge_backend SHARED ${SOURCES})
+target_link_libraries(neuropod_pythonbridge_backend neuropods ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})
diff --git a/source/neuropods/backends/tensorflow/CMakeLists.txt b/source/neuropods/backends/tensorflow/CMakeLists.txt
@@ -0,0 +1,12 @@
+project(tensorflow_backend)
+
+# Find the TF headers and shared object
+find_path(TENSORFLOW_INCLUDE_DIR tensorflow/c/c_api.h)
+find_library(LIBTENSORFLOW libtensorflow.so)
+
+include_directories(${TENSORFLOW_INCLUDE_DIR})
+
+# Build the backend
+file(GLOB SOURCES "*.cc")
+add_library( neuropod_tensorflow_backend SHARED ${SOURCES})
+target_link_libraries(neuropod_tensorflow_backend ${LIBTENSORFLOW} neuropods)
diff --git a/source/neuropods/backends/test_backend/CMakeLists.txt b/source/neuropods/backends/test_backend/CMakeLists.txt
@@ -0,0 +1,6 @@
+project(test_backend)
+
+# Build the backend
+file(GLOB SOURCES "*.cc")
+add_library( neuropod_test_backend SHARED ${SOURCES})
+target_link_libraries(neuropod_test_backend neuropods)
diff --git a/source/neuropods/backends/torchscript/CMakeLists.txt b/source/neuropods/backends/torchscript/CMakeLists.txt
@@ -0,0 +1,6 @@
+project(torchscript_backend)
+
+# Build the backend
+file(GLOB SOURCES "*.cc")
+add_library( neuropod_torchscript_backend SHARED ${SOURCES})
+target_link_libraries(neuropod_torchscript_backend neuropods ${TORCH_LIBRARIES})
diff --git a/source/neuropods/tests/CMakeLists.txt b/source/neuropods/tests/CMakeLists.txt
@@ -0,0 +1,60 @@
+project(tests)
+
+# Begin gtest configuration
+# Based on https://github.com/google/googletest/tree/master/googletest#using-cmake
+# Download and unpack googletest at configure time
+configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt)
+execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
+  RESULT_VARIABLE result
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download )
+if(result)
+  message(FATAL_ERROR "CMake step for googletest failed: ${result}")
+endif()
+execute_process(COMMAND ${CMAKE_COMMAND} --build .
+  RESULT_VARIABLE result
+  WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download )
+if(result)
+  message(FATAL_ERROR "Build step for googletest failed: ${result}")
+endif()
+
+# Add googletest directly to our build. This defines
+# the gtest and gtest_main targets.
+add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src
+                 ${CMAKE_CURRENT_BINARY_DIR}/googletest-build
+                 EXCLUDE_FROM_ALL)
+
+# End gtest configuration
+
+
+# Build the tests
+function(add_neuropod_test test_name additional_deps)
+  add_executable( ${test_name} ${test_name}.cc)
+  target_link_libraries(
+    ${test_name} gtest gmock_main ${additional_deps}
+
+    # Needed so the tests can find the backends to dynamically load
+    "-Wl,-rpath=$ORIGIN/../backends/torchscript/"
+    "-Wl,-rpath=$ORIGIN/../backends/tensorflow/"
+    "-Wl,-rpath=$ORIGIN/../backends/python_bridge/"
+  )
+  add_test( ${test_name} ${test_name} )
+endfunction()
+
+enable_testing()
+
+
+# Unit tests
+add_neuropod_test(test_input_output neuropod_test_backend)
+add_neuropod_test(test_config_utils neuropod_test_backend)
+add_neuropod_test(test_internal_neuropod_tensor neuropod_test_backend)
+
+# Integration tests with different backends
+add_neuropod_test(test_python_bridge neuropod_pythonbridge_backend)
+add_neuropod_test(test_torchscript_backend neuropod_torchscript_backend)
+add_neuropod_test(test_tensorflow_backend neuropod_tensorflow_backend)
+add_neuropod_test(test_default_backend neuropods)
+
+# Eigen conversion test
+find_package (Eigen3 REQUIRED NO_MODULE)
+include_directories ( ${EIGEN3_INCLUDE_DIRS} )
+add_neuropod_test(test_conversions_eigen neuropod_test_backend)
diff --git a/source/neuropods/tests/CMakeLists.txt.in b/source/neuropods/tests/CMakeLists.txt.in
@@ -0,0 +1,15 @@
+# Based on https://github.com/google/googletest/tree/master/googletest#using-cmake
+cmake_minimum_required(VERSION 2.8.2)
+
+project(googletest-download NONE)
+
+include(ExternalProject)
+ExternalProject_Add(googletest
+  URL               https://github.com/google/googletest/archive/release-1.8.0.zip
+  SOURCE_DIR        "${CMAKE_CURRENT_BINARY_DIR}/googletest-src"
+  BINARY_DIR        "${CMAKE_CURRENT_BINARY_DIR}/googletest-build"
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND     ""
+  INSTALL_COMMAND   ""
+  TEST_COMMAND      ""
+)
diff --git a/source/neuropods/tests/test_default_backend.cc b/source/neuropods/tests/test_default_backend.cc
@@ -7,35 +7,35 @@
 TEST(test_default_backend, test_pytorch_addition_model)
 {
     // Test the PyTorch addition model
-    test_addition_model("neuropods/tests/test_data/pytorch_addition_model/");
+    test_addition_model("../../tests/test_data/pytorch_addition_model/");
 }
 
 TEST(test_default_backend, test_pytorch_strings_model)
 {
     // Test the PyTorch strings model
-    test_strings_model("neuropods/tests/test_data/pytorch_strings_model/");
+    test_strings_model("../../tests/test_data/pytorch_strings_model/");
 }
 
 TEST(test_default_backend, test_torchscript_addition_model)
 {
     // Test the TorchScript addition model
-    test_addition_model("neuropods/tests/test_data/torchscript_addition_model/");
+    test_addition_model("../../tests/test_data/torchscript_addition_model/");
 }
 
 TEST(test_default_backend, test_torchscript_strings_model)
 {
     // Test the TorchScript strings model
-    test_strings_model("neuropods/tests/test_data/torchscript_strings_model/");
+    test_strings_model("../../tests/test_data/torchscript_strings_model/");
 }
 
 TEST(test_default_backend, test_tensorflow_addition_model)
 {
     // Test the TensorFlow addition model
-    test_addition_model("neuropods/tests/test_data/tf_addition_model/");
+    test_addition_model("../../tests/test_data/tf_addition_model/");
 }
 
 TEST(test_default_backend, test_tensorflow_strings_model)
 {
     // Test the TensorFlow strings model
-    test_strings_model("neuropods/tests/test_data/tf_strings_model/");
+    test_strings_model("../../tests/test_data/tf_strings_model/");
 }
diff --git a/source/neuropods/tests/test_python_bridge.cc b/source/neuropods/tests/test_python_bridge.cc
@@ -7,35 +7,35 @@
 TEST(test_models, test_pytorch_addition_model)
 {
     // Test the PyTorch addition model using the python bridge
-    test_addition_model("neuropods/tests/test_data/pytorch_addition_model/", "PythonBridge");
+    test_addition_model("../../tests/test_data/pytorch_addition_model/", "PythonBridge");
 }
 
 TEST(test_models, test_pytorch_strings_model)
 {
     // Test the PyTorch strings model using the python bridge
-    test_strings_model("neuropods/tests/test_data/pytorch_strings_model/", "PythonBridge");
+    test_strings_model("../../tests/test_data/pytorch_strings_model/", "PythonBridge");
 }
 
 TEST(test_models, test_torchscript_addition_model)
 {
     // Test the TorchScript addition model using the python bridge
-    test_addition_model("neuropods/tests/test_data/torchscript_addition_model/", "PythonBridge");
+    test_addition_model("../../tests/test_data/torchscript_addition_model/", "PythonBridge");
 }
 
 TEST(test_models, test_torchscript_strings_model)
 {
     // Test the TorchScript strings model using the python bridge
-    test_strings_model("neuropods/tests/test_data/torchscript_strings_model/", "PythonBridge");
+    test_strings_model("../../tests/test_data/torchscript_strings_model/", "PythonBridge");
 }
 
 TEST(test_models, test_tensorflow_addition_model)
 {
     // Test the TensorFlow addition model using the python bridge
-    test_addition_model("neuropods/tests/test_data/tf_addition_model/", "PythonBridge");
+    test_addition_model("../../tests/test_data/tf_addition_model/", "PythonBridge");
 }
 
 TEST(test_models, test_tensorflow_strings_model)
 {
     // Test the TensorFlow strings model using the python bridge
-    test_strings_model("neuropods/tests/test_data/tf_strings_model/", "PythonBridge");
+    test_strings_model("../../tests/test_data/tf_strings_model/", "PythonBridge");
 }
diff --git a/source/neuropods/tests/test_tensorflow_backend.cc b/source/neuropods/tests/test_tensorflow_backend.cc
@@ -7,11 +7,11 @@
 TEST(test_models, test_tensorflow_addition_model)
 {
     // Test the TensorFlow addition model using the native TensorFlow backend
-    test_addition_model("neuropods/tests/test_data/tf_addition_model/", "TensorflowNeuropodBackend");
+    test_addition_model("../../tests/test_data/tf_addition_model/", "TensorflowNeuropodBackend");
 }
 
 TEST(test_models, test_tensorflow_strings_model)
 {
     // Test the TensorFlow strings model using the native TensorFlow backend
-    test_strings_model("neuropods/tests/test_data/tf_strings_model/", "TensorflowNeuropodBackend");
+    test_strings_model("../../tests/test_data/tf_strings_model/", "TensorflowNeuropodBackend");
 }
diff --git a/source/neuropods/tests/test_torchscript_backend.cc b/source/neuropods/tests/test_torchscript_backend.cc
@@ -7,11 +7,11 @@
 TEST(test_models, test_torchscript_addition_model)
 {
     // Test the TorchScript addition model using the native torchscript backend
-    test_addition_model("neuropods/tests/test_data/torchscript_addition_model/", "TorchNeuropodBackend");
+    test_addition_model("../../tests/test_data/torchscript_addition_model/", "TorchNeuropodBackend");
 }
 
 TEST(test_models, test_torchscript_strings_model)
 {
     // Test the TorchScript strings model using the native torchscript backend
-    test_strings_model("neuropods/tests/test_data/torchscript_strings_model/", "TorchNeuropodBackend");
+    test_strings_model("../../tests/test_data/torchscript_strings_model/", "TorchNeuropodBackend");
 }