Add install target. (#495)

* Add install target.

* Export `dmlc` in CMake.
* Use GNUInstallDir instead.

* Add config target.

* Move filesys.h into io.h.

* Replace the rest of filesys.h.

* Address linter.

* Address linter.

* Improve Travis CI on Mac target

.travis.yml

@@ -1,6 +1,8 @@
 group: travis_latest
 language: python
 
+osx_image: xcode10.1
+
 matrix:
   include:
   - env: TASK=lint
@@ -32,6 +34,11 @@ addons:
       - unzip
       - gcc-4.8
       - g++-4.8
+  homebrew:
+    packages:
+      - gcc@7
+      - python3
+    update: true
 
 before_install:
   - export TRAVIS=scripts/travis

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.2)
 
-project(dmlc C CXX)
+project(dmlc VERSION 0.3 LANGUAGES C CXX)
 
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
   include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake)
@@ -11,7 +11,6 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_LOCAL}/Modules)
 
 include(CheckCXXSymbolExists)
 include(cmake/Utils.cmake)
-#include(cmake/dmlccore.cmake)
 
 # Options
 dmlccore_option(USE_HDFS "Build with HDFS support" OFF)
@@ -24,7 +23,7 @@ dmlccore_option(GOOGLE_TEST "Build google tests" OFF)
 # include path
 set(INCLUDE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/include")
 set(INCLUDE_DMLC_DIR "${INCLUDE_ROOT}/dmlc")
-include_directories("${INCLUDE_ROOT}")
+# include_directories("${INCLUDE_ROOT}")
 
 set(dmlccore_LINKER_LIBS "")
 # HDFS configurations
@@ -187,6 +186,9 @@ endif()
 
 add_library(dmlc ${SOURCE})
 target_link_libraries(dmlc ${dmlccore_LINKER_LIBS})
+target_include_directories(dmlc PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>)
 target_compile_definitions(dmlc PRIVATE -D_XOPEN_SOURCE=700
   -D_POSIX_SOURCE -D_POSIX_C_SOURCE=200809L -D_DARWIN_C_SOURCE)
 
@@ -199,10 +201,36 @@ if(INSTALL_INCLUDE_DIR)
 endif()
 
 # ---[ Install the archive static lib and header files
-install(TARGETS dmlc ARCHIVE DESTINATION lib${LIB_SUFFIX} LIBRARY DESTINATION lib${LIB_SUFFIX})
+include(GNUInstallDirs)
+install(TARGETS dmlc
+  EXPORT DMLCTargets
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(EXPORT DMLCTargets
+  FILE DMLCTargets.cmake
+  NAMESPACE dmlc::
+  EXPORT_LINK_INTERFACE_LIBRARIES
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc)
+
 install(DIRECTORY include DESTINATION .)
 install(DIRECTORY doc DESTINATION .)
 
+# ---[ Package configurations
+include(CMakePackageConfigHelpers)
+configure_package_config_file(
+  ${CMAKE_SOURCE_DIR}/cmake/dmlc-config.cmake.in
+  ${CMAKE_BINARY_DIR}/cmake/dmlc-config.cmake
+  INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc)
+write_basic_package_version_file(
+  ${CMAKE_BINARY_DIR}/cmake/dmlc-config-version.cmake
+  VERSION ${DMLC_VERSION}
+  COMPATIBILITY AnyNewerVersion)
+install(
+  FILES
+  ${CMAKE_BINARY_DIR}/cmake/dmlc-config.cmake
+  ${CMAKE_BINARY_DIR}/cmake/dmlc-config-version.cmake
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc)
+
 # ---[ Linter target
 if(MSVC)
   find_package(PythonInterp)
@@ -216,4 +244,3 @@ if(GOOGLE_TEST)
   include(CTest)
   add_subdirectory(test/unittest)
 endif()
-

cmake/dmlc-config.cmake.in

@@ -0,0 +1,5 @@
+@PACKAGE_INIT@
+
+if(NOT TARGET dmlc::dmlc)
+  include(${CMAKE_CURRENT_LIST_DIR}/DMLCTargets.cmake)
+endif()

doc/build.md

@@ -0,0 +1,16 @@
+Using dmlc-core with CMake
+==========================
+dmlc defines a exported CMake target which can be used by `find_package` command.
+
+For example, if you have a simple C++ project that contains only a main.cc file,
+which uses dmlc-core as dependency, the CMakeLists.txt for your project can be
+defined as follow:
+
+``` cmake
+project(demo)
+cmake_minimum_required(VERSION 3.2)
+
+find_package(dmlc REQUIRED)
+add_executable(demo main.cc)
+target_link_libraries(demo dmlc::dmlc)
+```

doc/index.md

@@ -4,6 +4,7 @@ DMLC Core contains common codebase to help us build machine learning toolkits ea
 
 Contents
 --------
+* [Using dmlc-core with CMake](build.md)
 * [Parameter Structure for Machine Learning](parameter.md)
 * [Doxygen C++ API Reference](https://dmlc-core.readthedocs.org/en/latest/doxygen)
 

include/dmlc/filesystem.h

@@ -8,6 +8,7 @@
 #define DMLC_FILESYSTEM_H_
 
 #include <dmlc/logging.h>
+#include <dmlc/io.h>
 #include <algorithm>
 #include <string>
 #include <vector>
@@ -25,8 +26,6 @@
 #include <sys/types.h>
 #endif  // _WIN32
 
-#include "../../src/io/filesys.h"
-
 namespace dmlc {
 
 /*!
@@ -152,35 +151,7 @@ class TemporaryDirectory {
    * \brief Delete a directory recursively, along with sub-directories and files.
    * \param path String representation of path. It must refer to a directory.
    */
-  inline void RecursiveDelete(const std::string& path) {
-    io::URI uri(path.c_str());
-    io::FileSystem* fs = io::FileSystem::GetInstance(uri);
-    std::vector<io::FileInfo> file_list;
-    fs->ListDirectory(uri, &file_list);
-    for (io::FileInfo info : file_list) {
-      CHECK(!IsSymlink(info.path.name))
-        << "Symlink not supported in TemporaryDirectory";
-      if (info.type == io::FileType::kDirectory) {
-        RecursiveDelete(info.path.name);
-      } else {
-        CHECK_EQ(std::remove(info.path.name.c_str()), 0)
-          << "Couldn't remove file " << info.path.name;
-      }
-    }
-#if _WIN32
-    const bool rmdir_success = (RemoveDirectoryA(path.c_str()) != 0);
-#else
-    const bool rmdir_success = (rmdir(path.c_str()) == 0);
-#endif
-    if (rmdir_success) {
-      if (verbose_) {
-        LOG(INFO) << "Successfully deleted temporary directory " << path;
-      }
-    } else {
-      LOG(FATAL) << "~TemporaryDirectory(): "
-                 << "Could not remove temporary directory " << path;
-    }
-  }
+  void RecursiveDelete(const std::string& path);
 };
 
 }  // namespace dmlc

include/dmlc/io.h

@@ -7,6 +7,7 @@
 #define DMLC_IO_H_
 #include <cstdio>
 #include <string>
+#include <cstring>
 #include <vector>
 #include <istream>
 #include <ostream>
@@ -518,5 +519,117 @@ inline int istream::InBuf::underflow() {
   }
 }
 #endif
+
+namespace io {
+/*! \brief common data structure for URI */
+struct URI {
+  /*! \brief protocol */
+  std::string protocol;
+  /*!
+   * \brief host name, namenode for HDFS, bucket name for s3
+   */
+  std::string host;
+  /*! \brief name of the path */
+  std::string name;
+  /*! \brief enable default constructor */
+  URI(void) {}
+  /*!
+   * \brief construct from URI string
+   */
+  explicit URI(const char *uri) {
+    const char *p = std::strstr(uri, "://");
+    if (p == NULL) {
+      name = uri;
+    } else {
+      protocol = std::string(uri, p - uri + 3);
+      uri = p + 3;
+      p = std::strchr(uri, '/');
+      if (p == NULL) {
+        host = uri; name = '/';
+      } else {
+        host = std::string(uri, p - uri);
+        name = p;
+      }
+    }
+  }
+  /*! \brief string representation */
+  inline std::string str(void) const {
+    return protocol + host + name;
+  }
+};
+
+/*! \brief type of file */
+enum FileType {
+  /*! \brief the file is file */
+  kFile,
+  /*! \brief the file is directory */
+  kDirectory
+};
+
+/*! \brief use to store file information */
+struct FileInfo {
+  /*! \brief full path to the file */
+  URI path;
+  /*! \brief the size of the file */
+  size_t size;
+  /*! \brief the type of the file */
+  FileType type;
+  /*! \brief default constructor */
+  FileInfo() : size(0), type(kFile) {}
+};
+
+/*! \brief file system system interface */
+class FileSystem {
+ public:
+  /*!
+   * \brief get singleton of filesystem instance according to URI
+   * \param path can be s3://..., hdfs://..., file://...,
+   *            empty string(will return local)
+   * \return a corresponding filesystem, report error if
+   *         we cannot find a matching system
+   */
+  static FileSystem *GetInstance(const URI &path);
+  /*! \brief virtual destructor */
+  virtual ~FileSystem() {}
+  /*!
+   * \brief get information about a path
+   * \param path the path to the file
+   * \return the information about the file
+   */
+  virtual FileInfo GetPathInfo(const URI &path) = 0;
+  /*!
+   * \brief list files in a directory
+   * \param path to the file
+   * \param out_list the output information about the files
+   */
+  virtual void ListDirectory(const URI &path, std::vector<FileInfo> *out_list) = 0;
+  /*!
+   * \brief list files in a directory recursively using ListDirectory
+   * \param path to the file
+   * \param out_list the output information about the files
+   */
+  virtual void ListDirectoryRecursive(const URI &path,
+                                      std::vector<FileInfo> *out_list);
+  /*!
+   * \brief open a stream
+   * \param path path to file
+   * \param flag can be "w", "r", "a
+   * \param allow_null whether NULL can be returned, or directly report error
+   * \return the created stream, can be NULL when allow_null == true and file do not exist
+   */
+  virtual Stream *Open(const URI &path,
+                       const char* const flag,
+                       bool allow_null = false) = 0;
+  /*!
+   * \brief open a seekable stream for read
+   * \param path the path to the file
+   * \param allow_null whether NULL can be returned, or directly report error
+   * \return the created stream, can be NULL when allow_null == true and file do not exist
+   */
+  virtual SeekStream *OpenForRead(const URI &path,
+                                  bool allow_null = false) = 0;
+};
+
+}  // namespace io
 }  // namespace dmlc
 #endif  // DMLC_IO_H_

scripts/travis/travis_osx_install.sh

@@ -6,12 +6,4 @@ if [ ${TRAVIS_OS_NAME} != "osx" ]; then
     exit 0
 fi
 
-# Update command line tool to avoid an error:
-# "_stdio.h: No such file or directory"
-softwareupdate --list
-softwareupdate --install "Command Line Tools (macOS High Sierra version 10.13) for Xcode-10.1"
-
-brew install gcc@7 || brew link --overwrite gcc@7
-brew update
-brew upgrade python3
 python3 -m pip install --upgrade pip

src/io.cc

@@ -9,7 +9,6 @@
 #include "io/recordio_split.h"
 #include "io/indexed_recordio_split.h"
 #include "io/single_file_split.h"
-#include "io/filesys.h"
 #include "io/local_filesys.h"
 #include "io/cached_input_split.h"
 #include "io/threaded_input_split.h"

src/io/azure_filesys.h

@@ -7,9 +7,9 @@
 #ifndef DMLC_IO_AZURE_FILESYS_H_
 #define DMLC_IO_AZURE_FILESYS_H_
 
+#include <dmlc/filesystem.h>
 #include <vector>
 #include <string>
-#include "./filesys.h"
 
 namespace dmlc {
 namespace io {

src/io/filesys.cc

@@ -1,7 +1,7 @@
 // Copyright by Contributors
-#include <queue>
 
-#include "./filesys.h"
+#include <dmlc/filesystem.h>
+#include <queue>
 
 namespace dmlc {
 namespace io {
@@ -25,4 +25,34 @@ void FileSystem::ListDirectoryRecursive(const URI &path,
 }
 
 }  // namespace io
+
+void TemporaryDirectory::RecursiveDelete(const std::string &path) {
+  io::URI uri(path.c_str());
+  io::FileSystem* fs = io::FileSystem::GetInstance(uri);
+  std::vector<io::FileInfo> file_list;
+  fs->ListDirectory(uri, &file_list);
+  for (io::FileInfo info : file_list) {
+    CHECK(!IsSymlink(info.path.name))
+        << "Symlink not supported in TemporaryDirectory";
+    if (info.type == io::FileType::kDirectory) {
+      RecursiveDelete(info.path.name);
+    } else {
+      CHECK_EQ(std::remove(info.path.name.c_str()), 0)
+          << "Couldn't remove file " << info.path.name;
+    }
+  }
+#if _WIN32
+  const bool rmdir_success = (RemoveDirectoryA(path.c_str()) != 0);
+#else
+  const bool rmdir_success = (rmdir(path.c_str()) == 0);
+#endif
+  if (rmdir_success) {
+    if (verbose_) {
+      LOG(INFO) << "Successfully deleted temporary directory " << path;
+    }
+  } else {
+    LOG(FATAL) << "~TemporaryDirectory(): "
+               << "Could not remove temporary directory " << path;
+  }
+}
 }  // namespace dmlc