From 692216e8633eacae001a023ebbdcafbad0985854 Mon Sep 17 00:00:00 2001 From: Jiaming Yuan Date: Tue, 5 Feb 2019 11:26:32 +0800 Subject: [PATCH] Add install target. (#495) * Add install target. * Export `dmlc` in CMake. * Use GNUInstallDir instead. * Add config target. * Move filesys.h into io.h. * Replace the rest of filesys.h. * Address linter. * Address linter. * Improve Travis CI on Mac target --- .travis.yml | 7 ++ CMakeLists.txt | 37 ++++++-- cmake/dmlc-config.cmake.in | 5 ++ doc/build.md | 16 ++++ doc/index.md | 1 + include/dmlc/filesystem.h | 33 +------ include/dmlc/io.h | 113 +++++++++++++++++++++++ scripts/travis/travis_osx_install.sh | 8 -- src/io.cc | 1 - src/io/azure_filesys.h | 2 +- src/io/filesys.cc | 34 ++++++- src/io/filesys.h | 128 --------------------------- src/io/hdfs_filesys.h | 3 +- src/io/input_split_base.h | 2 +- src/io/local_filesys.h | 2 +- src/io/s3_filesys.h | 2 +- src/io/uri_spec.h | 1 - test/filesys_test.cc | 2 +- 18 files changed, 215 insertions(+), 182 deletions(-) create mode 100644 cmake/dmlc-config.cmake.in create mode 100644 doc/build.md delete mode 100644 src/io/filesys.h diff --git a/.travis.yml b/.travis.yml index f7d4373860..b9c49268ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ group: travis_latest language: python +osx_image: xcode10.1 + matrix: include: - env: TASK=lint @@ -32,6 +34,11 @@ addons: - unzip - gcc-4.8 - g++-4.8 + homebrew: + packages: + - gcc@7 + - python3 + update: true before_install: - export TRAVIS=scripts/travis diff --git a/CMakeLists.txt b/CMakeLists.txt index 735e4be8ed..17837e74d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.2) -project(dmlc C CXX) +project(dmlc VERSION 0.3 LANGUAGES C CXX) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/build/private/local_config.cmake) @@ -11,7 +11,6 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_LOCAL}/Modules) include(CheckCXXSymbolExists) include(cmake/Utils.cmake) -#include(cmake/dmlccore.cmake) # Options dmlccore_option(USE_HDFS "Build with HDFS support" OFF) @@ -24,7 +23,7 @@ dmlccore_option(GOOGLE_TEST "Build google tests" OFF) # include path set(INCLUDE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/include") set(INCLUDE_DMLC_DIR "${INCLUDE_ROOT}/dmlc") -include_directories("${INCLUDE_ROOT}") +# include_directories("${INCLUDE_ROOT}") set(dmlccore_LINKER_LIBS "") # HDFS configurations @@ -187,6 +186,9 @@ endif() add_library(dmlc ${SOURCE}) target_link_libraries(dmlc ${dmlccore_LINKER_LIBS}) +target_include_directories(dmlc PUBLIC + $ + $) target_compile_definitions(dmlc PRIVATE -D_XOPEN_SOURCE=700 -D_POSIX_SOURCE -D_POSIX_C_SOURCE=200809L -D_DARWIN_C_SOURCE) @@ -199,10 +201,36 @@ if(INSTALL_INCLUDE_DIR) endif() # ---[ Install the archive static lib and header files -install(TARGETS dmlc ARCHIVE DESTINATION lib${LIB_SUFFIX} LIBRARY DESTINATION lib${LIB_SUFFIX}) +include(GNUInstallDirs) +install(TARGETS dmlc + EXPORT DMLCTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install(EXPORT DMLCTargets + FILE DMLCTargets.cmake + NAMESPACE dmlc:: + EXPORT_LINK_INTERFACE_LIBRARIES + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc) + install(DIRECTORY include DESTINATION .) install(DIRECTORY doc DESTINATION .) +# ---[ Package configurations +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${CMAKE_SOURCE_DIR}/cmake/dmlc-config.cmake.in + ${CMAKE_BINARY_DIR}/cmake/dmlc-config.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc) +write_basic_package_version_file( + ${CMAKE_BINARY_DIR}/cmake/dmlc-config-version.cmake + VERSION ${DMLC_VERSION} + COMPATIBILITY AnyNewerVersion) +install( + FILES + ${CMAKE_BINARY_DIR}/cmake/dmlc-config.cmake + ${CMAKE_BINARY_DIR}/cmake/dmlc-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmlc) + # ---[ Linter target if(MSVC) find_package(PythonInterp) @@ -216,4 +244,3 @@ if(GOOGLE_TEST) include(CTest) add_subdirectory(test/unittest) endif() - diff --git a/cmake/dmlc-config.cmake.in b/cmake/dmlc-config.cmake.in new file mode 100644 index 0000000000..a318d48e3f --- /dev/null +++ b/cmake/dmlc-config.cmake.in @@ -0,0 +1,5 @@ +@PACKAGE_INIT@ + +if(NOT TARGET dmlc::dmlc) + include(${CMAKE_CURRENT_LIST_DIR}/DMLCTargets.cmake) +endif() diff --git a/doc/build.md b/doc/build.md new file mode 100644 index 0000000000..71b766701c --- /dev/null +++ b/doc/build.md @@ -0,0 +1,16 @@ +Using dmlc-core with CMake +========================== +dmlc defines a exported CMake target which can be used by `find_package` command. + +For example, if you have a simple C++ project that contains only a main.cc file, +which uses dmlc-core as dependency, the CMakeLists.txt for your project can be +defined as follow: + +``` cmake +project(demo) +cmake_minimum_required(VERSION 3.2) + +find_package(dmlc REQUIRED) +add_executable(demo main.cc) +target_link_libraries(demo dmlc::dmlc) +``` \ No newline at end of file diff --git a/doc/index.md b/doc/index.md index 73d671e3c0..9f1f27cb43 100644 --- a/doc/index.md +++ b/doc/index.md @@ -4,6 +4,7 @@ DMLC Core contains common codebase to help us build machine learning toolkits ea Contents -------- +* [Using dmlc-core with CMake](build.md) * [Parameter Structure for Machine Learning](parameter.md) * [Doxygen C++ API Reference](https://dmlc-core.readthedocs.org/en/latest/doxygen) diff --git a/include/dmlc/filesystem.h b/include/dmlc/filesystem.h index ea605e4694..64d1073041 100644 --- a/include/dmlc/filesystem.h +++ b/include/dmlc/filesystem.h @@ -8,6 +8,7 @@ #define DMLC_FILESYSTEM_H_ #include +#include #include #include #include @@ -25,8 +26,6 @@ #include #endif // _WIN32 -#include "../../src/io/filesys.h" - namespace dmlc { /*! @@ -152,35 +151,7 @@ class TemporaryDirectory { * \brief Delete a directory recursively, along with sub-directories and files. * \param path String representation of path. It must refer to a directory. */ - inline void RecursiveDelete(const std::string& path) { - io::URI uri(path.c_str()); - io::FileSystem* fs = io::FileSystem::GetInstance(uri); - std::vector file_list; - fs->ListDirectory(uri, &file_list); - for (io::FileInfo info : file_list) { - CHECK(!IsSymlink(info.path.name)) - << "Symlink not supported in TemporaryDirectory"; - if (info.type == io::FileType::kDirectory) { - RecursiveDelete(info.path.name); - } else { - CHECK_EQ(std::remove(info.path.name.c_str()), 0) - << "Couldn't remove file " << info.path.name; - } - } -#if _WIN32 - const bool rmdir_success = (RemoveDirectoryA(path.c_str()) != 0); -#else - const bool rmdir_success = (rmdir(path.c_str()) == 0); -#endif - if (rmdir_success) { - if (verbose_) { - LOG(INFO) << "Successfully deleted temporary directory " << path; - } - } else { - LOG(FATAL) << "~TemporaryDirectory(): " - << "Could not remove temporary directory " << path; - } - } + void RecursiveDelete(const std::string& path); }; } // namespace dmlc diff --git a/include/dmlc/io.h b/include/dmlc/io.h index 5e76e4c6e2..4e86f9d6e2 100644 --- a/include/dmlc/io.h +++ b/include/dmlc/io.h @@ -7,6 +7,7 @@ #define DMLC_IO_H_ #include #include +#include #include #include #include @@ -518,5 +519,117 @@ inline int istream::InBuf::underflow() { } } #endif + +namespace io { +/*! \brief common data structure for URI */ +struct URI { + /*! \brief protocol */ + std::string protocol; + /*! + * \brief host name, namenode for HDFS, bucket name for s3 + */ + std::string host; + /*! \brief name of the path */ + std::string name; + /*! \brief enable default constructor */ + URI(void) {} + /*! + * \brief construct from URI string + */ + explicit URI(const char *uri) { + const char *p = std::strstr(uri, "://"); + if (p == NULL) { + name = uri; + } else { + protocol = std::string(uri, p - uri + 3); + uri = p + 3; + p = std::strchr(uri, '/'); + if (p == NULL) { + host = uri; name = '/'; + } else { + host = std::string(uri, p - uri); + name = p; + } + } + } + /*! \brief string representation */ + inline std::string str(void) const { + return protocol + host + name; + } +}; + +/*! \brief type of file */ +enum FileType { + /*! \brief the file is file */ + kFile, + /*! \brief the file is directory */ + kDirectory +}; + +/*! \brief use to store file information */ +struct FileInfo { + /*! \brief full path to the file */ + URI path; + /*! \brief the size of the file */ + size_t size; + /*! \brief the type of the file */ + FileType type; + /*! \brief default constructor */ + FileInfo() : size(0), type(kFile) {} +}; + +/*! \brief file system system interface */ +class FileSystem { + public: + /*! + * \brief get singleton of filesystem instance according to URI + * \param path can be s3://..., hdfs://..., file://..., + * empty string(will return local) + * \return a corresponding filesystem, report error if + * we cannot find a matching system + */ + static FileSystem *GetInstance(const URI &path); + /*! \brief virtual destructor */ + virtual ~FileSystem() {} + /*! + * \brief get information about a path + * \param path the path to the file + * \return the information about the file + */ + virtual FileInfo GetPathInfo(const URI &path) = 0; + /*! + * \brief list files in a directory + * \param path to the file + * \param out_list the output information about the files + */ + virtual void ListDirectory(const URI &path, std::vector *out_list) = 0; + /*! + * \brief list files in a directory recursively using ListDirectory + * \param path to the file + * \param out_list the output information about the files + */ + virtual void ListDirectoryRecursive(const URI &path, + std::vector *out_list); + /*! + * \brief open a stream + * \param path path to file + * \param flag can be "w", "r", "a + * \param allow_null whether NULL can be returned, or directly report error + * \return the created stream, can be NULL when allow_null == true and file do not exist + */ + virtual Stream *Open(const URI &path, + const char* const flag, + bool allow_null = false) = 0; + /*! + * \brief open a seekable stream for read + * \param path the path to the file + * \param allow_null whether NULL can be returned, or directly report error + * \return the created stream, can be NULL when allow_null == true and file do not exist + */ + virtual SeekStream *OpenForRead(const URI &path, + bool allow_null = false) = 0; +}; + +} // namespace io } // namespace dmlc #endif // DMLC_IO_H_ diff --git a/scripts/travis/travis_osx_install.sh b/scripts/travis/travis_osx_install.sh index fc6abcd8f7..5307b24540 100755 --- a/scripts/travis/travis_osx_install.sh +++ b/scripts/travis/travis_osx_install.sh @@ -6,12 +6,4 @@ if [ ${TRAVIS_OS_NAME} != "osx" ]; then exit 0 fi -# Update command line tool to avoid an error: -# "_stdio.h: No such file or directory" -softwareupdate --list -softwareupdate --install "Command Line Tools (macOS High Sierra version 10.13) for Xcode-10.1" - -brew install gcc@7 || brew link --overwrite gcc@7 -brew update -brew upgrade python3 python3 -m pip install --upgrade pip diff --git a/src/io.cc b/src/io.cc index 2d973920e1..732f304abd 100644 --- a/src/io.cc +++ b/src/io.cc @@ -9,7 +9,6 @@ #include "io/recordio_split.h" #include "io/indexed_recordio_split.h" #include "io/single_file_split.h" -#include "io/filesys.h" #include "io/local_filesys.h" #include "io/cached_input_split.h" #include "io/threaded_input_split.h" diff --git a/src/io/azure_filesys.h b/src/io/azure_filesys.h index 8ff5ec1be2..8f22d8930f 100644 --- a/src/io/azure_filesys.h +++ b/src/io/azure_filesys.h @@ -7,9 +7,9 @@ #ifndef DMLC_IO_AZURE_FILESYS_H_ #define DMLC_IO_AZURE_FILESYS_H_ +#include #include #include -#include "./filesys.h" namespace dmlc { namespace io { diff --git a/src/io/filesys.cc b/src/io/filesys.cc index d3e0a144e5..736c82d973 100644 --- a/src/io/filesys.cc +++ b/src/io/filesys.cc @@ -1,7 +1,7 @@ // Copyright by Contributors -#include -#include "./filesys.h" +#include +#include namespace dmlc { namespace io { @@ -25,4 +25,34 @@ void FileSystem::ListDirectoryRecursive(const URI &path, } } // namespace io + +void TemporaryDirectory::RecursiveDelete(const std::string &path) { + io::URI uri(path.c_str()); + io::FileSystem* fs = io::FileSystem::GetInstance(uri); + std::vector file_list; + fs->ListDirectory(uri, &file_list); + for (io::FileInfo info : file_list) { + CHECK(!IsSymlink(info.path.name)) + << "Symlink not supported in TemporaryDirectory"; + if (info.type == io::FileType::kDirectory) { + RecursiveDelete(info.path.name); + } else { + CHECK_EQ(std::remove(info.path.name.c_str()), 0) + << "Couldn't remove file " << info.path.name; + } + } +#if _WIN32 + const bool rmdir_success = (RemoveDirectoryA(path.c_str()) != 0); +#else + const bool rmdir_success = (rmdir(path.c_str()) == 0); +#endif + if (rmdir_success) { + if (verbose_) { + LOG(INFO) << "Successfully deleted temporary directory " << path; + } + } else { + LOG(FATAL) << "~TemporaryDirectory(): " + << "Could not remove temporary directory " << path; + } +} } // namespace dmlc diff --git a/src/io/filesys.h b/src/io/filesys.h deleted file mode 100644 index ca7f70aa0a..0000000000 --- a/src/io/filesys.h +++ /dev/null @@ -1,128 +0,0 @@ -/*! - * Copyright (c) 2015 by Contributors - * \file filesystem.h - * \brief general file system io interface - * \author Tianqi Chen - */ -#ifndef DMLC_IO_FILESYS_H_ -#define DMLC_IO_FILESYS_H_ - -#include -#include -#include -#include - -namespace dmlc { -namespace io { -/*! \brief common data structure for URI */ -struct URI { - /*! \brief protocol */ - std::string protocol; - /*! - * \brief host name, namenode for HDFS, bucket name for s3 - */ - std::string host; - /*! \brief name of the path */ - std::string name; - /*! \brief enable default constructor */ - URI(void) {} - /*! - * \brief construct from URI string - */ - explicit URI(const char *uri) { - const char *p = std::strstr(uri, "://"); - if (p == NULL) { - name = uri; - } else { - protocol = std::string(uri, p - uri + 3); - uri = p + 3; - p = std::strchr(uri, '/'); - if (p == NULL) { - host = uri; name = '/'; - } else { - host = std::string(uri, p - uri); - name = p; - } - } - } - /*! \brief string representation */ - inline std::string str(void) const { - return protocol + host + name; - } -}; - -/*! \brief type of file */ -enum FileType { - /*! \brief the file is file */ - kFile, - /*! \brief the file is directory */ - kDirectory -}; - -/*! \brief use to store file information */ -struct FileInfo { - /*! \brief full path to the file */ - URI path; - /*! \brief the size of the file */ - size_t size; - /*! \brief the type of the file */ - FileType type; - /*! \brief default constructor */ - FileInfo() : size(0), type(kFile) {} -}; - -/*! \brief file system system interface */ -class FileSystem { - public: - /*! - * \brief get singleton of filesystem instance according to URI - * \param path can be s3://..., hdfs://..., file://..., - * empty string(will return local) - * \return a corresponding filesystem, report error if - * we cannot find a matching system - */ - static FileSystem *GetInstance(const URI &path); - /*! \brief virtual destructor */ - virtual ~FileSystem() {} - /*! - * \brief get information about a path - * \param path the path to the file - * \return the information about the file - */ - virtual FileInfo GetPathInfo(const URI &path) = 0; - /*! - * \brief list files in a directory - * \param path to the file - * \param out_list the output information about the files - */ - virtual void ListDirectory(const URI &path, std::vector *out_list) = 0; - /*! - * \brief list files in a directory recursively using ListDirectory - * \param path to the file - * \param out_list the output information about the files - */ - virtual void ListDirectoryRecursive(const URI &path, - std::vector *out_list); - /*! - * \brief open a stream - * \param path path to file - * \param uri the uri of the input, can contain hdfs prefix - * \param flag can be "w", "r", "a - * \param allow_null whether NULL can be returned, or directly report error - * \return the created stream, can be NULL when allow_null == true and file do not exist - */ - virtual Stream *Open(const URI &path, - const char* const flag, - bool allow_null = false) = 0; - /*! - * \brief open a seekable stream for read - * \param path the path to the file - * \param allow_null whether NULL can be returned, or directly report error - * \return the created stream, can be NULL when allow_null == true and file do not exist - */ - virtual SeekStream *OpenForRead(const URI &path, - bool allow_null = false) = 0; -}; -} // namespace io -} // namespace dmlc -#endif // DMLC_IO_FILESYS_H_ diff --git a/src/io/hdfs_filesys.h b/src/io/hdfs_filesys.h index 4ebe2d311a..ef057b0d19 100644 --- a/src/io/hdfs_filesys.h +++ b/src/io/hdfs_filesys.h @@ -9,9 +9,10 @@ extern "C" { #include } +#include + #include #include -#include "./filesys.h" namespace dmlc { namespace io { diff --git a/src/io/input_split_base.h b/src/io/input_split_base.h index 0679eec5f7..366278864d 100644 --- a/src/io/input_split_base.h +++ b/src/io/input_split_base.h @@ -8,12 +8,12 @@ #define DMLC_IO_INPUT_SPLIT_BASE_H_ #include +#include #include #include #include #include #include -#include "./filesys.h" namespace dmlc { namespace io { diff --git a/src/io/local_filesys.h b/src/io/local_filesys.h index 1866201833..a1b5f5cf87 100644 --- a/src/io/local_filesys.h +++ b/src/io/local_filesys.h @@ -7,8 +7,8 @@ #ifndef DMLC_IO_LOCAL_FILESYS_H_ #define DMLC_IO_LOCAL_FILESYS_H_ +#include #include -#include "./filesys.h" namespace dmlc { namespace io { diff --git a/src/io/s3_filesys.h b/src/io/s3_filesys.h index 791a3ac900..3cf6640e6b 100644 --- a/src/io/s3_filesys.h +++ b/src/io/s3_filesys.h @@ -7,9 +7,9 @@ #ifndef DMLC_IO_S3_FILESYS_H_ #define DMLC_IO_S3_FILESYS_H_ +#include #include #include -#include "./filesys.h" namespace dmlc { namespace io { diff --git a/src/io/uri_spec.h b/src/io/uri_spec.h index 4e62c413d9..c9c4b38495 100644 --- a/src/io/uri_spec.h +++ b/src/io/uri_spec.h @@ -15,7 +15,6 @@ #include #include #include -#include "./filesys.h" namespace dmlc { namespace io { diff --git a/test/filesys_test.cc b/test/filesys_test.cc index 06e639cd62..940baf65ab 100644 --- a/test/filesys_test.cc +++ b/test/filesys_test.cc @@ -3,7 +3,7 @@ #include #include #include -#include "../src/io/filesys.h" +#include int main(int argc, char *argv[]) { if (argc < 3) {