Move filesys.h into io.h.

include/dmlc/build_config.h

@@ -1,35 +1,24 @@
-/*!
- * Copyright (c) 2018 by Contributors
- * \file build_config.h
- * \brief Default detection logic for fopen64 and other symbols.
- *        May be overriden by CMake
- * \author KOLANICH
- */
 #ifndef DMLC_BUILD_CONFIG_H_
 #define DMLC_BUILD_CONFIG_H_
 
-/* default logic for fopen64 */
-#if DMLC_USE_FOPEN64 && \
-  (!defined(__GNUC__) || (defined __ANDROID__) || (defined __FreeBSD__) \
-  || (defined __APPLE__) || ((defined __MINGW32__) && !(defined __MINGW64__)) \
-  || (defined __CYGWIN__) )
+/* #undef DMLC_FOPEN_64_PRESENT */
+
+#if !defined(DMLC_FOPEN_64_PRESENT) && DMLC_USE_FOPEN64
   #define DMLC_EMIT_FOPEN64_REDEFINE_WARNING
   #define fopen64 std::fopen
 #endif
 
-/* default logic for stack trace */
-#if (defined(__GNUC__) && !defined(__MINGW32__)\
-     && !defined(__sun) && !defined(__SVR4)\
-     && !(defined __MINGW64__) && !(defined __ANDROID__))\
-     && !defined(__CYGWIN__)
+#define DMLC_CXXABI_H_PRESENT
+#define DMLC_EXECINFO_H_PRESENT
+
+#if (defined DMLC_CXXABI_H_PRESENT) && (defined DMLC_EXECINFO_H_PRESENT)
   #define DMLC_LOG_STACK_TRACE 1
   #define DMLC_LOG_STACK_TRACE_SIZE 10
   #define DMLC_EXECINFO_H <execinfo.h>
 #endif
 
-/* default logic for detecting existence of nanosleep() */
-#if !(defined _WIN32) || (defined __CYGWIN__)
-  #define DMLC_NANOSLEEP_PRESENT
-#endif
+#define DMLC_NANOSLEEP_PRESENT
+
+#define DMLC_CMAKE_LITTLE_ENDIAN 1
 
 #endif  // DMLC_BUILD_CONFIG_H_

include/dmlc/filesystem.h

@@ -8,6 +8,7 @@
 #define DMLC_FILESYSTEM_H_
 
 #include <dmlc/logging.h>
+#include <dmlc/io.h>
 #include <algorithm>
 #include <string>
 #include <vector>

include/dmlc/io.h

@@ -7,6 +7,7 @@
 #define DMLC_IO_H_
 #include <cstdio>
 #include <string>
+#include <cstring>
 #include <vector>
 #include <istream>
 #include <ostream>
@@ -518,5 +519,118 @@ inline int istream::InBuf::underflow() {
   }
 }
 #endif
+
+namespace io {
+/*! \brief common data structure for URI */
+struct URI {
+  /*! \brief protocol */
+  std::string protocol;
+  /*!
+   * \brief host name, namenode for HDFS, bucket name for s3
+   */
+  std::string host;
+  /*! \brief name of the path */
+  std::string name;
+  /*! \brief enable default constructor */
+  URI(void) {}
+  /*!
+   * \brief construct from URI string
+   */
+  explicit URI(const char *uri) {
+    const char *p = std::strstr(uri, "://");
+    if (p == NULL) {
+      name = uri;
+    } else {
+      protocol = std::string(uri, p - uri + 3);
+      uri = p + 3;
+      p = std::strchr(uri, '/');
+      if (p == NULL) {
+        host = uri; name = '/';
+      } else {
+        host = std::string(uri, p - uri);
+        name = p;
+      }
+    }
+  }
+  /*! \brief string representation */
+  inline std::string str(void) const {
+    return protocol + host + name;
+  }
+};
+
+/*! \brief type of file */
+enum FileType {
+  /*! \brief the file is file */
+  kFile,
+  /*! \brief the file is directory */
+  kDirectory
+};
+
+/*! \brief use to store file information */
+struct FileInfo {
+  /*! \brief full path to the file */
+  URI path;
+  /*! \brief the size of the file */
+  size_t size;
+  /*! \brief the type of the file */
+  FileType type;
+  /*! \brief default constructor */
+  FileInfo() : size(0), type(kFile) {}
+};
+
+/*! \brief file system system interface */
+class FileSystem {
+ public:
+  /*!
+   * \brief get singleton of filesystem instance according to URI
+   * \param path can be s3://..., hdfs://..., file://...,
+   *            empty string(will return local)
+   * \return a corresponding filesystem, report error if
+   *         we cannot find a matching system
+   */
+  static FileSystem *GetInstance(const URI &path);
+  /*! \brief virtual destructor */
+  virtual ~FileSystem() {}
+  /*!
+   * \brief get information about a path
+   * \param path the path to the file
+   * \return the information about the file
+   */
+  virtual FileInfo GetPathInfo(const URI &path) = 0;
+  /*!
+   * \brief list files in a directory
+   * \param path to the file
+   * \param out_list the output information about the files
+   */
+  virtual void ListDirectory(const URI &path, std::vector<FileInfo> *out_list) = 0;
+  /*!
+   * \brief list files in a directory recursively using ListDirectory
+   * \param path to the file
+   * \param out_list the output information about the files
+   */
+  virtual void ListDirectoryRecursive(const URI &path,
+                                      std::vector<FileInfo> *out_list);
+  /*!
+   * \brief open a stream
+   * \param path path to file
+   * \param uri the uri of the input, can contain hdfs prefix
+   * \param flag can be "w", "r", "a
+   * \param allow_null whether NULL can be returned, or directly report error
+   * \return the created stream, can be NULL when allow_null == true and file do not exist
+   */
+  virtual Stream *Open(const URI &path,
+                       const char* const flag,
+                       bool allow_null = false) = 0;
+  /*!
+   * \brief open a seekable stream for read
+   * \param path the path to the file
+   * \param allow_null whether NULL can be returned, or directly report error
+   * \return the created stream, can be NULL when allow_null == true and file do not exist
+   */
+  virtual SeekStream *OpenForRead(const URI &path,
+                                  bool allow_null = false) = 0;
+};
+
+}  // namespace io
 }  // namespace dmlc
 #endif  // DMLC_IO_H_

src/io.cc

@@ -9,7 +9,6 @@
 #include "io/recordio_split.h"
 #include "io/indexed_recordio_split.h"
 #include "io/single_file_split.h"
-#include "io/filesys.h"
 #include "io/local_filesys.h"
 #include "io/cached_input_split.h"
 #include "io/threaded_input_split.h"

src/io/filesys.cc

@@ -2,8 +2,6 @@
 #include <queue>
 #include <dmlc/filesystem.h>
 
-#include "./filesys.h"
-
 namespace dmlc {
 namespace io {
 

src/io/input_split_base.h

@@ -8,12 +8,12 @@
 #define DMLC_IO_INPUT_SPLIT_BASE_H_
 
 #include <dmlc/io.h>
+#include <dmlc/filesystem.h>
 #include <cstdio>
 #include <cstring>
 #include <vector>
 #include <string>
 #include <algorithm>
-#include "./filesys.h"
 
 namespace dmlc {
 namespace io {

src/io/local_filesys.h

@@ -8,7 +8,7 @@
 #define DMLC_IO_LOCAL_FILESYS_H_
 
 #include <vector>
-#include "./filesys.h"
+#include <dmlc/filesystem.h>
 
 namespace dmlc {
 namespace io {

src/io/uri_spec.h

@@ -15,7 +15,6 @@
 #include <map>
 #include <vector>
 #include <utility>
-#include "./filesys.h"
 
 namespace dmlc {
 namespace io {