fs: uv_fs_{open,read}_dir for unix

Tested on Linux, MacOS X, SmartOS and Windows.

Fixes #1430

Makefile.am

@@ -140,6 +140,7 @@ test_run_tests_SOURCES = test/blackhole-server.c \
                          test/test-fs-event.c \
                          test/test-fs-poll.c \
                          test/test-fs.c \
+                         test/test-fs-readdir.c \
                          test/test-get-currentexe.c \
                          test/test-get-loadavg.c \
                          test/test-get-memory.c \

include/uv-unix.h

@@ -158,6 +158,13 @@ typedef gid_t uv_gid_t;
 typedef uid_t uv_uid_t;
 
 typedef struct dirent uv__dirent_t;
+/*
+ * "dirent" is used to hold a buffer large enough for any dirent in the
+ * directory being read. Avoids allocating for each directory entry.
+ */
+#define UV_DIR_PRIVATE_FIELDS \
+  uv__dirent_t* dirent; \
+  DIR* dir;
 
 #if defined(DT_UNKNOWN)
 # define HAVE_DIRENT_TYPES

include/uv-win.h

@@ -294,6 +294,16 @@ typedef struct uv__dirent_s {
   char d_name[1];
 } uv__dirent_t;
 
+/*
+  * "dirent" is used to hold a buffer large enough for any dirent in the
+  * directory being read. Avoids allocating for each directory entry.
+  */
+#define UV_DIR_PRIVATE_FIELDS \
+  HANDLE dir_handle; \
+  WIN32_FIND_DATAW find_data; \
+  BOOL need_find_call; \
+  uv__dirent_t* dirent;
+
 #define UV__DT_DIR     UV_DIRENT_DIR
 #define UV__DT_FILE    UV_DIRENT_FILE
 #define UV__DT_LINK    UV_DIRENT_LINK

include/uv.h

@@ -156,6 +156,7 @@ extern "C" {
   XX(TTY, tty)                                                                \
   XX(UDP, udp)                                                                \
   XX(SIGNAL, signal)                                                          \
+  XX(DIR, dir)                                                                \
 
 #define UV_REQ_TYPE_MAP(XX)                                                   \
   XX(REQ, req)                                                                \
@@ -197,6 +198,7 @@ typedef enum {
 /* Handle types. */
 typedef struct uv_loop_s uv_loop_t;
 typedef struct uv_handle_s uv_handle_t;
+typedef struct uv_dir_s uv_dir_t;
 typedef struct uv_stream_s uv_stream_t;
 typedef struct uv_tcp_s uv_tcp_t;
 typedef struct uv_udp_s uv_udp_t;
@@ -1032,13 +1034,21 @@ typedef enum {
   UV_FS_MKDTEMP,
   UV_FS_RENAME,
   UV_FS_SCANDIR,
+  UV_FS_OPENDIR,
+  UV_FS_READDIR,
   UV_FS_LINK,
   UV_FS_SYMLINK,
   UV_FS_READLINK,
   UV_FS_CHOWN,
   UV_FS_FCHOWN
 } uv_fs_type;
 
+struct uv_dir_s {
+  UV_HANDLE_FIELDS
+  int dir_flags;
+  UV_DIR_PRIVATE_FIELDS
+};
+
 /* uv_fs_t is a subclass of uv_req_t. */
 struct uv_fs_s {
   UV_REQ_FIELDS
@@ -1049,6 +1059,8 @@ struct uv_fs_s {
   void* ptr;
   const char* path;
   uv_stat_t statbuf;  /* Stores the result of uv_fs_stat() and uv_fs_fstat(). */
+  uv_dir_t* dir_handle; /* Stores the result of uv_fs_opendir() */
+  uv_dirent_t* dir_entry; /* Stores the result of uv_fs_readdir() */
   UV_FS_PRIVATE_FIELDS
 };
 
@@ -1101,6 +1113,27 @@ UV_EXTERN int uv_fs_scandir(uv_loop_t* loop,
                             uv_fs_cb cb);
 UV_EXTERN int uv_fs_scandir_next(uv_fs_t* req,
                                  uv_dirent_t* ent);
+
+/*
+ * In the future, this flag could be used to specify a custom filter to be
+ * applied each time a directory entry is read. It removes the need for the
+ * user to iterate through all directory entries again to filter out the ones
+ * that they are interested in.
+ */
+#define UV_DIR_FLAGS_NONE 0
+
+UV_EXTERN int uv_fs_opendir(uv_loop_t* loop,
+                            uv_fs_t* req,
+                            uv_dir_t* dirh,
+                            const char* path,
+                            int flags,
+                            uv_fs_cb cb);
+UV_EXTERN int uv_fs_readdir(uv_loop_t* loop,
+                            uv_fs_t* req,
+                            uv_dir_t* dirh,
+                            uv_dirent_t* dirent,
+                            uv_fs_cb cb);
+
 UV_EXTERN int uv_fs_stat(uv_loop_t* loop,
                          uv_fs_t* req,
                          const char* path,
@@ -1390,7 +1423,6 @@ struct uv_loop_s {
   UV_LOOP_PRIVATE_FIELDS
 };
 
-
 /* Don't export the private CPP symbols. */
 #undef UV_HANDLE_TYPE_PRIVATE
 #undef UV_REQ_TYPE_PRIVATE

src/unix/core.c

@@ -156,6 +156,10 @@ void uv_close(uv_handle_t* handle, uv_close_cb close_cb) {
     /* itself close uv__make_close_pending whenever appropriate. */
     return;
 
+  case UV_DIR:
+    uv__dir_close((uv_dir_t*)handle);
+    break;
+
   default:
     assert(0);
   }
@@ -221,6 +225,7 @@ static void uv__finish_close(uv_handle_t* handle) {
     case UV_FS_POLL:
     case UV_POLL:
     case UV_SIGNAL:
+    case UV_DIR:
       break;
 
     case UV_NAMED_PIPE:

src/unix/fs.c

@@ -136,6 +136,95 @@ static ssize_t uv__fs_fdatasync(uv_fs_t* req) {
 }
 
 
+static ssize_t uv__fs_clamp_max_path_len(ssize_t len) {
+  if (len == -1) {
+  #if defined(PATH_MAX)
+    len = PATH_MAX;
+  #else
+    len = 4096;
+  #endif
+  }
+
+  return len;
+}
+
+/*
+ * Computes the required size in bytes for directory entries read from the
+ * directory identified by the handle "dir".
+ *
+ * This code does not trust values of NAME_MAX that are less than
+ * 255, since some systems (including at least HP-UX) incorrectly
+ * define it to be a smaller value.
+ *
+ * This code was copied from
+ * http://womble.decadent.org.uk/readdir_r-advisory.html
+ * and slightly adapted.
+ */
+static size_t uv__fs_direntry_size(const DIR* dir) {
+  long name_max;
+  size_t name_end;
+
+  name_max = -1;
+  name_end = -1;
+
+#if defined(HAVE_FPATHCONF) && defined(HAVE_DIRFD) \
+    && defined(_PC_NAME_MAX)
+  name_max = fpathconf(dirfd(dirp), _PC_NAME_MAX);
+  if (name_max == -1)
+#   if defined(NAME_MAX)
+      name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
+#   else
+      return (size_t)(-1);
+#   endif /* defined(NAME_MAX)
+#else
+# if defined(NAME_MAX)
+    name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
+# else
+#   error "buffer size for readdir_r cannot be determined"
+# endif /* defined(NAME_MAX) */
+#endif
+
+  /*
+   * Most of the time, struct dirent is defined by operating systems' header
+   * files as either one of the following possible implementations:
+   * 1)
+   *
+   * struct dirent {
+   *   char d_name[ NAME_MAX + 1 ];
+   *   other stuff;
+   * };
+   *
+   * 2)
+   *
+   * struct dirent {
+   *   char d_name[1];
+   *   other stuff;
+   * };
+   *
+   * In case 1, using sizeof(struct dirent) to compute the size for the buffer
+   * would account for the path name. In case 2), it would not.
+   *
+   * The following test ensures that this function always allocate enough
+   * memory to hold the longest path names that can exist for a given
+   * directory, regardless of how system headers implement struct dirent.
+   */
+  name_end = (size_t)offsetof(struct dirent, d_name) + name_max + 1;
+  if (name_end > sizeof(struct dirent))
+    return name_end;
+  return sizeof(struct dirent);
+}
+
+static ssize_t uv__fs_max_path_len(const char* path) {
+  ssize_t len;
+
+  assert(path);
+
+  len = pathconf(path, _PC_PATH_MAX);
+  len = uv__fs_clamp_max_path_len(len);
+
+  return len;
+}
+
 static ssize_t uv__fs_futime(uv_fs_t* req) {
 #if defined(__linux__)
   /* utimesat() has nanosecond resolution but we stick to microseconds
@@ -339,21 +428,74 @@ static ssize_t uv__fs_scandir(uv_fs_t* req) {
   return n;
 }
 
+static int uv__fs_opendir(uv_fs_t* req) {
+  DIR *dir;
+  size_t len;
 
-static ssize_t uv__fs_readlink(uv_fs_t* req) {
-  ssize_t len;
-  char* buf;
+  assert(req && req->dir_handle);
 
-  len = pathconf(req->path, _PC_PATH_MAX);
+  dir = opendir(req->path);
+  if (dir == NULL)
+    return -1;
 
-  if (len == -1) {
-#if defined(PATH_MAX)
-    len = PATH_MAX;
-#else
-    len = 4096;
-#endif
+  len = uv__fs_direntry_size(dir);
+  if (len == (size_t)-1)
+    return -1;
+
+  /*
+   * Allocate the space for each directory entry just once instead of
+   * once per directory entry.
+   */
+  req->dir_handle->dirent = malloc(len + 1);
+  if (req->dir_handle->dirent == NULL) {
+    errno = ENOMEM;
+    return -1;
+  }
+
+  req->dir_handle->dir = dir;
+  req->ptr = req->dir_handle;
+
+  return 0;
+}
+
+static int uv__fs_readdir(uv_fs_t* req) {
+  struct dirent *de;
+  int r;
+
+  assert(req);
+  assert(req->dir_handle);
+  assert(req->dir_handle->dirent);
+  assert(req->dir_entry);
+
+  req->ptr = req->dir_entry;
+
+  de = NULL;
+
+  r = readdir_r(req->dir_handle->dir, req->dir_handle->dirent, &de);
+
+  if (r == 0) {
+    if (de != NULL) {
+      req->dir_entry->name = strdup(de->d_name);
+      if (req->dir_entry->name != NULL) {
+        req->dir_entry->type = uv__fs_get_dirent_type(de);
+        r = 1;
+      } else {
+        errno = ENOMEM;
+        r = -1;
+      }
+    } else {
+      r = UV_EOF;
+    }
   }
 
+  return r;
+}
+
+static ssize_t uv__fs_readlink(uv_fs_t* req) {
+  ssize_t len;
+  char* buf;
+
+  len = uv__fs_max_path_len(req->path);
   buf = malloc(len + 1);
 
   if (buf == NULL) {
@@ -779,6 +921,8 @@ static void uv__fs_work(struct uv__work* w) {
     X(MKDTEMP, uv__fs_mkdtemp(req));
     X(READ, uv__fs_read(req));
     X(SCANDIR, uv__fs_scandir(req));
+    X(OPENDIR, uv__fs_opendir(req));
+    X(READDIR, uv__fs_readdir(req));
     X(READLINK, uv__fs_readlink(req));
     X(RENAME, rename(req->path, req->new_path));
     X(RMDIR, rmdir(req->path));
@@ -1050,6 +1194,54 @@ int uv_fs_scandir(uv_loop_t* loop,
   POST;
 }
 
+int uv_fs_opendir(uv_loop_t* loop,
+                  uv_fs_t* req,
+                  uv_dir_t* dirh,
+                  const char* path,
+                  int flags,
+                  uv_fs_cb cb) {
+
+  INIT(OPENDIR);
+  PATH;
+
+  uv__handle_init(loop, dirh, UV_DIR);
+  uv__handle_start(dirh);
+
+  dirh->dir_flags = flags;
+
+  dirh->dir = NULL;
+  req->dir_handle = dirh;
+
+  POST;
+}
+
+void uv__dir_close(uv_dir_t* dirh) {
+  if (!uv__is_active(dirh))
+    return;
+
+  if (dirh->dir) {
+    closedir(dirh->dir);
+    dirh->dir = NULL;
+  }
+
+  if (dirh->dirent) {
+    free(dirh->dirent);
+    dirh->dirent = NULL;
+  }
+}
+
+int uv_fs_readdir(uv_loop_t* loop,
+                  uv_fs_t* req,
+                  uv_dir_t* dirh,
+                  uv_dirent_t* dirent,
+                  uv_fs_cb cb) {
+  INIT(READDIR);
+
+  req->dir_handle = dirh;
+  req->dir_entry = dirent;
+
+  POST;
+}
 
 int uv_fs_readlink(uv_loop_t* loop,
                    uv_fs_t* req,
@@ -1169,7 +1361,12 @@ void uv_fs_req_cleanup(uv_fs_t* req) {
   if (req->fs_type == UV_FS_SCANDIR && req->ptr != NULL)
     uv__fs_scandir_cleanup(req);
 
-  if (req->ptr != &req->statbuf)
+  if (req->fs_type == UV_FS_READDIR)
+    uv__fs_readdir_cleanup(req);
+
+  if (req->fs_type != UV_FS_READDIR &&
+      req->fs_type != UV_FS_OPENDIR &&
+      req->ptr != &req->statbuf)
     free(req->ptr);
   req->ptr = NULL;
 }