Initial commit of hadoop file system (credit vnvo2409) (tensorflow#1197)

* Initial commit of hadoop file system

Signed-off-by: Yong Tang <[email protected]>

* Update namespace to avoid collision/conflict

Signed-off-by: Yong Tang <[email protected]>

* Add a wrapper to HadoopFileSystem so that it will only be lazily loaded.

Signed-off-by: Yong Tang <[email protected]>

* Address review comments

Signed-off-by: Yong Tang <[email protected]>

Author: yongtang

WORKSPACE

@@ -1052,3 +1052,14 @@ http_archive(
         "https://pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.2.1.tar.gz",
     ],
 )
+
+http_archive(
+    name = "hadoop",
+    build_file = "//third_party:hadoop.BUILD",
+    sha256 = "5fd5831b12b1e0999bd352d6cca11ef80f883c81ffa898e53c68d8fe8d170e9f",
+    strip_prefix = "hadoop-3.3.0-src",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/downloads.apache.org/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-src.tar.gz",
+        "https://downloads.apache.org/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-src.tar.gz",
+    ],
+)

tensorflow_io/core/plugins/BUILD

@@ -30,6 +30,7 @@ cc_library(
     linkstatic = True,
     deps = [
         "//tensorflow_io/core/plugins/az",
+        "//tensorflow_io/core/plugins/hdfs",
         "//tensorflow_io/core/plugins/http",
         "//tensorflow_io/core/plugins/s3",
     ],

tensorflow_io/core/plugins/file_system_plugins.cc

@@ -18,11 +18,12 @@ limitations under the License.
 void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
   info->plugin_memory_allocate = tensorflow::io::plugin_memory_allocate;
   info->plugin_memory_free = tensorflow::io::plugin_memory_free;
-  info->num_schemes = 3;
+  info->num_schemes = 4;
   info->ops = static_cast<TF_FilesystemPluginOps*>(
       tensorflow::io::plugin_memory_allocate(info->num_schemes *
                                              sizeof(info->ops[0])));
   tensorflow::io::az::ProvideFilesystemSupportFor(&info->ops[0], "az");
   tensorflow::io::http::ProvideFilesystemSupportFor(&info->ops[1], "http");
   tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e");
+  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse");
 }

tensorflow_io/core/plugins/file_system_plugins.h

@@ -32,6 +32,12 @@ void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, const char* uri);
 
 }  // namespace az
 
+namespace hdfs {
+
+void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, const char* uri);
+
+}  // namespace hdfs
+
 namespace http {
 
 void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, const char* uri);

tensorflow_io/core/plugins/hdfs/BUILD

@@ -0,0 +1,22 @@
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "//:tools/build/tensorflow_io.bzl",
+    "tf_io_copts",
+)
+
+cc_library(
+    name = "hdfs",
+    srcs = [
+        "hadoop_filesystem.cc",
+    ],
+    copts = tf_io_copts(),
+    linkstatic = True,
+    deps = [
+        "//tensorflow_io/core/plugins:plugins_header",
+        "@hadoop",
+    ],
+    alwayslink = 1,
+)