Switch to modular file system for hdfs (#1309)

* Switch to modular file system for hdfs

This PR is part of the effort to switch to modular file system for hdfs.
When TF_ENABLE_LEGACY_FILESYSTEM=1 is provided, old behavior will
be preserved.

Signed-off-by: Yong Tang <[email protected]>

* Build against tf-nightly

Signed-off-by: Yong Tang <[email protected]>

* Update tests

Signed-off-by: Yong Tang <[email protected]>

* Adjust the if else logic, follow review comment

Signed-off-by: Yong Tang <[email protected]>

Author: yongtang

tensorflow_io/core/plugins/file_system_plugins.cc

@@ -15,7 +15,13 @@ limitations under the License.
 
 #include "tensorflow_io/core/plugins/file_system_plugins.h"
 
+#include "absl/strings/ascii.h"
+
 void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
+  const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM");
+  std::string enable_legacy =
+      enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : "";
+
   info->plugin_memory_allocate = tensorflow::io::plugin_memory_allocate;
   info->plugin_memory_free = tensorflow::io::plugin_memory_free;
   info->num_schemes = 7;
@@ -25,8 +31,14 @@ void TF_InitPlugin(TF_FilesystemPluginInfo* info) {
   tensorflow::io::az::ProvideFilesystemSupportFor(&info->ops[0], "az");
   tensorflow::io::http::ProvideFilesystemSupportFor(&info->ops[1], "http");
   tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e");
-  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse");
-  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse");
-  tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare");
+  if (enable_legacy == "true" || enable_legacy == "1") {
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare");
+  } else {
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfs");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfs");
+    tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "har");
+  }
   tensorflow::io::gs::ProvideFilesystemSupportFor(&info->ops[6], "gse");
 }

tensorflow_io/core/python/ops/version_ops.py

@@ -14,5 +14,5 @@
 # ==============================================================================
 """version_ops"""
 
-package = "tensorflow>=2.4.0,<2.5.0"
-version = "0.17.0"
+package = "tf-nightly"
+version = "0.18.0"

tests/test_hdfs.py

@@ -35,8 +35,8 @@ def test_read_file():
     print("ADDRESS: {}".format(address))
 
     body = b"1234567"
-    tf.io.write_file("hdfse://{}:9000/file.txt".format(address), body)
+    tf.io.write_file("hdfs://{}:9000/file.txt".format(address), body)
 
-    content = tf.io.read_file("hdfse://{}:9000/file.txt".format(address))
+    content = tf.io.read_file("hdfs://{}:9000/file.txt".format(address))
     print("CONTENT: {}".format(content))
     assert content == body