From 2d95471d78c949cb74e20de31cd705f712c1f3f8 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 8 Feb 2021 19:47:04 -0800 Subject: [PATCH 1/4] Switch to modular file system for hdfs This PR is part of the effort to switch to modular file system for hdfs. When TF_ENABLE_LEGACY_FILESYSTEM=1 is provided, old behavior will be preserved. Signed-off-by: Yong Tang --- .../core/plugins/file_system_plugins.cc | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tensorflow_io/core/plugins/file_system_plugins.cc b/tensorflow_io/core/plugins/file_system_plugins.cc index bc408dcb7..fe64df7b2 100644 --- a/tensorflow_io/core/plugins/file_system_plugins.cc +++ b/tensorflow_io/core/plugins/file_system_plugins.cc @@ -15,7 +15,13 @@ limitations under the License. #include "tensorflow_io/core/plugins/file_system_plugins.h" +#include "absl/strings/ascii.h" + void TF_InitPlugin(TF_FilesystemPluginInfo* info) { + const char* enable_legacy_env = getenv("TF_ENABLE_LEGACY_FILESYSTEM"); + std::string enable_legacy = + enable_legacy_env ? absl::AsciiStrToLower(enable_legacy_env) : ""; + info->plugin_memory_allocate = tensorflow::io::plugin_memory_allocate; info->plugin_memory_free = tensorflow::io::plugin_memory_free; info->num_schemes = 7; @@ -25,8 +31,14 @@ void TF_InitPlugin(TF_FilesystemPluginInfo* info) { tensorflow::io::az::ProvideFilesystemSupportFor(&info->ops[0], "az"); tensorflow::io::http::ProvideFilesystemSupportFor(&info->ops[1], "http"); tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e"); - tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse"); - tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse"); - tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare"); + if (!(enable_legacy == "true" || enable_legacy == "1")) { + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfs"); + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfs"); + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "har"); + } else { + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse"); + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse"); + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare"); + } tensorflow::io::gs::ProvideFilesystemSupportFor(&info->ops[6], "gse"); } From 255b2862b81f09842a36ccc459da9296800dc335 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 8 Feb 2021 20:00:34 -0800 Subject: [PATCH 2/4] Build against tf-nightly Signed-off-by: Yong Tang --- tensorflow_io/core/python/ops/version_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_io/core/python/ops/version_ops.py b/tensorflow_io/core/python/ops/version_ops.py index d6d6e3122..e61f2470d 100644 --- a/tensorflow_io/core/python/ops/version_ops.py +++ b/tensorflow_io/core/python/ops/version_ops.py @@ -14,5 +14,5 @@ # ============================================================================== """version_ops""" -package = "tensorflow>=2.4.0,<2.5.0" -version = "0.17.0" +package = "tf-nightly" +version = "0.18.0" From 9e1782ed18bd3c7e2a56a749aaae98bf76f1ac22 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 9 Feb 2021 13:52:20 -0800 Subject: [PATCH 3/4] Update tests Signed-off-by: Yong Tang --- tests/test_hdfs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_hdfs.py b/tests/test_hdfs.py index 468d04b06..1324a597d 100644 --- a/tests/test_hdfs.py +++ b/tests/test_hdfs.py @@ -35,8 +35,8 @@ def test_read_file(): print("ADDRESS: {}".format(address)) body = b"1234567" - tf.io.write_file("hdfse://{}:9000/file.txt".format(address), body) + tf.io.write_file("hdfs://{}:9000/file.txt".format(address), body) - content = tf.io.read_file("hdfse://{}:9000/file.txt".format(address)) + content = tf.io.read_file("hdfs://{}:9000/file.txt".format(address)) print("CONTENT: {}".format(content)) assert content == body From dfbb82f17c4cbd50d1722f6847eaf39d8c8e83d5 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Thu, 11 Feb 2021 11:59:11 -0800 Subject: [PATCH 4/4] Adjust the if else logic, follow review comment Signed-off-by: Yong Tang --- tensorflow_io/core/plugins/file_system_plugins.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_io/core/plugins/file_system_plugins.cc b/tensorflow_io/core/plugins/file_system_plugins.cc index fe64df7b2..447e8757a 100644 --- a/tensorflow_io/core/plugins/file_system_plugins.cc +++ b/tensorflow_io/core/plugins/file_system_plugins.cc @@ -31,14 +31,14 @@ void TF_InitPlugin(TF_FilesystemPluginInfo* info) { tensorflow::io::az::ProvideFilesystemSupportFor(&info->ops[0], "az"); tensorflow::io::http::ProvideFilesystemSupportFor(&info->ops[1], "http"); tensorflow::io::s3::ProvideFilesystemSupportFor(&info->ops[2], "s3e"); - if (!(enable_legacy == "true" || enable_legacy == "1")) { - tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfs"); - tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfs"); - tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "har"); - } else { + if (enable_legacy == "true" || enable_legacy == "1") { tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfse"); tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfse"); tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "hare"); + } else { + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[3], "hdfs"); + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[4], "viewfs"); + tensorflow::io::hdfs::ProvideFilesystemSupportFor(&info->ops[5], "har"); } tensorflow::io::gs::ProvideFilesystemSupportFor(&info->ops[6], "gse"); }