From c24ac35774a65a38bdd32ff2b7b7cee97f5da022 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 19 Dec 2020 19:42:10 +0530 Subject: [PATCH 01/15] HDFS-15740. Add x-platform utilities * Adding x-platform object library. It is a collection of components that will help in writing cross platform code. * Upgrading the C++ standard to 17 for this purpose. --- .../src/CMakeLists.txt | 2 ++ .../src/main/native/libhdfspp/CMakeLists.txt | 6 ++--- .../main/native/libhdfspp/lib/CMakeLists.txt | 1 + .../libhdfspp/lib/bindings/c/CMakeLists.txt | 7 ++--- .../native/libhdfspp/lib/bindings/c/hdfs.cc | 19 +++++--------- .../libhdfspp/lib/x-platform/CMakeLists.txt | 19 ++++++++++++++ .../native/libhdfspp/lib/x-platform/utils.cc | 23 ++++++++++++++++ .../native/libhdfspp/lib/x-platform/utils.h | 26 +++++++++++++++++++ .../native/libhdfspp/tests/CMakeLists.txt | 12 ++++----- 9 files changed, 89 insertions(+), 26 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt index 6b8a795204639..baf5f8cc30e79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt @@ -22,6 +22,8 @@ project(hadoop_hdfs_native_client) enable_testing() +set(CMAKE_CXX_STANDARD 17) + list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common) include(HadoopCommon) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 6528fa8897279..f19cb5bd7c5d7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -31,6 +31,7 @@ cmake_minimum_required(VERSION 2.8) find_package (Boost 1.72.0 REQUIRED) enable_testing() +set(CMAKE_CXX_STANDARD 17) include (CTest) SET(BUILD_SHARED_HDFSPP TRUE CACHE STRING "BUILD_SHARED_HDFSPP defaulting to 'TRUE'") @@ -78,7 +79,6 @@ add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src # Check if thread_local is supported unset (THREAD_LOCAL_SUPPORTED CACHE) -set (CMAKE_CXX_STANDARD 11) set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) check_cxx_source_compiles( @@ -175,13 +175,11 @@ set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0") if(UNIX) -set (CMAKE_CXX_STANDARD 11) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -g -fPIC -fno-strict-aliasing") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fPIC -fno-strict-aliasing") endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(CMAKE_CXX_STANDARD 11) add_definitions(-DASIO_HAS_STD_ADDRESSOF -DASIO_HAS_STD_ARRAY -DASIO_HAS_STD_ATOMIC -DASIO_HAS_CSTDINT -DASIO_HAS_STD_SHARED_PTR -DASIO_HAS_STD_TYPE_TRAITS -DASIO_HAS_VARIADIC_TEMPLATES -DASIO_HAS_STD_FUNCTION -DASIO_HAS_STD_CHRONO -DASIO_HAS_STD_SYSTEM_ERROR) endif () @@ -280,7 +278,7 @@ if(NEED_LINK_DL) endif() set(LIBHDFSPP_VERSION "0.1.0") -set(LIBHDFSPP_ALL_OBJECTS $ $ $ $ $ $ $ $) +set(LIBHDFSPP_ALL_OBJECTS $ $ $ $ $ $ $ $ $) if (HADOOP_BUILD) hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) hadoop_target_link_dual_libraries(hdfspp diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt index c8515979d75ce..db11bb9d3068c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt @@ -16,6 +16,7 @@ # limitations under the License. # +add_subdirectory(x-platform) add_subdirectory(common) add_subdirectory(fs) add_subdirectory(reader) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt index 98b081ff3350a..1339098df9cb6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. +add_library(bindings_c_obj OBJECT hdfs.cc $) +add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common x_platform_utils_obj) -add_library(bindings_c_obj OBJECT hdfs.cc) -add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common) -add_library(bindings_c $) +add_library(bindings_c $ $) +add_dependencies(bindings_c x_platform_utils_obj) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc index 6b2468fd5dbdc..7e5815d8a364a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc @@ -24,10 +24,9 @@ #include "common/logging.h" #include "fs/filesystem.h" #include "fs/filehandle.h" +#include "x-platform/utils.h" - -#include -#include "limits.h" +#include #include #include #include @@ -40,7 +39,7 @@ using namespace std::placeholders; static constexpr tPort kDefaultPort = 8020; -/** Annotate what parts of the code below are implementatons of API functions +/** Annotate what parts of the code below are implementations of API functions * and if they are normal vs. extended API. */ #define LIBHDFS_C_API @@ -767,15 +766,9 @@ void StatInfoToHdfsFileInfo(hdfsFileInfo * file_info, LOG_WARN(kFileSystem, << "Symlink is not supported! Reporting as a file: "); } - /* the name of the file */ - char copyOfPath[PATH_MAX]; - strncpy(copyOfPath, stat_info.path.c_str(), PATH_MAX); - copyOfPath[PATH_MAX - 1] = '\0'; // in case strncpy ran out of space - - char * mName = basename(copyOfPath); - size_t mName_size = strlen(mName); - file_info->mName = new char[mName_size+1]; - strncpy(file_info->mName, basename(copyOfPath), mName_size + 1); + const auto filename = XPlatform::Utils::Basename(stat_info.path); + file_info->mName = new char[filename.size() + 1]; + strncpy(file_info->mName, filename.c_str(), filename.size() + 1); /* the last modification time for the file in seconds */ file_info->mLastMod = (tTime) stat_info.modification_time; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt new file mode 100644 index 0000000000000..152cbcb21e407 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(x_platform_utils OBJECT utils.cc) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc new file mode 100644 index 0000000000000..7842573eda2b9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc @@ -0,0 +1,23 @@ +#include "x-platform/utils.h" + +#include +#include +#include + +std::string XPlatform::Utils::Basename(const std::string& file_path) { + if (file_path.empty()) { + return "."; + } + + const std::filesystem::path path(file_path); + std::vector parts; + for (const auto& part : std::filesystem::path(file_path)) { + parts.emplace_back(part.string()); + } + + /*Handle the case of trailing slash*/ + if (parts.back().empty()) { + parts.pop_back(); + } + return parts.back(); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h new file mode 100644 index 0000000000000..cafd6a748b29f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h @@ -0,0 +1,26 @@ +#ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS +#define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS + +#include + +/* + * The XPlatform namespace contains components that + * aid in writing cross-platform code. + */ +namespace XPlatform { +class Utils { + public: + /* + * A cross-platform implementation of basename in linux. + * Please refer https://www.man7.org/linux/man-pages/man3/basename.3.html + * for more details. + * + * @param file_path The input path to get the basename. + * + * @returns The trailing component of the given {@link file_path} + */ + static std::string Basename(const std::string& file_path); +}; +} // namespace XPlatform + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 2b2f4f16f1677..107d098d346dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -110,9 +110,10 @@ add_executable(hdfs_builder_test hdfs_builder_test.cc) target_link_libraries(hdfs_builder_test test_common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(hdfs_builder_test hdfs_builder_test) -add_executable(logging_test logging_test.cc) +add_executable(logging_test logging_test.cc $) target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(logging_test logging_test) +add_dependencies(logging_test x_platform_utils_obj) add_executable(hdfs_ioservice_test hdfs_ioservice_test.cc) target_link_libraries(hdfs_ioservice_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) @@ -142,12 +143,11 @@ include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/../../libhdfs-tests/ ) -add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc) -add_library(hdfspp_test_static STATIC ${LIBHDFSPP_BINDING_C}/hdfs.cc) +add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc $ ${LIBHDFSPP_BINDING_C}/hdfs.cc) +add_dependencies(hdfspp_test_shim_static proto x_platform_utils_obj) -# Add dependencies -add_dependencies(hdfspp_test_shim_static proto) -add_dependencies(hdfspp_test_static proto) +add_library(hdfspp_test_static STATIC $ ${LIBHDFSPP_BINDING_C}/hdfs.cc) +add_dependencies(hdfspp_test_static proto x_platform_utils_obj) # TODO: get all of the mini dfs library bits here in one place # add_library(hdfspp_mini_cluster native_mini_dfs ${JAVA_JVM_LIBRARY} ) From 9431b5263b6b79e0b207ff490d46d0d010197e60 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 19 Dec 2020 21:52:20 +0530 Subject: [PATCH 02/15] Add license header --- .../native/libhdfspp/lib/x-platform/utils.cc | 18 ++++++++++++++++++ .../native/libhdfspp/lib/x-platform/utils.h | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc index 7842573eda2b9..9c3adeb1f965d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include "x-platform/utils.h" #include diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h index cafd6a748b29f..b9650da77d5b4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS #define NATIVE_LIBHDFSPP_LIB_CROSS_PLATFORM_UTILS From 7a9b8e6c37c4f604ba0ae407c29fd114050ecc3f Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 19 Dec 2020 21:59:18 +0530 Subject: [PATCH 03/15] Fix typo in contribution guide --- .../src/main/native/libhdfspp/CONTRIBUTING.md | 8 +++----- .../src/main/native/libhdfspp/lib/x-platform/utils.cc | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CONTRIBUTING.md b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CONTRIBUTING.md index d36a38e972e88..0d081b2c1b034 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CONTRIBUTING.md +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CONTRIBUTING.md @@ -42,14 +42,12 @@ also be followed as well as portability requirements. Automated Formatting -------------------- -Prior to submitting a patch for code review use llvm's formatting tool, clang-format, on the .h, .c, and .cc files included in the patch. Use the -style=google switch when doing so. +Prior to submitting a patch for code review use LLVM's formatting tool, clang-format, on the .h, .c, and .cc files included in the patch. Use the -style=google switch when doing so. -Example presubmission usage: +Example pre-submission usage: ``` shell -cat my_source_file.cc | clang-format -style=goole > temp_file.cc -#optionally diff the source and temp file to get an idea what changed -mv temp_file.cc my_source_file.cc +$ clang-format -i -style=google temp_file.cc ``` * note: On some linux distributions clang-format already exists in repositories but don't show up without an appended version number. On Ubuntu you'll find it with: diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc index 9c3adeb1f965d..04c5a238a8c31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.cc @@ -33,7 +33,7 @@ std::string XPlatform::Utils::Basename(const std::string& file_path) { parts.emplace_back(part.string()); } - /*Handle the case of trailing slash*/ + /* Handle the case of trailing slash */ if (parts.back().empty()) { parts.pop_back(); } From 6f8a084d8de751e7decdb8446e41c23fd3a4d01f Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 19 Dec 2020 22:02:31 +0530 Subject: [PATCH 04/15] Fix documentation headers in utils --- .../src/main/native/libhdfspp/lib/x-platform/utils.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h index b9650da77d5b4..06c608ba969c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/utils.h @@ -21,14 +21,14 @@ #include -/* - * The XPlatform namespace contains components that +/** + * The {@link XPlatform} namespace contains components that * aid in writing cross-platform code. */ namespace XPlatform { class Utils { public: - /* + /** * A cross-platform implementation of basename in linux. * Please refer https://www.man7.org/linux/man-pages/man3/basename.3.html * for more details. From 2045d7737e672144bd8fb000d4bd3cb13d197563 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 20 Dec 2020 21:52:28 +0530 Subject: [PATCH 05/15] Fix CMakeLists.txt * Had missed renaming target name. Fixing it now. --- .../src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt | 3 +-- .../src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt index 1339098df9cb6..6dfeed0a580af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt @@ -18,5 +18,4 @@ add_library(bindings_c_obj OBJECT hdfs.cc $) add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common x_platform_utils_obj) -add_library(bindings_c $ $) -add_dependencies(bindings_c x_platform_utils_obj) +add_library(bindings_c $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt index 152cbcb21e407..2d81c031993f6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt @@ -16,4 +16,4 @@ # limitations under the License. # -add_library(x_platform_utils OBJECT utils.cc) +add_library(x_platform_utils_obj OBJECT utils.cc) From c79c795969bfc731462b06c63c4b45347636889a Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sun, 20 Dec 2020 23:06:46 +0530 Subject: [PATCH 06/15] Add correct linking * bindings_c_obj doesn't link with x_platform_utils_obj through object linking. Thus, need to use dynamic linking. --- .../src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt | 5 +++-- .../src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt index 6dfeed0a580af..1db7eb643cede 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -add_library(bindings_c_obj OBJECT hdfs.cc $) +add_library(bindings_c_obj OBJECT hdfs.cc) add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common x_platform_utils_obj) +target_link_libraries(bindings_c_obj PRIVATE x_platform_utils_obj) -add_library(bindings_c $) +add_library(bindings_c $ $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt index 2d81c031993f6..8bc80e74cf7e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt @@ -17,3 +17,4 @@ # add_library(x_platform_utils_obj OBJECT utils.cc) +add_library(x_platform_utils $) \ No newline at end of file From 30ffca47b6b1ece4447cc6dd58183974c1c9aadf Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Fri, 25 Dec 2020 15:15:13 +0530 Subject: [PATCH 07/15] Do not link obj lib * Object libraries can not be linked with or linked to. Thus, removing it. --- .../src/main/native/libhdfspp/CMakeLists.txt | 2 +- .../main/native/libhdfspp/lib/bindings/c/CMakeLists.txt | 3 +-- .../main/native/libhdfspp/lib/x-platform/CMakeLists.txt | 3 +-- .../src/main/native/libhdfspp/tests/CMakeLists.txt | 7 +++---- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index f19cb5bd7c5d7..5125f57ed0108 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -278,7 +278,7 @@ if(NEED_LINK_DL) endif() set(LIBHDFSPP_VERSION "0.1.0") -set(LIBHDFSPP_ALL_OBJECTS $ $ $ $ $ $ $ $ $) +set(LIBHDFSPP_ALL_OBJECTS $ $ $ $ $ $ $ $ $) if (HADOOP_BUILD) hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) hadoop_target_link_dual_libraries(hdfspp diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt index 1db7eb643cede..ff8d6b04465d9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt @@ -15,8 +15,7 @@ # specific language governing permissions and limitations # under the License. -add_library(bindings_c_obj OBJECT hdfs.cc) +add_library(bindings_c_obj OBJECT $ hdfs.cc) add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common x_platform_utils_obj) -target_link_libraries(bindings_c_obj PRIVATE x_platform_utils_obj) add_library(bindings_c $ $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt index 8bc80e74cf7e5..c9bc3df282408 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/CMakeLists.txt @@ -16,5 +16,4 @@ # limitations under the License. # -add_library(x_platform_utils_obj OBJECT utils.cc) -add_library(x_platform_utils $) \ No newline at end of file +add_library(x_platform_utils_obj OBJECT utils.cc) \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 107d098d346dd..9029efb44a7d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -113,7 +113,6 @@ add_memcheck_test(hdfs_builder_test hdfs_builder_test) add_executable(logging_test logging_test.cc $) target_link_libraries(logging_test common gmock_main bindings_c fs rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(logging_test logging_test) -add_dependencies(logging_test x_platform_utils_obj) add_executable(hdfs_ioservice_test hdfs_ioservice_test.cc) target_link_libraries(hdfs_ioservice_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${SASL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) @@ -143,11 +142,11 @@ include_directories ( ${CMAKE_CURRENT_SOURCE_DIR}/../../libhdfs-tests/ ) -add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc $ ${LIBHDFSPP_BINDING_C}/hdfs.cc) -add_dependencies(hdfspp_test_shim_static proto x_platform_utils_obj) +add_library(hdfspp_test_shim_static STATIC $ hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc) +add_dependencies(hdfspp_test_shim_static proto) add_library(hdfspp_test_static STATIC $ ${LIBHDFSPP_BINDING_C}/hdfs.cc) -add_dependencies(hdfspp_test_static proto x_platform_utils_obj) +add_dependencies(hdfspp_test_static proto) # TODO: get all of the mini dfs library bits here in one place # add_library(hdfspp_mini_cluster native_mini_dfs ${JAVA_JVM_LIBRARY} ) From c227d4af2e7936d01eb9daaba712f466aeacda71 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Fri, 25 Dec 2020 20:40:50 +0530 Subject: [PATCH 08/15] Upgrade CMake version to 3.19 * This upgrade is necessary for this PR as it enables x_platform_utils_obj to be used as part of the bindings_c_obj object library. This isn't supported in CMake 3.10 and the compilation would fail in the configuration step. --- dev-support/docker/Dockerfile | 1 - dev-support/docker/Dockerfile_aarch64 | 1 - 2 files changed, 2 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 4d9413b72f63b..ea24b4430f0a3 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -41,7 +41,6 @@ RUN apt-get -q update \ build-essential \ bzip2 \ clang \ - cmake \ curl \ doxygen \ findbugs \ diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index cfcfdcecccba5..f9ebe075f1fa1 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -45,7 +45,6 @@ RUN apt-get -q update \ build-essential \ bzip2 \ clang \ - cmake \ curl \ doxygen \ findbugs \ From 9d3b64373b393e1d9537f87baefddb640dcc7c7d Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Fri, 25 Dec 2020 23:21:42 +0530 Subject: [PATCH 09/15] Upgrade GCC to 9 * Need to use GCC 9 since this PR uses C++17 features. --- dev-support/docker/Dockerfile | 11 ++++++++--- dev-support/docker/Dockerfile_aarch64 | 11 ++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index ea24b4430f0a3..5c4471d84572c 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -34,6 +34,10 @@ ENV DEBCONF_TERSE true # hadolint ignore=DL3008 RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends \ + software-properties-common \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && apt-get -q update \ && apt-get -q install -y --no-install-recommends \ ant \ apt-utils \ @@ -45,8 +49,8 @@ RUN apt-get -q update \ doxygen \ findbugs \ fuse \ - g++ \ - gcc \ + g++-9 \ + gcc-9 \ git \ gnupg-agent \ hugo \ @@ -82,7 +86,8 @@ RUN apt-get -q update \ valgrind \ zlib1g-dev \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 ###### # Set env vars required to build Hadoop diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index f9ebe075f1fa1..f0d7f6c924531 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -38,6 +38,10 @@ ENV DEBCONF_TERSE true ###### # hadolint ignore=DL3008 RUN apt-get -q update \ + && apt-get -q install -y --no-install-recommends \ + software-properties-common \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && apt-get -q update \ && apt-get -q install -y --no-install-recommends \ ant \ apt-utils \ @@ -49,8 +53,8 @@ RUN apt-get -q update \ doxygen \ findbugs \ fuse \ - g++ \ - gcc \ + g++-9 \ + gcc-9 \ git \ gnupg-agent \ hugo \ @@ -86,7 +90,8 @@ RUN apt-get -q update \ valgrind \ zlib1g-dev \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 ###### # Set env vars required to build Hadoop From 6a74a73592050fc46ca3244ee1cd541b381bc914 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 26 Dec 2020 19:53:36 +0530 Subject: [PATCH 10/15] Add unit tests * Adding unit tests for x-platform/utils. --- .../native/libhdfspp/tests/CMakeLists.txt | 2 + .../libhdfspp/tests/x-platform/CMakeLists.txt | 22 +++++ .../libhdfspp/tests/x-platform/utils_test.cc | 87 +++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test.cc diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 9029efb44a7d8..0d877a2c4558f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -65,6 +65,8 @@ endfunction(add_memcheck_test) # # +add_subdirectory(x-platform) + add_executable(uri_test uri_test.cc) target_link_libraries(uri_test common gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(uri uri_test) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt new file mode 100644 index 0000000000000..d81908903bfba --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_executable(x_platform_utils_test $ utils_test.cc) +target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR}) +target_link_libraries(x_platform_utils_test gmock_main) +add_test(x_platform_utils_test x_platform_utils_test) \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test.cc new file mode 100644 index 0000000000000..1af829ccb7c0b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test.cc @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "x-platform/utils.h" + +#include + +#include + +TEST(XPlatformUtils, BasenameEmpty) { + const std::string expected("."); + const auto actual = XPlatform::Utils::Basename(""); + EXPECT_EQ(expected, actual); +} + +TEST(XPlatformUtils, BasenameRelativePath) { + const std::string expected("x"); + const auto actual = XPlatform::Utils::Basename("x"); + EXPECT_EQ(expected, actual); +} + +TEST(XPlatformUtils, BasenameRoot) { + const std::string win_expected(R"(\)"); + const auto win_actual_1 = XPlatform::Utils::Basename(R"(\)"); + EXPECT_EQ(win_expected, win_actual_1); + + const auto win_actual_2 = XPlatform::Utils::Basename(R"(C:\)"); + EXPECT_EQ(win_expected, win_actual_2); + + const std::string nix_expected("/"); + const auto nix_actual = XPlatform::Utils::Basename("/"); + EXPECT_EQ(nix_expected, nix_actual); +} + +TEST(XPlatformUtils, BasenameSpecialFiles) { + const std::string current_dir_expected("."); + const auto current_dir_actual = XPlatform::Utils::Basename("."); + EXPECT_EQ(current_dir_expected, current_dir_actual); + + const std::string parent_dir_expected(".."); + const auto parent_dir_actual = XPlatform::Utils::Basename(".."); + EXPECT_EQ(parent_dir_expected, parent_dir_actual); +} + +TEST(XPlatformUtils, BasenameTrailingSlash) { + const std::string expected("def"); + + const std::string win_path(R"(C:\abc\def\)"); + const auto win_actual = XPlatform::Utils::Basename(win_path); + EXPECT_EQ(expected, win_actual); + + const std::string nix_path("/abc/def/"); + const auto nix_actual = XPlatform::Utils::Basename(nix_path); + EXPECT_EQ(expected, nix_actual); +} + +TEST(XPlatformUtils, BasenameBasic) { + const std::string expected("def"); + + const std::string win_path(R"(C:\abc\def)"); + const auto win_actual = XPlatform::Utils::Basename(win_path); + EXPECT_EQ(expected, win_actual); + + const std::string nix_path("/abc/def"); + const auto nix_actual = XPlatform::Utils::Basename(nix_path); + EXPECT_EQ(expected, nix_actual); +} + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 0812534fd86af7f3aec4a11b51fb83b132e901df Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Sat, 26 Dec 2020 23:40:00 +0530 Subject: [PATCH 11/15] Fix unit tests * Separated the test files for the appropriate platforms. --- .../libhdfspp/tests/x-platform/CMakeLists.txt | 8 ++- .../{utils_test.cc => utils_common_test.cc} | 46 +---------------- .../tests/x-platform/utils_nix_test.cc | 43 ++++++++++++++++ .../tests/x-platform/utils_test_main.cc | 24 +++++++++ .../tests/x-platform/utils_win_test.cc | 49 +++++++++++++++++++ 5 files changed, 124 insertions(+), 46 deletions(-) rename hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/{utils_test.cc => utils_common_test.cc} (55%) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_nix_test.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test_main.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_win_test.cc diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt index d81908903bfba..5a949387d687c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt @@ -16,7 +16,11 @@ # limitations under the License. # -add_executable(x_platform_utils_test $ utils_test.cc) +if(WIN32) + add_executable(x_platform_utils_test $ utils_common_test.cc utils_test_main.cc utils_win_test.cc) +else(WIN32) + add_executable(x_platform_utils_test $ utils_common_test.cc utils_test_main.cc utils_nix_test.cc) +endif(WIN32) target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR}) target_link_libraries(x_platform_utils_test gmock_main) -add_test(x_platform_utils_test x_platform_utils_test) \ No newline at end of file +add_test(x_platform_utils_test x_platform_utils_test) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_common_test.cc similarity index 55% rename from hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test.cc rename to hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_common_test.cc index 1af829ccb7c0b..ae8d4505d299b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_common_test.cc @@ -16,12 +16,12 @@ * limitations under the License. */ -#include "x-platform/utils.h" - #include #include +#include "x-platform/utils.h" + TEST(XPlatformUtils, BasenameEmpty) { const std::string expected("."); const auto actual = XPlatform::Utils::Basename(""); @@ -34,19 +34,6 @@ TEST(XPlatformUtils, BasenameRelativePath) { EXPECT_EQ(expected, actual); } -TEST(XPlatformUtils, BasenameRoot) { - const std::string win_expected(R"(\)"); - const auto win_actual_1 = XPlatform::Utils::Basename(R"(\)"); - EXPECT_EQ(win_expected, win_actual_1); - - const auto win_actual_2 = XPlatform::Utils::Basename(R"(C:\)"); - EXPECT_EQ(win_expected, win_actual_2); - - const std::string nix_expected("/"); - const auto nix_actual = XPlatform::Utils::Basename("/"); - EXPECT_EQ(nix_expected, nix_actual); -} - TEST(XPlatformUtils, BasenameSpecialFiles) { const std::string current_dir_expected("."); const auto current_dir_actual = XPlatform::Utils::Basename("."); @@ -56,32 +43,3 @@ TEST(XPlatformUtils, BasenameSpecialFiles) { const auto parent_dir_actual = XPlatform::Utils::Basename(".."); EXPECT_EQ(parent_dir_expected, parent_dir_actual); } - -TEST(XPlatformUtils, BasenameTrailingSlash) { - const std::string expected("def"); - - const std::string win_path(R"(C:\abc\def\)"); - const auto win_actual = XPlatform::Utils::Basename(win_path); - EXPECT_EQ(expected, win_actual); - - const std::string nix_path("/abc/def/"); - const auto nix_actual = XPlatform::Utils::Basename(nix_path); - EXPECT_EQ(expected, nix_actual); -} - -TEST(XPlatformUtils, BasenameBasic) { - const std::string expected("def"); - - const std::string win_path(R"(C:\abc\def)"); - const auto win_actual = XPlatform::Utils::Basename(win_path); - EXPECT_EQ(expected, win_actual); - - const std::string nix_path("/abc/def"); - const auto nix_actual = XPlatform::Utils::Basename(nix_path); - EXPECT_EQ(expected, nix_actual); -} - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_nix_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_nix_test.cc new file mode 100644 index 0000000000000..6827f0034611b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_nix_test.cc @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "x-platform/utils.h" + +TEST(XPlatformUtils, BasenameRoot) { + const std::string nix_expected("/"); + const auto nix_actual = XPlatform::Utils::Basename("/"); + EXPECT_EQ(nix_expected, nix_actual); +} + +TEST(XPlatformUtils, BasenameTrailingSlash) { + const std::string expected("def"); + const std::string nix_path("/abc/def/"); + const auto nix_actual = XPlatform::Utils::Basename(nix_path); + EXPECT_EQ(expected, nix_actual); +} + +TEST(XPlatformUtils, BasenameBasic) { + const std::string expected("def"); + const std::string nix_path("/abc/def"); + const auto nix_actual = XPlatform::Utils::Basename(nix_path); + EXPECT_EQ(expected, nix_actual); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test_main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test_main.cc new file mode 100644 index 0000000000000..7fe49a2ce4286 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_test_main.cc @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +int main(int argc, char* argv[]) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_win_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_win_test.cc new file mode 100644 index 0000000000000..136ac3be1a9c0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/utils_win_test.cc @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "x-platform/utils.h" + +TEST(XPlatformUtils, BasenameRoot) { + const std::string win_expected(R"(\)"); + const auto win_actual = XPlatform::Utils::Basename(R"(\)"); + EXPECT_EQ(win_expected, win_actual); +} + +TEST(XPlatformUtils, BasenameRootLabel) { + const std::string win_expected(R"(C:\)"); + const auto win_actual = XPlatform::Utils::Basename(R"(C:\)"); + EXPECT_EQ(win_expected, win_actual); +} + +TEST(XPlatformUtils, BasenameTrailingSlash) { + const std::string expected("def"); + const std::string win_path(R"(C:\abc\def\)"); + const auto win_actual = XPlatform::Utils::Basename(win_path); + EXPECT_EQ(expected, win_actual); +} + +TEST(XPlatformUtils, BasenameBasic) { + const std::string expected("def"); + const std::string win_path(R"(C:\abc\def)"); + const auto win_actual = XPlatform::Utils::Basename(win_path); + EXPECT_EQ(expected, win_actual); +} From 9306a6f4abe0442ba0c2db8f91c8f98b041eafad Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Tue, 26 Jan 2021 11:05:16 +0530 Subject: [PATCH 12/15] Revert "Upgrade CMake version to 3.19" This reverts commit 8587ccdc988487069936789588566b84710af67c. --- dev-support/docker/Dockerfile | 1 + dev-support/docker/Dockerfile_aarch64 | 1 + 2 files changed, 2 insertions(+) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 5c4471d84572c..f5f593fea43a6 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -45,6 +45,7 @@ RUN apt-get -q update \ build-essential \ bzip2 \ clang \ + cmake \ curl \ doxygen \ findbugs \ diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index f0d7f6c924531..4e056f7ae29e8 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -49,6 +49,7 @@ RUN apt-get -q update \ build-essential \ bzip2 \ clang \ + cmake \ curl \ doxygen \ findbugs \ From eea44482d59567c89ac6e5d6b1d3aadbf31814fe Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Tue, 26 Jan 2021 11:06:00 +0530 Subject: [PATCH 13/15] Revert "Upgrade GCC to 9" This reverts commit fa36f845911dfebb205aca42b077cc08006a91f4. --- dev-support/docker/Dockerfile | 11 +++-------- dev-support/docker/Dockerfile_aarch64 | 11 +++-------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index f5f593fea43a6..4d9413b72f63b 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -34,10 +34,6 @@ ENV DEBCONF_TERSE true # hadolint ignore=DL3008 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - software-properties-common \ - && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ - && apt-get -q update \ && apt-get -q install -y --no-install-recommends \ ant \ apt-utils \ @@ -50,8 +46,8 @@ RUN apt-get -q update \ doxygen \ findbugs \ fuse \ - g++-9 \ - gcc-9 \ + g++ \ + gcc \ git \ gnupg-agent \ hugo \ @@ -87,8 +83,7 @@ RUN apt-get -q update \ valgrind \ zlib1g-dev \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 + && rm -rf /var/lib/apt/lists/* ###### # Set env vars required to build Hadoop diff --git a/dev-support/docker/Dockerfile_aarch64 b/dev-support/docker/Dockerfile_aarch64 index 4e056f7ae29e8..cfcfdcecccba5 100644 --- a/dev-support/docker/Dockerfile_aarch64 +++ b/dev-support/docker/Dockerfile_aarch64 @@ -38,10 +38,6 @@ ENV DEBCONF_TERSE true ###### # hadolint ignore=DL3008 RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends \ - software-properties-common \ - && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ - && apt-get -q update \ && apt-get -q install -y --no-install-recommends \ ant \ apt-utils \ @@ -54,8 +50,8 @@ RUN apt-get -q update \ doxygen \ findbugs \ fuse \ - g++-9 \ - gcc-9 \ + g++ \ + gcc \ git \ gnupg-agent \ hugo \ @@ -91,8 +87,7 @@ RUN apt-get -q update \ valgrind \ zlib1g-dev \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 + && rm -rf /var/lib/apt/lists/* ###### # Set env vars required to build Hadoop From d17cb914dd5210cc7b868093324bb3003162aee4 Mon Sep 17 00:00:00 2001 From: Gautham Banasandra Date: Wed, 27 Jan 2021 09:48:38 +0530 Subject: [PATCH 14/15] Update version in BUILDING.txt * Updated the BUILDING.txt document regarding GCC and CMake versions. --- BUILDING.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index c34946aa993b7..4167404e8e7e8 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -8,10 +8,10 @@ Requirements: * Maven 3.3 or later * Boost 1.72 (if compiling native code) * Protocol Buffers 3.7.1 (if compiling native code) -* CMake 3.1 or newer (if compiling native code) +* CMake 3.19 or newer (if compiling native code) * Zlib devel (if compiling native code) * Cyrus SASL devel (if compiling native code) -* One of the compilers that support thread_local storage: GCC 4.8.1 or later, Visual Studio, +* One of the compilers that support thread_local storage: GCC 9.3.0 or later, Visual Studio, Clang (community version), Clang (version for iOS 9 and later) (if compiling native code) * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance) * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs) From 4c70a7182d5fe756f9a55c448e96ccc7519ac21b Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Wed, 27 Jan 2021 09:58:51 +0530 Subject: [PATCH 15/15] Update build instructions for Windows * Updated the CMake version required for Windows. --- BUILDING.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILDING.txt b/BUILDING.txt index 4167404e8e7e8..2753c6fd4ca90 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -489,7 +489,7 @@ Requirements: * Maven 3.0 or later * Boost 1.72 * Protocol Buffers 3.7.1 -* CMake 3.1 or newer +* CMake 3.19 or newer * Visual Studio 2010 Professional or Higher * Windows SDK 8.1 (if building CPU rate control for the container executor) * zlib headers (if building native code bindings for zlib)