diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt index b5c21c7915baf..210243d61b233 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt @@ -23,6 +23,9 @@ add_executable(hdfs_tool_tests hdfs-delete-snapshot-mock.cc hdfs-create-snapshot-mock.cc hdfs-cat-mock.cc + hdfs-chown-mock.cc + hdfs-chmod-mock.cc + hdfs-chgrp-mock.cc hdfs-tool-test-fixtures.cc hdfs-tool-tests.cc hdfs-df-mock.cc @@ -36,6 +39,9 @@ target_include_directories(hdfs_tool_tests PRIVATE ../../tools/hdfs-delete-snapshot ../../tools/hdfs-create-snapshot ../../tools/hdfs-rename-snapshot + ../../tools/hdfs-chown + ../../tools/hdfs-chgrp + ../../tools/hdfs-chmod ../../tools/hdfs-cat) target_link_libraries(hdfs_tool_tests PRIVATE gmock_main @@ -45,5 +51,8 @@ target_link_libraries(hdfs_tool_tests PRIVATE hdfs_deleteSnapshot_lib hdfs_createSnapshot_lib hdfs_renameSnapshot_lib + hdfs_chown_lib + hdfs_chgrp_lib + hdfs_chmod_lib hdfs_cat_lib) add_test(hdfs_tool_tests hdfs_tool_tests) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc new file mode 100644 index 0000000000000..5948a7cc942bd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.cc @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-chgrp-mock.h" +#include "hdfs-chgrp.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +ChgrpMock::~ChgrpMock() = default; + +void ChgrpMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassOwnerAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + + EXPECT_CALL(*this, HandlePath(arg1, false, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursiveOwnerAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + + EXPECT_CALL(*this, HandlePath(arg1, true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h new file mode 100644 index 0000000000000..a7fb95cc81723 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chgrp-mock.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHGRP_MOCK +#define LIBHDFSPP_TOOLS_HDFS_CHGRP_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-chgrp.h" + +namespace hdfs::tools::test { +/** + * {@class ChgrpMock} is an {@class Chgrp} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class ChgrpMock : public hdfs::tools::Chgrp { +public: + /** + * {@inheritdoc} + */ + ChgrpMock(const int argc, char **argv) : Chgrp(argc, argv) {} + + // Abiding to the Rule of 5 + ChgrpMock(const ChgrpMock &) = delete; + ChgrpMock(ChgrpMock &&) = delete; + ChgrpMock &operator=(const ChgrpMock &) = delete; + ChgrpMock &operator=(ChgrpMock &&) = delete; + ~ChgrpMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, + (const std::string &, bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc new file mode 100644 index 0000000000000..e0df32cc2a16a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.cc @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-chmod-mock.h" +#include "hdfs-chmod.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +ChmodMock::~ChmodMock() = default; + +void ChmodMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassPermissionsAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + + EXPECT_CALL(*this, HandlePath(arg1, false, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursivePermissionsAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + + EXPECT_CALL(*this, HandlePath(arg1, true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h new file mode 100644 index 0000000000000..cd193b9048914 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chmod-mock.h @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHMOD_MOCK +#define LIBHDFSPP_TOOLS_HDFS_CHMOD_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-chmod.h" + +namespace hdfs::tools::test { +/** + * {@class ChmodMock} is an {@class Chmod} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class ChmodMock : public hdfs::tools::Chmod { +public: + /** + * {@inheritdoc} + */ + ChmodMock(const int argc, char **argv) : Chmod(argc, argv) {} + + // Abiding to the Rule of 5 + ChmodMock(const ChmodMock &) = delete; + ChmodMock(ChmodMock &&) = delete; + ChmodMock &operator=(const ChmodMock &) = delete; + ChmodMock &operator=(ChmodMock &&) = delete; + ~ChmodMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, + (const std::string &, bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc new file mode 100644 index 0000000000000..a126e9c6fe29a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.cc @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-chown-mock.h" +#include "hdfs-chown.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +ChownMock::~ChownMock() = default; + +void ChownMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassOwnerAndAPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + const Ownership ownership(arg1); + + EXPECT_CALL(*this, HandlePath(ownership, false, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursiveOwnerAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + const Ownership ownership(arg1); + + EXPECT_CALL(*this, HandlePath(ownership, true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h new file mode 100644 index 0000000000000..5cde9416bb6d4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-chown-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHOWN_MOCK +#define LIBHDFSPP_TOOLS_HDFS_CHOWN_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-chown.h" + +namespace hdfs::tools::test { +/** + * {@class ChownMock} is an {@class Chown} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class ChownMock : public hdfs::tools::Chown { +public: + /** + * {@inheritdoc} + */ + ChownMock(const int argc, char **argv) : Chown(argc, argv) {} + + // Abiding to the Rule of 5 + ChownMock(const ChownMock &) = delete; + ChownMock(ChownMock &&) = delete; + ChownMock &operator=(const ChownMock &) = delete; + ChownMock &operator=(ChownMock &&) = delete; + ~ChownMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const Ownership &, bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc index 33480a933df0b..1a879aa2fe60f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc @@ -21,6 +21,9 @@ #include "hdfs-allow-snapshot-mock.h" #include "hdfs-cat-mock.h" +#include "hdfs-chgrp-mock.h" +#include "hdfs-chmod-mock.h" +#include "hdfs-chown-mock.h" #include "hdfs-create-snapshot-mock.h" #include "hdfs-delete-snapshot-mock.h" #include "hdfs-df-mock.h" @@ -69,6 +72,25 @@ INSTANTIATE_TEST_SUITE_P( testing::Values(CallHelp, Pass2Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsChown, HdfsToolBasicTest, + testing::Values(CallHelp, + PassOwnerAndAPath, + PassRecursiveOwnerAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChmod, HdfsToolBasicTest, + testing::Values( + CallHelp, + PassPermissionsAndAPath, + PassRecursivePermissionsAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChgrp, HdfsToolBasicTest, + testing::Values(CallHelp, + PassOwnerAndAPath, + PassRecursiveOwnerAndAPath)); + // Negative tests INSTANTIATE_TEST_SUITE_P( HdfsAllowSnapshot, HdfsToolNegativeTestThrows, @@ -99,3 +121,27 @@ INSTANTIATE_TEST_SUITE_P( INSTANTIATE_TEST_SUITE_P( HdfsDeleteSnapshot, HdfsToolNegativeTestNoThrow, testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChown, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChown, HdfsToolNegativeTestThrows, + testing::Values(PassNOptAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChmod, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChmod, HdfsToolNegativeTestThrows, + testing::Values(PassNOptAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChgrp, HdfsToolNegativeTestNoThrow, + testing::Values(PassAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsChgrp, HdfsToolNegativeTestThrows, + testing::Values(PassNOptAndAPath)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h index 188a76c439478..9d012e3d37be1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h @@ -97,4 +97,59 @@ template std::unique_ptr PassNOptAndAPath() { return hdfs_tool; } +template std::unique_ptr PassOwnerAndAPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("new_owner:new_group"); + static std::string arg2("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassOwnerAndAPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassRecursiveOwnerAndAPath() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-R"); + static std::string arg2("new_owner:new_group"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassRecursiveOwnerAndAPath, {arg1, arg2, arg3}); + return hdfs_tool; +} + +template std::unique_ptr PassPermissionsAndAPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("757"); + static std::string arg2("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassPermissionsAndAPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassRecursivePermissionsAndAPath() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-R"); + static std::string arg2("757"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassRecursivePermissionsAndAPath, + {arg1, arg2, arg3}); + return hdfs_tool; +} + #endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt index c4aed54eeabb7..e93322e09a0a7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt @@ -28,19 +28,18 @@ link_directories( ${LIBHDFSPP_DIR}/lib ) add_library(tools_common_obj OBJECT tools_common.cc) add_library(tools_common $) +add_subdirectory(internal) + add_library(hdfs_tool_obj OBJECT hdfs-tool.cc) target_include_directories(hdfs_tool_obj PRIVATE ../tools) add_subdirectory(hdfs-cat) -add_executable(hdfs_chgrp hdfs_chgrp.cc) -target_link_libraries(hdfs_chgrp tools_common hdfspp_static) +add_subdirectory(hdfs-chgrp) -add_executable(hdfs_chown hdfs_chown.cc) -target_link_libraries(hdfs_chown tools_common hdfspp_static) +add_subdirectory(hdfs-chown) -add_executable(hdfs_chmod hdfs_chmod.cc) -target_link_libraries(hdfs_chmod tools_common hdfspp_static) +add_subdirectory(hdfs-chmod) add_executable(hdfs_find hdfs_find.cc) target_link_libraries(hdfs_find tools_common hdfspp_static) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt new file mode 100644 index 0000000000000..101365900a404 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_chgrp_lib STATIC $ $ hdfs-chgrp.cc) +target_include_directories(hdfs_chgrp_lib PRIVATE ../../tools hdfs-chgrp ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_chgrp_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_chgrp main.cc) +target_include_directories(hdfs_chgrp PRIVATE ../../tools) +target_link_libraries(hdfs_chgrp PRIVATE hdfs_chgrp_lib) + +install(TARGETS hdfs_chgrp RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc new file mode 100644 index 0000000000000..cbb3b4514b13d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.cc @@ -0,0 +1,220 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-chgrp.h" +#include "internal/hdfs-ownership.h" +#include "tools_common.h" + +namespace hdfs::tools { +Chgrp::Chgrp(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Chgrp::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Change the group association of each FILE to GROUP."); + add_options("file", po::value(), + "The path to the file whose group needs to be modified"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options( + "group", po::value(), + "The group to which the file's group association needs to be changed to"); + + // An exception is thrown if these arguments are missing or if the arguments' + // count doesn't tally. + pos_opt_desc_.add("group", 1); + pos_opt_desc_.add("file", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool Chgrp::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help"); + } + + // Rest of the cases must contain more than 2 arguments on the command line + return argc_ > 2; +} + +std::string Chgrp ::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_chgrp [OPTION] GROUP FILE" << std::endl + << std::endl + << "Change the group association of each FILE to GROUP." << std::endl + << "The user must be the owner of files. Additional information is in " + "the Permissions Guide:" + << std::endl + << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/" + "hadoop-hdfs/HdfsPermissionsGuide.html" + << std::endl + << std::endl + << " -R operate on files and directories recursively" << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:8020/dir/file" + << std::endl + << "hdfs_chgrp new_group /dir/file" << std::endl; + return desc.str(); +} + +bool Chgrp::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS chgrp tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("file") > 0 && opt_val_.count("group") > 0) { + const auto file = opt_val_["file"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + const auto group = opt_val_["group"].as(); + return HandlePath(group, recursive, file); + } + + return true; +} + +bool Chgrp::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Chgrp::HandlePath(const std::string &group, const bool recursive, + const std::string &file) const { + // Building a URI object from the given file + auto uri = hdfs::parse_path_or_exit(file); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + // Wrap async FileSystem::SetOwner with promise to make it a blocking call + const auto promise = std::make_shared>(); + auto future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + if (!recursive) { + fs->SetOwner(uri.get_path(), "", group, handler); + } else { + /* + * Allocating shared state, which includes: username and groupname to be + * set, handler to be called, request counter, and a boolean to keep track + * if find is done + */ + const auto state = + std::make_shared("", group, handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking + * in handler_find. + */ + auto handler_find = [fs, + state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async SetOwner with the + * handler below. SetOwner DOES NOT guarantee that the handler will only + * be called once at a time, so we DO need locking in handler_set_owner. + */ + auto handler_set_owner = [state](const hdfs::Status &status_set_owner) { + std::lock_guard guard(state->lock); + + // Decrement the counter once since we are done with this async call + if (!status_set_owner.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_set_owner; + } + + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // exit + } + }; + + if (!stat_infos.empty() && state->status.ok()) { + for (const auto &s : stat_infos) { + // Launch an asynchronous call to SetOwner for every returned result + state->request_counter++; + fs->SetOwner(s.full_path, state->user, state->group, + handler_set_owner); + } + } + + /* + * Lock this section because handler_set_owner might be accessing the same + * shared variables simultaneously. + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // exit + } + return false; + } + return true; + }; + + // Asynchronous call to Find + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + } + + // Block until promise is set + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h new file mode 100644 index 0000000000000..bd1920ff07492 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/hdfs-chgrp.h @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHGRP +#define LIBHDFSPP_TOOLS_HDFS_CHGRP + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Chgrp} is an {@class HdfsTool} that changes the owner and/or group of + * each file to owner and/or group. + */ +class Chgrp : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Chgrp(int argc, char **argv); + + // Abiding to the Rule of 5 + Chgrp(const Chgrp &) = default; + Chgrp(Chgrp &&) = default; + Chgrp &operator=(const Chgrp &) = delete; + Chgrp &operator=(Chgrp &&) = delete; + ~Chgrp() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path to the file argument that's passed to this tool. + * + * @param group The name of the group to which to change to. + * @param recursive Whether this operation needs to be performed recursively + * on all the files in the given path's sub-directory. + * @param file The path to the file whose group needs to be changed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &group, + bool recursive, + const std::string &file) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc new file mode 100644 index 0000000000000..52f36748b5e46 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chgrp/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-chgrp.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chgrp " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Chgrp chgrp(argc, argv); + auto success = false; + + try { + success = chgrp.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt new file mode 100644 index 0000000000000..a1e17f87ebac5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/CMakeLists.txt @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_chmod_lib STATIC $ $ hdfs-chmod.cc) +target_include_directories(hdfs_chmod_lib PRIVATE ../../tools hdfs-chmod ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_chmod_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_chmod main.cc) +target_include_directories(hdfs_chmod PRIVATE ../../tools) +target_link_libraries(hdfs_chmod PRIVATE hdfs_chmod_lib) + +install(TARGETS hdfs_chmod RUNTIME DESTINATION bin) + diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc new file mode 100644 index 0000000000000..4775860fb4483 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.cc @@ -0,0 +1,232 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-chmod.h" +#include "tools_common.h" + +namespace hdfs::tools { +Chmod::Chmod(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Chmod::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Change the permissions of each FILE to MODE."); + add_options("file", po::value(), + "The path to the file whose permissions needs to be modified"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options("permissions", po::value(), + "Octal representation of the permission bits"); + + // An exception is thrown if these arguments are missing or if the arguments' + // count doesn't tally. + pos_opt_desc_.add("permissions", 1); + pos_opt_desc_.add("file", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool Chmod::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help"); + } + + // Rest of the cases must contain more than 2 arguments on the command line + return argc_ > 2; +} + +std::string Chmod ::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_chmod [OPTION] FILE" + << std::endl + << std::endl + << "Change the permissions of each FILE to MODE." << std::endl + << "The user must be the owner of the file, or else a super-user." + << std::endl + << "Additional information is in the Permissions Guide:" << std::endl + << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/" + "hadoop-hdfs/HdfsPermissionsGuide.html" + << std::endl + << std::endl + << " -R operate on files and directories recursively" << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_chmod -R 755 hdfs://localhost.localdomain:8020/dir/file" + << std::endl + << "hdfs_chmod 777 /dir/file" << std::endl; + return desc.str(); +} + +bool Chmod::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS chmod tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("file") > 0 && opt_val_.count("permissions") > 0) { + const auto file = opt_val_["file"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + const auto permissions = opt_val_["permissions"].as(); + return HandlePath(permissions, recursive, file); + } + + return true; +} + +bool Chmod::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Chmod::HandlePath(const std::string &permissions, const bool recursive, + const std::string &file) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(file); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + /* + * Wrap async FileSystem::SetPermission with promise to make it a blocking + * call. + */ + const auto promise = std::make_shared>(); + auto future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + /* + * strtol is reading the value with base 8, NULL because we are reading in + * just one value. + */ + auto perm = static_cast(strtol(permissions.c_str(), nullptr, 8)); + if (!recursive) { + fs->SetPermission(uri.get_path(), perm, handler); + } else { + /* + * Allocating shared state, which includes - + * 1. Permissions to be set + * 2. Handler to be called + * 3. Request counter + * 4. A boolean to keep track if find is done + */ + auto state = std::make_shared(perm, handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking + * in handler_find. + */ + auto handler_find = [fs, + state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async SetPermission with the + * handler below. SetPermission DOES NOT guarantee that the handler will + * only be called once at a time, so we DO need locking in + * handler_set_permission. + */ + auto handler_set_permission = + [state](const hdfs::Status &status_set_permission) { + std::lock_guard guard(state->lock); + + // Decrement the counter once since we are done with this async call + if (!status_set_permission.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_set_permission; + } + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // exit + } + }; + + if (!stat_infos.empty() && state->status.ok()) { + for (const auto &s : stat_infos) { + /* + * Launch an asynchronous call to SetPermission for every returned + * result + */ + state->request_counter++; + fs->SetPermission(s.full_path, state->permissions, + handler_set_permission); + } + } + + /* + * Lock this section because handler_set_permission might be accessing the + * same shared variables simultaneously + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // exit + } + return false; + } + return true; + }; + + // Asynchronous call to Find + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + } + + // Block until promise is set + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h new file mode 100644 index 0000000000000..4400625a3b947 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/hdfs-chmod.h @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHMOD +#define LIBHDFSPP_TOOLS_HDFS_CHMOD + +#include +#include +#include +#include + +#include + +#include "hdfs-tool.h" +#include "hdfspp/status.h" + +namespace hdfs::tools { +struct PermissionState { + PermissionState(const uint16_t permissions, + std::function handler, + const uint64_t request_counter, const bool find_is_done) + : permissions(permissions), handler(std::move(handler)), + request_counter(request_counter), find_is_done(find_is_done) {} + + const uint16_t permissions; + const std::function handler; + + /** + * The request counter is incremented once every time SetOwner async call is + * made + */ + uint64_t request_counter; + + /** + * This boolean will be set when find returns the last result + */ + bool find_is_done{false}; + + /** + * Final status to be returned + */ + hdfs::Status status; + + /** + * Shared variables will need protection with a lock + */ + std::mutex lock; +}; + +/** + * {@class Chmod} is an {@class HdfsTool} that changes the permissions to a + * file or folder. + */ +class Chmod : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Chmod(int argc, char **argv); + + // Abiding to the Rule of 5 + Chmod(const Chmod &) = default; + Chmod(Chmod &&) = default; + Chmod &operator=(const Chmod &) = delete; + Chmod &operator=(Chmod &&) = delete; + ~Chmod() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the file to the file argument that's passed to this tool. + * + * @param permissions An octal representation of the new permissions to be + * assigned. + * @param recursive Whether this operation needs to be performed recursively + * on all the files in the given path's sub-directory. + * @param file The path to the file whose ownership needs to be changed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const std::string &permissions, + bool recursive, + const std::string &file) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc new file mode 100644 index 0000000000000..f79be66ccc6aa --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chmod/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-chmod.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chmod " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Chmod chmod(argc, argv); + auto success = false; + + try { + success = chmod.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt new file mode 100644 index 0000000000000..6773d8f88683a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_chown_lib STATIC $ $ hdfs-chown.cc) +target_include_directories(hdfs_chown_lib PRIVATE ../../tools ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_chown_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_chown main.cc) +target_include_directories(hdfs_chown PRIVATE ../../tools) +target_link_libraries(hdfs_chown PRIVATE hdfs_chown_lib) + +install(TARGETS hdfs_chown RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc new file mode 100644 index 0000000000000..41e4ca91bf92e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.cc @@ -0,0 +1,229 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hdfs-chown.h" +#include "tools_common.h" + +namespace hdfs::tools { +Chown::Chown(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Chown::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options( + "help,h", + "Change the owner and/or group of each FILE to OWNER and/or GROUP."); + add_options("file", po::value(), + "The path to the file whose ownership needs to be modified"); + add_options("recursive,R", "Operate on files and directories recursively"); + add_options( + "user-group", po::value(), + "The user:group to which the file's ownership needs to be changed to"); + + // An exception is thrown if these arguments are missing or if the arguments' + // count doesn't tally. + pos_opt_desc_.add("user-group", 1); + pos_opt_desc_.add("file", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +bool Chown::ValidateConstraints() const { + // Only "help" is allowed as single argument + if (argc_ == 2) { + return opt_val_.count("help"); + } + + // Rest of the cases must contain more than 2 arguments on the command line + return argc_ > 2; +} + +std::string Chown ::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE" << std::endl + << std::endl + << "Change the owner and/or group of each FILE to OWNER and/or GROUP." + << std::endl + << "The user must be a super-user. Additional information is in the " + "Permissions Guide:" + << std::endl + << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/" + "hadoop-hdfs/HdfsPermissionsGuide.html" + << std::endl + << std::endl + << " -R operate on files and directories recursively" << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Owner is unchanged if missing. Group is unchanged if missing." + << std::endl + << "OWNER and GROUP may be numeric as well as symbolic." << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_chown -R new_owner:new_group " + "hdfs://localhost.localdomain:8020/dir/file" + << std::endl + << "hdfs_chown new_owner /dir/file" << std::endl; + return desc.str(); +} + +bool Chown::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS chown tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("file") > 0 && opt_val_.count("user-group") > 0) { + const auto file = opt_val_["file"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + const Ownership ownership(opt_val_["user-group"].as()); + return HandlePath(ownership, recursive, file); + } + + return true; +} + +bool Chown::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Chown::HandlePath(const Ownership &ownership, const bool recursive, + const std::string &file) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(file); + + const auto fs = hdfs::doConnect(uri, true); + if (!fs) { + std::cerr << "Could not connect the file system. " << std::endl; + return false; + } + + // Wrap async FileSystem::SetOwner with promise to make it a blocking call + auto promise = std::make_shared>(); + auto future(promise->get_future()); + auto handler = [promise](const hdfs::Status &s) { promise->set_value(s); }; + + if (!recursive) { + fs->SetOwner(uri.get_path(), ownership.GetUser(), + ownership.GetGroup().value_or(""), handler); + } else { + /* + * Allocating shared state, which includes: username and groupname to be + * set, handler to be called, request counter, and a boolean to keep track + * if find is done + */ + auto state = std::make_shared(ownership.GetUser(), + ownership.GetGroup().value_or(""), + handler, 0, false); + + /* + * Keep requesting more from Find until we process the entire listing. Call + * handler when Find is done and request counter is 0. Find guarantees that + * the handler will only be called once at a time so we do not need locking + * in handler_find. + */ + auto handler_find = [fs, + state](const hdfs::Status &status_find, + const std::vector &stat_infos, + const bool has_more_results) -> bool { + /* + * For each result returned by Find we call async SetOwner with the + * handler below. SetOwner DOES NOT guarantee that the handler will only + * be called once at a time, so we DO need locking in handler_set_owner. + */ + auto handler_set_owner = [state](const hdfs::Status &status_set_owner) { + std::lock_guard guard(state->lock); + + // Decrement the counter once since we are done with this async call + if (!status_set_owner.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_set_owner; + } + + state->request_counter--; + if (state->request_counter == 0 && state->find_is_done) { + state->handler(state->status); // exit + } + }; + + if (!stat_infos.empty() && state->status.ok()) { + for (const auto &s : stat_infos) { + // Launch an asynchronous call to SetOwner for every returned result + state->request_counter++; + fs->SetOwner(s.full_path, state->user, state->group, + handler_set_owner); + } + } + + /* + * Lock this section because handler_set_owner might be accessing the same + * shared variables simultaneously. + */ + std::lock_guard guard(state->lock); + if (!status_find.ok() && state->status.ok()) { + // We make sure we set state->status only on the first error. + state->status = status_find; + } + + if (!has_more_results) { + state->find_is_done = true; + if (state->request_counter == 0) { + state->handler(state->status); // exit + } + return false; + } + return true; + }; + + // Asynchronous call to Find + fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), + handler_find); + } + + // Block until promise is set + const auto status = future.get(); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h new file mode 100644 index 0000000000000..25774be25ec6e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/hdfs-chown.h @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_CHOWN +#define LIBHDFSPP_TOOLS_HDFS_CHOWN + +#include + +#include + +#include "hdfs-tool.h" +#include "internal/hdfs-ownership.h" + +namespace hdfs::tools { +/** + * {@class Chown} is an {@class HdfsTool} that changes the owner and/or group of + * each file to owner and/or group. + */ +class Chown : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Chown(int argc, char **argv); + + // Abiding to the Rule of 5 + Chown(const Chown &) = default; + Chown(Chown &&) = default; + Chown &operator=(const Chown &) = delete; + Chown &operator=(Chown &&) = delete; + ~Chown() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the file to the file argument that's passed to this tool. + * + * @param ownership The owner's user and group names. + * @param recursive Whether this operation needs to be performed recursively + * on all the files in the given path's sub-directory. + * @param file The path to the file whose ownership needs to be changed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(const Ownership &ownership, + bool recursive, + const std::string &file) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc new file mode 100644 index 0000000000000..26492c7679ef2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-chown/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-chown.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS chown " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Chown chown(argc, argv); + auto success = false; + + try { + success = chown.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc deleted file mode 100644 index d64affeb14467..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chgrp.cc +++ /dev/null @@ -1,185 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_chgrp [OPTION] GROUP FILE" - << std::endl - << std::endl << "Change the group association of each FILE to GROUP." - << std::endl << "The user must be the owner of files. Additional information is in the Permissions Guide:" - << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html" - << std::endl - << std::endl << " -R operate on files and directories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_chgrp -R new_group hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_chgrp new_group /dir/file" - << std::endl; -} - -struct SetOwnerState { - const std::string username; - const std::string groupname; - const std::function handler; - //The request counter is incremented once every time SetOwner async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - SetOwnerState(const std::string & username_, const std::string & groupname_, - const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : username(username_), - groupname(groupname_), - handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have 3 or 4 parameters - if (argc != 3 && argc != 4) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = 1; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string group = argv[optind]; - //Owner stays the same, just group association changes. - std::string owner = ""; - std::string uri_path = argv[optind + 1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::SetOwner with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - if(!recursive){ - fs->SetOwner(uri.get_path(), owner, group, handler); - } - else { - //Allocating shared state, which includes: - //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(owner, group, handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async SetOwner with the handler below. - //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner. - auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) { - std::lock_guard guard(state->lock); - - //Decrement the counter once since we are done with this async call - if (!status_set_owner.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_set_owner; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to SetOwner for every returned result - state->request_counter++; - fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner); - } - } - - //Lock this section because handlerSetOwner might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - } - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc deleted file mode 100644 index 091f18f904ecc..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chmod.cc +++ /dev/null @@ -1,183 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_chmod [OPTION] FILE" - << std::endl - << std::endl << "Change the permissions of each FILE to MODE." - << std::endl << "The user must be the owner of the file, or else a super-user." - << std::endl << "Additional information is in the Permissions Guide:" - << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html" - << std::endl - << std::endl << " -R operate on files and directories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_chmod -R 755 hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_chmod 777 /dir/file" - << std::endl; -} - -struct SetPermissionState { - const uint16_t permissions; - const std::function handler; - //The request counter is incremented once every time SetOwner async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - SetPermissionState(const uint16_t permissions_, const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : permissions(permissions_), - handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have 3 or 4 parameters - if (argc != 3 && argc != 4) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = 1; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string permissions = argv[optind]; - std::string uri_path = argv[optind + 1]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::SetPermission with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - //strtol() is reading the value with base 8, NULL because we are reading in just one value. - uint16_t perm = strtol(permissions.c_str(), NULL, 8); - if(!recursive){ - fs->SetPermission(uri.get_path(), perm, handler); - } - else { - //Allocating shared state, which includes: - //permissions to be set, handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(perm, handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async SetPermission with the handler below. - //SetPermission DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetPermission. - auto handlerSetPermission = [state](const hdfs::Status &status_set_permission) { - std::lock_guard guard(state->lock); - - //Decrement the counter once since we are done with this async call - if (!status_set_permission.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_set_permission; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to SetPermission for every returned result - state->request_counter++; - fs->SetPermission(s.full_path, state->permissions, handlerSetPermission); - } - } - - //Lock this section because handlerSetPermission might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - } - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc deleted file mode 100644 index 24f7470b4163e..0000000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_chown.cc +++ /dev/null @@ -1,195 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_chown [OPTION] [OWNER][:[GROUP]] FILE" - << std::endl - << std::endl << "Change the owner and/or group of each FILE to OWNER and/or GROUP." - << std::endl << "The user must be a super-user. Additional information is in the Permissions Guide:" - << std::endl << "https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html" - << std::endl - << std::endl << " -R operate on files and directories recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Owner is unchanged if missing. Group is unchanged if missing." - << std::endl << "OWNER and GROUP may be numeric as well as symbolic." - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_chown -R new_owner:new_group hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_chown new_owner /dir/file" - << std::endl; -} - -struct SetOwnerState { - const std::string username; - const std::string groupname; - const std::function handler; - //The request counter is incremented once every time SetOwner async call is made - uint64_t request_counter; - //This boolean will be set when find returns the last result - bool find_is_done; - //Final status to be returned - hdfs::Status status; - //Shared variables will need protection with a lock - std::mutex lock; - SetOwnerState(const std::string & username_, const std::string & groupname_, - const std::function & handler_, - uint64_t request_counter_, bool find_is_done_) - : username(username_), - groupname(groupname_), - handler(handler_), - request_counter(request_counter_), - find_is_done(find_is_done_), - status(), - lock() { - } -}; - -int main(int argc, char *argv[]) { - //We should have 3 or 4 parameters - if (argc != 3 && argc != 4) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = 1; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string owner_and_group = argv[optind]; - std::string uri_path = argv[optind + 1]; - - std::string owner, group; - size_t owner_end = owner_and_group.find(":"); - if(owner_end == std::string::npos) { - owner = owner_and_group; - } else { - owner = owner_and_group.substr(0, owner_end); - group = owner_and_group.substr(owner_end + 1); - } - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - /* wrap async FileSystem::SetOwner with promise to make it a blocking call */ - std::shared_ptr> promise = std::make_shared>(); - std::future future(promise->get_future()); - auto handler = [promise](const hdfs::Status &s) { - promise->set_value(s); - }; - - if(!recursive){ - fs->SetOwner(uri.get_path(), owner, group, handler); - } - else { - //Allocating shared state, which includes: - //username and groupname to be set, handler to be called, request counter, and a boolean to keep track if find is done - std::shared_ptr state = std::make_shared(owner, group, handler, 0, false); - - // Keep requesting more from Find until we process the entire listing. Call handler when Find is done and reques counter is 0. - // Find guarantees that the handler will only be called once at a time so we do not need locking in handlerFind. - auto handlerFind = [fs, state](const hdfs::Status &status_find, const std::vector & stat_infos, bool has_more_results) -> bool { - - //For each result returned by Find we call async SetOwner with the handler below. - //SetOwner DOES NOT guarantee that the handler will only be called once at a time, so we DO need locking in handlerSetOwner. - auto handlerSetOwner = [state](const hdfs::Status &status_set_owner) { - std::lock_guard guard(state->lock); - - //Decrement the counter once since we are done with this async call - if (!status_set_owner.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_set_owner; - } - state->request_counter--; - if(state->request_counter == 0 && state->find_is_done){ - state->handler(state->status); //exit - } - }; - if(!stat_infos.empty() && state->status.ok()) { - for (hdfs::StatInfo const& s : stat_infos) { - //Launch an asynchronous call to SetOwner for every returned result - state->request_counter++; - fs->SetOwner(s.full_path, state->username, state->groupname, handlerSetOwner); - } - } - - //Lock this section because handlerSetOwner might be accessing the same - //shared variables simultaneously - std::lock_guard guard(state->lock); - if (!status_find.ok() && state->status.ok()){ - //We make sure we set state->status only on the first error. - state->status = status_find; - } - if(!has_more_results){ - state->find_is_done = true; - if(state->request_counter == 0){ - state->handler(state->status); //exit - } - return false; - } - return true; - }; - - //Asynchronous call to Find - fs->Find(uri.get_path(), "*", hdfs::FileSystem::GetDefaultFindMaxDepth(), handlerFind); - } - - /* block until promise is set */ - hdfs::Status status = future.get(); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt new file mode 100644 index 0000000000000..b223905c9cc99 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/CMakeLists.txt @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_ownership_obj OBJECT hdfs-ownership.cc) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc new file mode 100644 index 0000000000000..e9dc74e541424 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.cc @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hdfs-ownership.h" + +namespace hdfs::tools { +Ownership::Ownership(const std::string &user_and_group) { + const auto owner_end = user_and_group.find(':'); + if (owner_end == std::string::npos) { + user_ = user_and_group; + return; + } + + user_ = user_and_group.substr(0, owner_end); + group_ = user_and_group.substr(owner_end + 1); +} + +bool Ownership::operator==(const Ownership &other) const { + const auto same_user = user_ == other.user_; + if (group_.has_value() && other.group_.has_value()) { + return same_user && group_.value() == other.group_.value(); + } + + if (!group_.has_value() && !other.group_.has_value()) { + return same_user; + } + return false; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h new file mode 100644 index 0000000000000..037ab61e58cda --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/internal/hdfs-ownership.h @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_OWNERSHIP +#define LIBHDFSPP_TOOLS_HDFS_OWNERSHIP + +#include +#include +#include +#include + +#include "hdfspp/status.h" + +namespace hdfs::tools { +/** + * {@class Ownership} contains the user and group ownership information. + */ +struct Ownership { + explicit Ownership(const std::string &user_and_group); + + [[nodiscard]] const std::string &GetUser() const { return user_; } + + [[nodiscard]] const std::optional &GetGroup() const { + return group_; + } + + bool operator==(const Ownership &other) const; + +private: + std::string user_; + std::optional group_; +}; + +/** + * {@class OwnerState} holds information needed for recursive traversal of some + * of the HDFS APIs. + */ +struct OwnerState { + OwnerState(std::string username, std::string group, + std::function handler, + const uint64_t request_counter, const bool find_is_done) + : user{std::move(username)}, group{std::move(group)}, handler{std::move( + handler)}, + request_counter{request_counter}, find_is_done{find_is_done} {} + + const std::string user; + const std::string group; + const std::function handler; + + /** + * The request counter is incremented once every time SetOwner async call is + * made. + */ + uint64_t request_counter; + + /** + * This boolean will be set when find returns the last result. + */ + bool find_is_done{false}; + + /** + * Final status to be returned. + */ + hdfs::Status status{}; + + /** + * Shared variables will need protection with a lock. + */ + std::mutex lock; +}; +} // namespace hdfs::tools + +#endif \ No newline at end of file