From 942ff36afe542df6f24522a247b2bc7590a9f131 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 23 Jun 2023 16:08:05 +0200 Subject: [PATCH 01/25] MINIFICPP-2146 Add support for SMB networking protocol MINIFICPP-2147 PutSmb MINIFICPP-2148 FetchSmb MINIFICPP-2150 ListSmb --- cmake/MiNiFiOptions.cmake | 1 + .../bustache/tests/ApplyTemplateTests.cpp | 139 +++--- .../tests/ExpressionLanguageTests.cpp | 2 +- .../http-curl/tests/unit/AlertTests.cpp | 2 +- extensions/python/ExecutePythonProcessor.h | 2 +- .../tests/ExecutePythonProcessorTests.cpp | 1 - extensions/sftp/tests/FetchSFTPTests.cpp | 2 +- extensions/sftp/tests/ListSFTPTests.cpp | 2 +- .../sftp/tests/ListThenFetchSFTPTests.cpp | 4 +- extensions/sftp/tests/PutSFTPTests.cpp | 2 +- extensions/smb/CMakeLists.txt | 34 ++ extensions/smb/FetchSmb.cpp | 84 ++++ extensions/smb/FetchSmb.h | 88 ++++ extensions/smb/ListSmb.cpp | 152 ++++++ extensions/smb/ListSmb.h | 152 ++++++ extensions/smb/PutSmb.cpp | 99 ++++ extensions/smb/PutSmb.h | 95 ++++ .../smb/SmbConnectionControllerService.cpp | 107 ++++ .../smb/SmbConnectionControllerService.h | 105 ++++ extensions/smb/tests/CMakeLists.txt | 36 ++ extensions/smb/tests/FetchSmbTests.cpp | 105 ++++ extensions/smb/tests/ListAndFetchSmbTests.cpp | 65 +++ extensions/smb/tests/ListSmbTests.cpp | 121 +++++ extensions/smb/tests/PutSmbTests.cpp | 184 +++++++ .../SmbConnectionControllerServiceTests.cpp | 72 +++ .../MockSmbConnectionControllerService.h | 105 ++++ extensions/smb/tests/utils/TempSmbShare.h | 76 +++ .../processors/FetchFile.cpp | 6 +- .../processors/GetFile.cpp | 2 +- .../processors/ListFile.cpp | 137 ++--- .../standard-processors/processors/ListFile.h | 25 +- .../processors/PutFile.cpp | 220 +++----- .../standard-processors/processors/PutFile.h | 54 +- .../tests/unit/PutFileTests.cpp | 12 - .../windows-event-log/wel/MetadataWalker.cpp | 4 - .../include/c2/triggers/FileUpdateTrigger.h | 6 +- libminifi/include/core/logging/Logger.h | 11 + libminifi/include/utils/LogUtils.h | 6 + libminifi/include/utils/OsUtils.h | 4 + .../utils/{ => file}/FileReaderCallback.h | 0 libminifi/include/utils/file/FileUtils.h | 33 +- .../include/utils/file/FileWriterCallback.h | 41 ++ libminifi/include/utils/file/ListedFile.h | 104 ++++ .../src/c2/triggers/FileUpdateTrigger.cpp | 4 +- libminifi/src/core/logging/Logger.cpp | 2 +- libminifi/src/utils/NetworkInterfaceInfo.cpp | 18 +- libminifi/src/utils/OsUtils.cpp | 47 +- .../utils/{ => file}/FileReaderCallback.cpp | 2 +- libminifi/src/utils/file/FileUtils.cpp | 26 +- .../src/utils/file/FileWriterCallback.cpp | 66 +++ libminifi/test/Utils.h | 2 +- .../test/archive-tests/FocusArchiveTests.cpp | 113 ++--- .../archive-tests/ManipulateArchiveTests.cpp | 472 +++++++++--------- libminifi/test/unit/OsUtilTests.cpp | 20 + run_clang_tidy.sh | 2 +- 55 files changed, 2522 insertions(+), 754 deletions(-) create mode 100644 extensions/smb/CMakeLists.txt create mode 100644 extensions/smb/FetchSmb.cpp create mode 100644 extensions/smb/FetchSmb.h create mode 100644 extensions/smb/ListSmb.cpp create mode 100644 extensions/smb/ListSmb.h create mode 100644 extensions/smb/PutSmb.cpp create mode 100644 extensions/smb/PutSmb.h create mode 100644 extensions/smb/SmbConnectionControllerService.cpp create mode 100644 extensions/smb/SmbConnectionControllerService.h create mode 100644 extensions/smb/tests/CMakeLists.txt create mode 100644 extensions/smb/tests/FetchSmbTests.cpp create mode 100644 extensions/smb/tests/ListAndFetchSmbTests.cpp create mode 100644 extensions/smb/tests/ListSmbTests.cpp create mode 100644 extensions/smb/tests/PutSmbTests.cpp create mode 100644 extensions/smb/tests/SmbConnectionControllerServiceTests.cpp create mode 100644 extensions/smb/tests/utils/MockSmbConnectionControllerService.h create mode 100644 extensions/smb/tests/utils/TempSmbShare.h rename libminifi/include/utils/{ => file}/FileReaderCallback.h (100%) create mode 100644 libminifi/include/utils/file/FileWriterCallback.h create mode 100644 libminifi/include/utils/file/ListedFile.h rename libminifi/src/utils/{ => file}/FileReaderCallback.cpp (98%) create mode 100644 libminifi/src/utils/file/FileWriterCallback.cpp diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index c20e7b9b7e..c020820450 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -85,6 +85,7 @@ if (WIN32) add_minifi_option(MSI_REDISTRIBUTE_UCRT_NONASL "Redistribute Universal C Runtime DLLs with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) add_minifi_option(ENABLE_WEL "Enables the suite of Windows Event Log extensions." OFF) add_minifi_option(ENABLE_PDH "Enables PDH support." OFF) + add_minifi_option(ENABLE_SMB "Enables SMB support." ON) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/extensions/bustache/tests/ApplyTemplateTests.cpp b/extensions/bustache/tests/ApplyTemplateTests.cpp index 0a908c2fa6..8f2f73001d 100644 --- a/extensions/bustache/tests/ApplyTemplateTests.cpp +++ b/extensions/bustache/tests/ApplyTemplateTests.cpp @@ -41,6 +41,8 @@ #include "processors/PutFile.h" #include "processors/ExtractText.h" +namespace org::apache::nifi::minifi::processors::test { + const char* TEMPLATE = "TemplateBegins\n{{ ExampleAttribute }}\nTemplateEnds"; const char* TEMPLATE_FILE = "test_template.txt"; const char* TEST_ATTR = "ExampleAttribute"; @@ -49,75 +51,76 @@ const char* TEST_FILE = "test_file.txt"; const char* EXPECT_OUTPUT = "TemplateBegins\nExampleValue\nTemplateEnds"; TEST_CASE("Test Creation of ApplyTemplate", "[ApplyTemplateCreate]") { - TestController testController; - std::shared_ptr processor = std::make_shared("processor_name"); - REQUIRE(processor->getName() == "processor_name"); - REQUIRE(processor->getUUID()); + TestController testController; + std::shared_ptr processor = std::make_shared("processor_name"); + REQUIRE(processor->getName() == "processor_name"); + REQUIRE(processor->getUUID()); } TEST_CASE("Test usage of ApplyTemplate", "[ApplyTemplateTest]") { - TestController testController; - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - - std::shared_ptr plan = testController.createPlan(); - std::shared_ptr repo = std::make_shared(); - - auto get_file_source_dir = testController.createTempDirectory(); - auto template_source_dir = testController.createTempDirectory(); - auto put_file_destination_dir = testController.createTempDirectory(); - - REQUIRE_FALSE(get_file_source_dir.empty()); - REQUIRE_FALSE(template_source_dir.empty()); - REQUIRE_FALSE(put_file_destination_dir.empty()); - - std::shared_ptr getfile = plan->addProcessor("GetFile", "getFile"); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::Directory, get_file_source_dir.string()); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::KeepSourceFile, "true"); - - std::shared_ptr extract_text = plan->addProcessor("ExtractText", "testExtractText", core::Relationship("success", "description"), true); - plan->setProperty(extract_text, org::apache::nifi::minifi::processors::ExtractText::Attribute, TEST_ATTR); - - std::shared_ptr apply_template = plan->addProcessor("ApplyTemplate", "testApplyTemplate", core::Relationship("success", "description"), true); - - std::shared_ptr put_file = plan->addProcessor("PutFile", "put_file", core::Relationship("success", "description"), true); - plan->setProperty(put_file, org::apache::nifi::minifi::processors::PutFile::Directory, put_file_destination_dir.string()); - plan->setProperty(put_file, org::apache::nifi::minifi::processors::PutFile::ConflictResolution, - org::apache::nifi::minifi::processors::PutFile::CONFLICT_RESOLUTION_STRATEGY_REPLACE); - - // Write attribute value to file for GetFile->ExtractText - - std::ofstream test_file(get_file_source_dir / TEST_FILE); - REQUIRE(test_file.is_open()); - test_file << TEST_VALUE; - test_file.close(); - - // Write template to file - auto template_path = template_source_dir / TEMPLATE_FILE; - std::ofstream template_file(template_path); - REQUIRE(template_file.is_open()); - template_file << TEMPLATE; - template_file.close(); - - plan->setProperty(apply_template, org::apache::nifi::minifi::processors::ApplyTemplate::Template, template_path.string()); - - // Run processor chain - plan->runNextProcessor(); // GetFile - plan->runNextProcessor(); // ExtractText - plan->runNextProcessor(); // ApplyTemplate - plan->runNextProcessor(); // PutFile - - // Read contents of file - std::ifstream output_file(put_file_destination_dir / TEST_FILE); - std::stringstream output_buf; - output_buf << output_file.rdbuf(); - std::string output_contents = output_buf.str(); - REQUIRE(output_contents == EXPECT_OUTPUT); + TestController testController; + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + + std::shared_ptr plan = testController.createPlan(); + std::shared_ptr repo = std::make_shared(); + + auto get_file_source_dir = testController.createTempDirectory(); + auto template_source_dir = testController.createTempDirectory(); + auto put_file_destination_dir = testController.createTempDirectory(); + + REQUIRE_FALSE(get_file_source_dir.empty()); + REQUIRE_FALSE(template_source_dir.empty()); + REQUIRE_FALSE(put_file_destination_dir.empty()); + + std::shared_ptr getfile = plan->addProcessor("GetFile", "getFile"); + plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::Directory, get_file_source_dir); + plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::KeepSourceFile, "true"); + + std::shared_ptr extract_text = plan->addProcessor("ExtractText", "testExtractText", core::Relationship("success", "description"), true); + plan->setProperty(extract_text, org::apache::nifi::minifi::processors::ExtractText::Attribute, TEST_ATTR); + + std::shared_ptr apply_template = plan->addProcessor("ApplyTemplate", "testApplyTemplate", core::Relationship("success", "description"), true); + + std::shared_ptr put_file = plan->addProcessor("PutFile", "put_file", core::Relationship("success", "description"), true); + plan->setProperty(put_file, org::apache::nifi::minifi::processors::PutFile::Directory, put_file_destination_dir); + plan->setProperty(put_file, org::apache::nifi::minifi::processors::PutFile::ConflictResolution, magic_enum::enum_name(minifi::processors::PutFile::FileExistsResolutionStrategy::replace)); + + // Write attribute value to file for GetFile->ExtractText + + std::ofstream test_file(get_file_source_dir / TEST_FILE); + REQUIRE(test_file.is_open()); + test_file << TEST_VALUE; + test_file.close(); + + // Write template to file + auto template_path = template_source_dir / TEMPLATE_FILE; + std::ofstream template_file(template_path); + REQUIRE(template_file.is_open()); + template_file << TEMPLATE; + template_file.close(); + + plan->setProperty(apply_template, org::apache::nifi::minifi::processors::ApplyTemplate::Template, template_path.string()); + + // Run processor chain + plan->runNextProcessor(); // GetFile + plan->runNextProcessor(); // ExtractText + plan->runNextProcessor(); // ApplyTemplate + plan->runNextProcessor(); // PutFile + + // Read contents of file + std::ifstream output_file(put_file_destination_dir / TEST_FILE); + std::stringstream output_buf; + output_buf << output_file.rdbuf(); + std::string output_contents = output_buf.str(); + REQUIRE(output_contents == EXPECT_OUTPUT); } + +} // namespace org::apache::nifi::minifi::processors::test diff --git a/extensions/expression-language/tests/ExpressionLanguageTests.cpp b/extensions/expression-language/tests/ExpressionLanguageTests.cpp index 0fa47935b3..325e8fd228 100644 --- a/extensions/expression-language/tests/ExpressionLanguageTests.cpp +++ b/extensions/expression-language/tests/ExpressionLanguageTests.cpp @@ -219,7 +219,7 @@ TEST_CASE("GetFile PutFile dynamic attribute", "[expressionLanguageTestGetFilePu plan->addProcessor("LogAttribute", "LogAttribute", core::Relationship("success", "description"), true); auto put_file = plan->addProcessor("PutFile", "PutFile", core::Relationship("success", "description"), true); plan->setProperty(put_file, minifi::processors::PutFile::Directory, (out_dir / "${extracted_attr_name}").string()); - plan->setProperty(put_file, minifi::processors::PutFile::ConflictResolution, minifi::processors::PutFile::CONFLICT_RESOLUTION_STRATEGY_REPLACE); + plan->setProperty(put_file, minifi::processors::PutFile::ConflictResolution, magic_enum::enum_name(minifi::processors::PutFile::FileExistsResolutionStrategy::replace)); plan->setProperty(put_file, minifi::processors::PutFile::CreateDirs, "true"); // Write test input diff --git a/extensions/http-curl/tests/unit/AlertTests.cpp b/extensions/http-curl/tests/unit/AlertTests.cpp index 72fcbeca28..20774156b4 100644 --- a/extensions/http-curl/tests/unit/AlertTests.cpp +++ b/extensions/http-curl/tests/unit/AlertTests.cpp @@ -45,7 +45,7 @@ class AlertHandler : public ServerAwareHandler { std::string id(doc["agentId"].GetString(), doc["agentId"].GetStringLength()); REQUIRE(id == agent_id_); std::vector batch; - for (size_t i = 0; i < doc["alerts"].Size(); ++i) { + for (rapidjson::SizeType i = 0; i < doc["alerts"].Size(); ++i) { REQUIRE(doc["alerts"][i].IsString()); batch.emplace_back(doc["alerts"][i].GetString(), doc["alerts"][i].GetStringLength()); } diff --git a/extensions/python/ExecutePythonProcessor.h b/extensions/python/ExecutePythonProcessor.h index e2adb26544..08df517d2a 100644 --- a/extensions/python/ExecutePythonProcessor.h +++ b/extensions/python/ExecutePythonProcessor.h @@ -130,7 +130,7 @@ class ExecutePythonProcessor : public core::Processor { std::string script_to_exec_; bool reload_on_script_change_; - std::optional last_script_write_time_; + std::optional last_script_write_time_; std::string script_file_path_; std::shared_ptr python_logger_; std::unique_ptr python_script_engine_; diff --git a/extensions/python/tests/ExecutePythonProcessorTests.cpp b/extensions/python/tests/ExecutePythonProcessorTests.cpp index 7cea60c0cd..fe4a6f0fa3 100644 --- a/extensions/python/tests/ExecutePythonProcessorTests.cpp +++ b/extensions/python/tests/ExecutePythonProcessorTests.cpp @@ -53,7 +53,6 @@ class ExecutePythonProcessorTestBase { plan_ = testController_->createPlan(); logTestController_.setDebug(); logTestController_.setDebug(); - logTestController_.setDebug(); } auto getScriptFullPath(const std::filesystem::path& script_file_name) { diff --git a/extensions/sftp/tests/FetchSFTPTests.cpp b/extensions/sftp/tests/FetchSFTPTests.cpp index bc016783ab..3148e7b48e 100644 --- a/extensions/sftp/tests/FetchSFTPTests.cpp +++ b/extensions/sftp/tests/FetchSFTPTests.cpp @@ -101,7 +101,7 @@ class FetchSFTPTestsFixture { // Configure PutFile processor plan->setProperty(put_file, "Directory", (dst_dir / "${path}").string()); - plan->setProperty(put_file, "Conflict Resolution Strategy", minifi::processors::PutFile::CONFLICT_RESOLUTION_STRATEGY_FAIL); + plan->setProperty(put_file, "Conflict Resolution Strategy", magic_enum::enum_name(minifi::processors::PutFile::FileExistsResolutionStrategy::fail)); plan->setProperty(put_file, "Create Missing Directories", "true"); } diff --git a/extensions/sftp/tests/ListSFTPTests.cpp b/extensions/sftp/tests/ListSFTPTests.cpp index f0c7ea6efe..fd554becfc 100644 --- a/extensions/sftp/tests/ListSFTPTests.cpp +++ b/extensions/sftp/tests/ListSFTPTests.cpp @@ -133,7 +133,7 @@ class ListSFTPTestsFixture { } // Create source file - void createFile(const std::filesystem::path& relative_path, const std::string& content, std::optional modification_time) { + void createFile(const std::filesystem::path& relative_path, const std::string& content, std::optional modification_time) { std::fstream file; std::filesystem::path full_path = src_dir / "vfs" / relative_path; std::filesystem::create_directories(full_path.parent_path()); diff --git a/extensions/sftp/tests/ListThenFetchSFTPTests.cpp b/extensions/sftp/tests/ListThenFetchSFTPTests.cpp index 359217b61f..dbe31d2f10 100644 --- a/extensions/sftp/tests/ListThenFetchSFTPTests.cpp +++ b/extensions/sftp/tests/ListThenFetchSFTPTests.cpp @@ -138,7 +138,7 @@ class ListThenFetchSFTPTestsFixture { // Configure PutFile processor plan->setProperty(put_file, "Directory", (dst_dir / "${path}").string()); - plan->setProperty(put_file, "Conflict Resolution Strategy", minifi::processors::PutFile::CONFLICT_RESOLUTION_STRATEGY_FAIL); + plan->setProperty(put_file, "Conflict Resolution Strategy", magic_enum::enum_name(minifi::processors::PutFile::FileExistsResolutionStrategy::fail)); plan->setProperty(put_file, "Create Missing Directories", "true"); } @@ -147,7 +147,7 @@ class ListThenFetchSFTPTestsFixture { } // Create source file - void createFile(const std::string& relative_path, const std::string& content, std::optional modification_time) { + void createFile(const std::string& relative_path, const std::string& content, std::optional modification_time) { std::fstream file; std::filesystem::path full_path = src_dir / "vfs" / relative_path; std::filesystem::create_directories(full_path.parent_path()); diff --git a/extensions/sftp/tests/PutSFTPTests.cpp b/extensions/sftp/tests/PutSFTPTests.cpp index 96aeed9151..b6cc04623c 100644 --- a/extensions/sftp/tests/PutSFTPTests.cpp +++ b/extensions/sftp/tests/PutSFTPTests.cpp @@ -154,7 +154,7 @@ class PutSFTPTestsFixture { REQUIRE(false == file.good()); } - void testModificationTime(const std::string& relative_path, std::filesystem::file_time_type mtime) { + void testModificationTime(const std::string& relative_path, std::chrono::file_clock::time_point mtime) { auto result_file = dst_dir / "vfs" / relative_path; REQUIRE(mtime == utils::file::last_write_time(result_file).value()); } diff --git a/extensions/smb/CMakeLists.txt b/extensions/smb/CMakeLists.txt new file mode 100644 index 0000000000..4cb049fcc1 --- /dev/null +++ b/extensions/smb/CMakeLists.txt @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +if (NOT (WIN32 AND ENABLE_SMB)) + return() +endif() + +include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) + +file(GLOB SOURCES "*.cpp") + +add_library(minifi-smb SHARED ${SOURCES}) +target_link_libraries(minifi-smb ${LIBMINIFI} Mpr) +target_include_directories(minifi-smb PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") + +register_extension(minifi-smb "SMB EXTENSIONS" SMB-EXTENSIONS "This enables SMB support" "extensions/smb/tests") + +register_extension_linter(minifi-smb-extensions-linter) diff --git a/extensions/smb/FetchSmb.cpp b/extensions/smb/FetchSmb.cpp new file mode 100644 index 0000000000..bb199c36a4 --- /dev/null +++ b/extensions/smb/FetchSmb.cpp @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "FetchSmb.h" +#include "core/Resource.h" +#include "utils/file/FileReaderCallback.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +void FetchSmb::initialize() { + setSupportedProperties(Properties); + setSupportedRelationships(Relationships); +} + +void FetchSmb::onSchedule(const std::shared_ptr& context, const std::shared_ptr&) { + gsl_Expects(context); + if (auto connection_controller_name = context->getProperty(FetchSmb::ConnectionControllerService)) { + smb_connection_controller_service_ = std::dynamic_pointer_cast(context->getControllerService(*connection_controller_name)); + } + if (!smb_connection_controller_service_) { + throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); + } +} + +namespace { +std::filesystem::path getPath(core::ProcessContext& context, const std::shared_ptr& flow_file) { + auto remote_file = context.getProperty(FetchSmb::RemoteFile, flow_file); + if (remote_file && !remote_file->empty()) { + if (remote_file->starts_with('/')) + remote_file->erase(remote_file->begin()); + return *remote_file; + } + std::filesystem::path path = flow_file->getAttribute(core::SpecialFlowAttribute::PATH).value_or(""); + std::filesystem::path filename = flow_file->getAttribute(core::SpecialFlowAttribute::FILENAME).value_or(""); + return path / filename; +} +} // namespace + +void FetchSmb::onTrigger(const std::shared_ptr& context, const std::shared_ptr& session) { + gsl_Expects(context && session && smb_connection_controller_service_); + + auto connection_error = smb_connection_controller_service_->validateConnection(); + if (connection_error) { + logger_->log_error("Couldn't establish connection to the specified network location due to %s", connection_error.message()); + context->yield(); + return; + } + + auto flow_file = session->get(); + if (!flow_file) { + context->yield(); + return; + } + + auto path = getPath(*context, flow_file); + + try { + session->write(flow_file, utils::FileReaderCallback{smb_connection_controller_service_->getPath() / path}); + session->transfer(flow_file, Success); + } catch (const utils::FileReaderCallbackIOError& io_error) { + flow_file->addAttribute(ErrorCode.name, fmt::format("{}", io_error.error_code)); + flow_file->addAttribute(ErrorMessage.name, io_error.what()); + session->transfer(flow_file, Failure); + return; + } +} + +REGISTER_RESOURCE(FetchSmb, Processor); + +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/FetchSmb.h b/extensions/smb/FetchSmb.h new file mode 100644 index 0000000000..462478be28 --- /dev/null +++ b/extensions/smb/FetchSmb.h @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "SmbConnectionControllerService.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/Property.h" +#include "core/PropertyDefinition.h" +#include "core/PropertyDefinitionBuilder.h" +#include "core/OutputAttributeDefinition.h" +#include "core/logging/LoggerConfiguration.h" +#include "utils/Enum.h" +#include "utils/ListingStateManager.h" +#include "utils/file/ListedFile.h" +#include "utils/file/FileUtils.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +class FetchSmb : public core::Processor { + public: + explicit FetchSmb(std::string name, const utils::Identifier& uuid = {}) + : core::Processor(std::move(name), uuid) { + } + + EXTENSIONAPI static constexpr const char* Description = "Fetches files from a SMB Share. Designed to be used in tandem with ListSmb."; + + EXTENSIONAPI static constexpr auto ConnectionControllerService = core::PropertyDefinitionBuilder<>::createProperty("SMB Connection Controller Service") + .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server.") + .isRequired(true) + .withAllowedTypes() + .build(); + EXTENSIONAPI static constexpr auto RemoteFile = core::PropertyDefinitionBuilder<>::createProperty("Input Directory") + .withDescription("The full path of the file to be retrieved from the remote server. Expression language supported. If left empty the path and filename attributes will be used.") + .isRequired(false) + .supportsExpressionLanguage(true) + .build(); + EXTENSIONAPI static constexpr auto Properties = std::array{ + ConnectionControllerService, + RemoteFile + }; + + EXTENSIONAPI static constexpr auto Success = core::RelationshipDefinition{"success", "A flowfile will be routed here for each successfully fetched file."}; + EXTENSIONAPI static constexpr auto Failure = core::RelationshipDefinition{"failure", "A flowfile will be routed here when failed to fetch its content."}; + EXTENSIONAPI static constexpr auto Relationships = std::array{Success, Failure}; + + EXTENSIONAPI static constexpr auto ErrorCode = core::OutputAttributeDefinition<>{"error.code", { Failure }, "The error code returned by SMB when the fetch of a file fails."}; + EXTENSIONAPI static constexpr auto ErrorMessage = core::OutputAttributeDefinition<>{"error.message", { Failure }, "The error message returned by SMB when the fetch of a file fails."}; + + EXTENSIONAPI static constexpr auto OutputAttributes = std::array{ FetchSmb::ErrorCode, ErrorMessage }; + + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; + EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; + EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; + EXTENSIONAPI static constexpr bool IsSingleThreaded = false; + + ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS + + void initialize() override; + void onSchedule(const std::shared_ptr &context, const std::shared_ptr &session_factory) override; + void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) override; + + private: + std::shared_ptr smb_connection_controller_service_; + std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); +}; + +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/ListSmb.cpp b/extensions/smb/ListSmb.cpp new file mode 100644 index 0000000000..33487135b5 --- /dev/null +++ b/extensions/smb/ListSmb.cpp @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ListSmb.h" +#include + +#include "utils/StringUtils.h" +#include "utils/TimeUtil.h" +#include "utils/OsUtils.h" +#include "utils/file/FileUtils.h" +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +void ListSmb::initialize() { + setSupportedProperties(Properties); + setSupportedRelationships(Relationships); +} + +void ListSmb::onSchedule(const std::shared_ptr &context, const std::shared_ptr &/*sessionFactory*/) { + gsl_Expects(context); + + if (auto connection_controller_name = context->getProperty(ListSmb::ConnectionControllerService)) { + smb_connection_controller_service_ = std::dynamic_pointer_cast(context->getControllerService(*connection_controller_name)); + } + if (!smb_connection_controller_service_) { + throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); + } + + auto state_manager = context->getStateManager(); + if (state_manager == nullptr) { + throw Exception(PROCESSOR_EXCEPTION, "Failed to get StateManager"); + } + state_manager_ = std::make_unique(state_manager); + + input_directory_ = context->getProperty(InputDirectory).value_or(""); + + context->getProperty(RecurseSubdirectories, recurse_subdirectories_); + + std::string value; + if (context->getProperty(FileFilter, value) && !value.empty()) { + file_filter_.filename_filter = std::regex(value); + } + + if (recurse_subdirectories_ && context->getProperty(PathFilter, value) && !value.empty()) { + file_filter_.path_filter = std::regex(value); + } + + if (auto minimum_file_age = context->getProperty(MinimumFileAge)) { + file_filter_.minimum_file_age = minimum_file_age->getMilliseconds(); + } + + if (auto maximum_file_age = context->getProperty(MaximumFileAge)) { + file_filter_.maximum_file_age = maximum_file_age->getMilliseconds(); + } + + uint64_t int_value = 0; + if (context->getProperty(MinimumFileSize, value) && !value.empty() && core::Property::StringToInt(value, int_value)) { + file_filter_.minimum_file_size = int_value; + } + + if (context->getProperty(MaximumFileSize, value) && !value.empty() && core::Property::StringToInt(value, int_value)) { + file_filter_.maximum_file_size = int_value; + } + + context->getProperty(IgnoreHiddenFiles, file_filter_.ignore_hidden_files); +} + +namespace { +std::string getDateTimeStr(std::chrono::file_clock::time_point time_point) { + return utils::timeutils::getDateTimeStr(std::chrono::time_point_cast(utils::file::to_sys(time_point))); +} +} + +std::shared_ptr ListSmb::createFlowFile(core::ProcessSession& session, const utils::ListedFile& listed_file) { + auto flow_file = session.create(); + session.putAttribute(flow_file, core::SpecialFlowAttribute::FILENAME, listed_file.getPath().filename().string()); + session.putAttribute(flow_file, core::SpecialFlowAttribute::ABSOLUTE_PATH, (listed_file.getPath().parent_path() / "").string()); + + auto relative_path = std::filesystem::relative(listed_file.getPath().parent_path(), listed_file.getDirectory()); + session.putAttribute(flow_file, core::SpecialFlowAttribute::PATH, (relative_path / "").string()); + + session.putAttribute(flow_file, Size.name, std::to_string(utils::file::file_size(listed_file.getPath()))); + + if (auto windows_file_times = utils::file::getWindowsFileTimes(listed_file.getPath())) { + session.putAttribute(flow_file, CreationTime.name, getDateTimeStr(windows_file_times->creation_time)); + session.putAttribute(flow_file, LastModifiedTime.name, getDateTimeStr(windows_file_times->last_write_time)); + session.putAttribute(flow_file, LastAccessTime.name, getDateTimeStr(windows_file_times->last_access_time)); + } else { + logger_->log_warn("Could not get file attributes due to %s", windows_file_times.error().message()); + } + + session.putAttribute(flow_file, ServiceLocation.name, smb_connection_controller_service_->getPath().string()); + + + return flow_file; +} + +void ListSmb::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { + gsl_Expects(context && session && smb_connection_controller_service_); + + auto connection_error = smb_connection_controller_service_->validateConnection(); + if (connection_error) { + logger_->log_error("Couldn't establish connection to the specified network location due to %s", connection_error.message()); + context->yield(); + return; + } + + auto stored_listing_state = state_manager_->getCurrentState(); + auto latest_listing_state = stored_listing_state; + uint32_t files_listed = 0; + + auto dir = smb_connection_controller_service_->getPath() / input_directory_; + auto process_files = [&](const std::filesystem::path& path, const std::filesystem::path& filename) { + auto listed_file = utils::ListedFile(path / filename, dir); + + if (stored_listing_state.wasObjectListedAlready(listed_file) || !listed_file.matches(file_filter_)) { + return true; + } + + session->transfer(createFlowFile(*session, listed_file), Success); + ++files_listed; + latest_listing_state.updateState(listed_file); + return true; + }; + utils::file::list_dir(dir, process_files, logger_, recurse_subdirectories_); + + state_manager_->storeState(latest_listing_state); + + if (files_listed == 0) { + logger_->log_debug("No new files were found in input directory '%s' to list", dir.string()); + context->yield(); + } +} + +REGISTER_RESOURCE(ListSmb, Processor); + +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/ListSmb.h b/extensions/smb/ListSmb.h new file mode 100644 index 0000000000..489329344a --- /dev/null +++ b/extensions/smb/ListSmb.h @@ -0,0 +1,152 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "SmbConnectionControllerService.h" +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "core/Property.h" +#include "core/PropertyDefinition.h" +#include "core/PropertyDefinitionBuilder.h" +#include "core/OutputAttributeDefinition.h" +#include "core/logging/LoggerConfiguration.h" +#include "utils/Enum.h" +#include "utils/ListingStateManager.h" +#include "utils/file/ListedFile.h" +#include "utils/file/FileUtils.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +class ListSmb : public core::Processor { + public: + explicit ListSmb(std::string name, const utils::Identifier& uuid = {}) + : core::Processor(std::move(name), uuid) { + } + + EXTENSIONAPI static constexpr const char* Description = "Retrieves a listing of files from an SMB share. For each file that is listed, " + "creates a FlowFile that represents the file so that it can be fetched in conjunction with FetchSmb."; + + EXTENSIONAPI static constexpr auto ConnectionControllerService = core::PropertyDefinitionBuilder<>::createProperty("SMB Connection Controller Service") + .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server.") + .isRequired(true) + .withAllowedTypes() + .build(); + EXTENSIONAPI static constexpr auto InputDirectory = core::PropertyDefinitionBuilder<>::createProperty("Input Directory") + .withDescription("The input directory from which files to pull files") + .isRequired(false) + .build(); + EXTENSIONAPI static constexpr auto RecurseSubdirectories = core::PropertyDefinitionBuilder<>::createProperty("Recurse Subdirectories") + .withDescription("Indicates whether to list files from subdirectories of the directory") + .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withDefaultValue("true") + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto FileFilter = core::PropertyDefinitionBuilder<>::createProperty("File Filter") + .withDescription("Only files whose names match the given regular expression will be picked up") + .build(); + EXTENSIONAPI static constexpr auto PathFilter = core::PropertyDefinitionBuilder<>::createProperty("Path Filter") + .withDescription("When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned") + .build(); + EXTENSIONAPI static constexpr auto MinimumFileAge = core::PropertyDefinitionBuilder<>::createProperty("Minimum File Age") + .withDescription("The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored") + .isRequired(true) + .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) + .withDefaultValue("0 sec") + .build(); + EXTENSIONAPI static constexpr auto MaximumFileAge = core::PropertyDefinitionBuilder<>::createProperty("Maximum File Age") + .withDescription("The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored") + .build(); + EXTENSIONAPI static constexpr auto MinimumFileSize = core::PropertyDefinitionBuilder<>::createProperty("Minimum File Size") + .withDescription("The minimum size that a file must be in order to be pulled") + .isRequired(true) + .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) + .withDefaultValue("0 B") + .build(); + EXTENSIONAPI static constexpr auto MaximumFileSize = core::PropertyDefinitionBuilder<>::createProperty("Maximum File Size") + .withDescription("The maximum size that a file can be in order to be pulled") + .build(); + EXTENSIONAPI static constexpr auto IgnoreHiddenFiles = core::PropertyDefinitionBuilder<>::createProperty("Ignore Hidden Files") + .withDescription("Indicates whether or not hidden files should be ignored") + .withPropertyType(core::StandardPropertyTypes::BOOLEAN_TYPE) + .withDefaultValue("true") + .isRequired(true) + .build(); + + EXTENSIONAPI static constexpr auto Properties = std::array{ + ConnectionControllerService, + InputDirectory, + RecurseSubdirectories, + FileFilter, + PathFilter, + MinimumFileAge, + MaximumFileAge, + MinimumFileSize, + MaximumFileSize, + IgnoreHiddenFiles + }; + + EXTENSIONAPI static constexpr auto Success = core::RelationshipDefinition{"success", "All FlowFiles that are received are routed to success"}; + EXTENSIONAPI static constexpr auto Relationships = std::array{Success}; + + EXTENSIONAPI static constexpr auto Filename = core::OutputAttributeDefinition<>{"filename", { Success }, "The name of the file that was read from filesystem."}; + EXTENSIONAPI static constexpr auto Path = core::OutputAttributeDefinition<>{"path", { Success }, + "The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. " + "For example, for a given remote locationsmb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file " + "then the path attribute will be set to \"DIRECTORY/sub/folder\"."}; + EXTENSIONAPI static constexpr auto ServiceLocation = core::OutputAttributeDefinition<>{"serviceLocation", { Success }, + "The SMB URL of the share."}; + EXTENSIONAPI static constexpr auto LastModifiedTime = core::OutputAttributeDefinition<>{"lastModifiedTime", { Success }, + "The timestamp of when the file's content changed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'."}; + EXTENSIONAPI static constexpr auto CreationTime = core::OutputAttributeDefinition<>{"creationTime", { Success }, + "The timestamp of when the file was created in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'."}; + EXTENSIONAPI static constexpr auto LastAccessTime = core::OutputAttributeDefinition<>{"lastAccessTime", { Success }, + "The timestamp of when the file was accessed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'."}; + + + EXTENSIONAPI static constexpr auto Size = core::OutputAttributeDefinition<>{"size", { Success }, "The size of the file in bytes.."}; + + EXTENSIONAPI static constexpr auto OutputAttributes = std::array {Filename, Path, ServiceLocation, LastModifiedTime, CreationTime, LastAccessTime, Size }; + + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; + EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; + EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_FORBIDDEN; + EXTENSIONAPI static constexpr bool IsSingleThreaded = true; + + ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS + + void initialize() override; + void onSchedule(const std::shared_ptr &context, const std::shared_ptr &session_factory) override; + void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) override; + + private: + std::shared_ptr createFlowFile(core::ProcessSession& session, const utils::ListedFile& listed_file); + + std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); + std::filesystem::path input_directory_; + std::shared_ptr smb_connection_controller_service_; + std::unique_ptr state_manager_; + bool recurse_subdirectories_ = true; + utils::FileFilter file_filter_{}; +}; + +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/PutSmb.cpp b/extensions/smb/PutSmb.cpp new file mode 100644 index 0000000000..36864947f3 --- /dev/null +++ b/extensions/smb/PutSmb.cpp @@ -0,0 +1,99 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "PutSmb.h" +#include "utils/gsl.h" +#include "utils/ProcessorConfigUtils.h" +#include "utils/OsUtils.h" +#include "utils/file/FileWriterCallback.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +void PutSmb::initialize() { + setSupportedProperties(Properties); + setSupportedRelationships(Relationships); +} + + +void PutSmb::onSchedule(core::ProcessContext* context, core::ProcessSessionFactory*) { + gsl_Expects(context); + if (auto connection_controller_name = context->getProperty(PutSmb::ConnectionControllerService)) { + smb_connection_controller_service_ = std::dynamic_pointer_cast(context->getControllerService(*connection_controller_name)); + } + if (!smb_connection_controller_service_) { + throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); + } + + create_missing_dirs_ = context->getProperty(PutSmb::CreateMissingDirectories).value_or(true); + conflict_resolution_strategy_ = utils::parseEnumProperty(*context, ConflictResolution); +} + +std::filesystem::path PutSmb::getFilePath(core::ProcessContext& context, const std::shared_ptr& flow_file) { + auto filename = flow_file->getAttribute(core::SpecialFlowAttribute::FILENAME).value_or(flow_file->getUUIDStr()); + return smb_connection_controller_service_->getPath() / context.getProperty(Directory, flow_file).value_or("") / filename; +} + +void PutSmb::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { + gsl_Expects(context && session && smb_connection_controller_service_); + + auto connection_error = smb_connection_controller_service_->validateConnection(); + if (connection_error) { + logger_->log_error("Couldn't establish connection to the specified network location due to %s", connection_error.message()); + context->yield(); + return; + } + + auto flow_file = session->get(); + if (!flow_file) { + context->yield(); + return; + } + + auto full_file_path = getFilePath(*context, flow_file); + + if (utils::file::exists(full_file_path)) { + logger_->log_warn("Destination file %s exists; applying Conflict Resolution Strategy: %s", full_file_path.string(), std::string(magic_enum::enum_name(conflict_resolution_strategy_))); + if (conflict_resolution_strategy_ == FileExistsResolutionStrategy::fail) { + session->transfer(flow_file, Failure); + return; + } else if (conflict_resolution_strategy_ == FileExistsResolutionStrategy::ignore) { + session->transfer(flow_file, Success); + return; + } + } + + if (!utils::file::exists(full_file_path.parent_path()) && create_missing_dirs_) { + logger_->log_debug("Destination directory does not exist; will attempt to create: %s", full_file_path.parent_path().string()); + utils::file::create_dir(full_file_path.parent_path(), true); + } + + bool success = false; + + utils::FileWriterCallback file_writer_callback(full_file_path); + auto read_result = session->read(flow_file, std::ref(file_writer_callback)); + if (io::isError(read_result)) { + logger_->log_error("Failed to write to %s", full_file_path.string()); + success = false; + } else { + success = file_writer_callback.commit(); + } + + session->transfer(flow_file, success ? Success : Failure); +} + +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/PutSmb.h b/extensions/smb/PutSmb.h new file mode 100644 index 0000000000..f1fbe44144 --- /dev/null +++ b/extensions/smb/PutSmb.h @@ -0,0 +1,95 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include "core/Processor.h" +#include "core/ProcessSession.h" +#include "utils/Enum.h" +#include "SmbConnectionControllerService.h" +#include "core/logging/LoggerConfiguration.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +class PutSmb : public core::Processor { + public: + explicit PutSmb(std::string name, const utils::Identifier& uuid = {}) + : core::Processor(std::move(name), uuid) { + } + + ~PutSmb() override = default; + + enum class FileExistsResolutionStrategy { + fail, + replace, + ignore + }; + + EXTENSIONAPI static constexpr const char* Description = "Writes the contents of a FlowFile to an smb network location"; + + EXTENSIONAPI static constexpr auto ConnectionControllerService = core::PropertyDefinitionBuilder<>::createProperty("SMB Connection Controller Service") + .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server.") + .isRequired(true) + .withAllowedTypes() + .build(); + EXTENSIONAPI static constexpr auto Directory = core::PropertyDefinitionBuilder<>::createProperty("Directory") + .withDescription("The output directory to which to put files") + .supportsExpressionLanguage(true) + .withDefaultValue(".") + .build(); + EXTENSIONAPI static constexpr auto ConflictResolution = core::PropertyDefinitionBuilder<3>::createProperty("Conflict Resolution Strategy") + .withDescription("Indicates what should happen when a file with the same name already exists in the output directory") + .withDefaultValue(magic_enum::enum_name(FileExistsResolutionStrategy::fail)) + .withAllowedValues(magic_enum::enum_names()) + .build(); + EXTENSIONAPI static constexpr auto CreateMissingDirectories = core::PropertyDefinitionBuilder<0, 0, 1>::createProperty("Create Missing Directories") + .withDescription("If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure.") + .withDefaultValue("true") + .isRequired(true) + .withDependentProperties({Directory.name}) + .build(); + + EXTENSIONAPI static constexpr auto Properties = std::array{ ConnectionControllerService, Directory, ConflictResolution, CreateMissingDirectories}; + + EXTENSIONAPI static constexpr auto Success = core::RelationshipDefinition{"success", "All files are routed to success"}; + EXTENSIONAPI static constexpr auto Failure = core::RelationshipDefinition{"failure", "Failed files (conflict, write failure, etc.) are transferred to failure"}; + EXTENSIONAPI static constexpr auto Relationships = std::array{Success, Failure}; + + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; + EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; + EXTENSIONAPI static constexpr core::annotation::Input InputRequirement = core::annotation::Input::INPUT_REQUIRED; + EXTENSIONAPI static constexpr bool IsSingleThreaded = false; + + ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_PROCESSORS + + void onSchedule(core::ProcessContext *context, core::ProcessSessionFactory *sessionFactory) override; + void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override; + void initialize() override; + + private: + std::filesystem::path getFilePath(core::ProcessContext& context, const std::shared_ptr& flow_file); + bool create_missing_dirs_ = true; + FileExistsResolutionStrategy conflict_resolution_strategy_ = FileExistsResolutionStrategy::fail; + std::shared_ptr smb_connection_controller_service_; + std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); +}; + +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/SmbConnectionControllerService.cpp b/extensions/smb/SmbConnectionControllerService.cpp new file mode 100644 index 0000000000..ad8ab5fa6e --- /dev/null +++ b/extensions/smb/SmbConnectionControllerService.cpp @@ -0,0 +1,107 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "SmbConnectionControllerService.h" +#include "core/Resource.h" +#include "utils/OsUtils.h" +#include "utils/expected.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +void SmbConnectionControllerService::initialize() { + setSupportedProperties(Properties); +} + +void SmbConnectionControllerService::onEnable() { + std::string hostname; + std::string share; + + if (!getProperty(Hostname, hostname)) + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Missing hostname"); + + if (!getProperty(Share, share)) + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Missing share"); + + + server_path_ = "\\\\" + hostname + "\\" + share; + + auto password = getProperty(Password); + auto username = getProperty(Username); + + if (password.has_value() != username.has_value()) + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Either a username and password must be provided, or neither of them should be provided."); + + if (username.has_value()) + credentials_.emplace(Credentials{.username = *username, .password = *password}); + else + credentials_.reset(); + + ZeroMemory(&net_resource_, sizeof(net_resource_)); + net_resource_.dwType = RESOURCETYPE_DISK; + net_resource_.lpLocalName = nullptr; + net_resource_.lpRemoteName = server_path_.data(); + net_resource_.lpProvider = nullptr; +} + +void SmbConnectionControllerService::notifyStop() { + auto disconnection_result = disconnect(); + if (!disconnection_result) + logger_->log_error("Error while disconnecting from SMB: %s", disconnection_result.error().message()); +} + +nonstd::expected SmbConnectionControllerService::connect() { + auto connection_result = WNetAddConnection2A(&net_resource_, + credentials_ ? credentials_->password.c_str() : nullptr, + credentials_ ? credentials_->username.c_str() : nullptr, + CONNECT_TEMPORARY); + if (connection_result == NO_ERROR) + return {}; + + return nonstd::make_unexpected(utils::OsUtils::windowsErrorToErrorCode(connection_result)); +} + +nonstd::expected SmbConnectionControllerService::disconnect() { + auto disconnection_result = WNetCancelConnection2A(server_path_.c_str(), 0, true); + if (disconnection_result == NO_ERROR) + return {}; + + return nonstd::make_unexpected(utils::OsUtils::windowsErrorToErrorCode(disconnection_result)); +} + +bool SmbConnectionControllerService::isConnected() { + std::error_code error_code; + auto exists = std::filesystem::exists(server_path_, error_code); + if (error_code) + return false; + return exists; +} + +std::error_code SmbConnectionControllerService::validateConnection() { + if (isConnected()) + return std::error_code(); + auto connection_result = connect(); + if (!connection_result) { + return connection_result.error(); + } + + return std::error_code(); +} + +REGISTER_RESOURCE(SmbConnectionControllerService, ControllerService); + +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/SmbConnectionControllerService.h b/extensions/smb/SmbConnectionControllerService.h new file mode 100644 index 0000000000..4a8f046786 --- /dev/null +++ b/extensions/smb/SmbConnectionControllerService.h @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include + +#include "core/PropertyDefinition.h" +#include "core/PropertyDefinitionBuilder.h" +#include "core/controller/ControllerService.h" +#include "core/logging/Logger.h" +#include "core/logging/LoggerConfiguration.h" +#include "utils/Enum.h" +#include "utils/expected.h" + +namespace org::apache::nifi::minifi::extensions::smb { + +class SmbConnectionController { + public: + virtual nonstd::expected connect() = 0; + virtual nonstd::expected disconnect() = 0; + virtual bool isConnected() const = 0; +}; + +class SmbConnectionControllerService : public core::controller::ControllerService { + public: + EXTENSIONAPI static constexpr const char* Description = "SMB Connection Controller Service"; + + EXTENSIONAPI static constexpr auto Hostname = core::PropertyDefinitionBuilder<>::createProperty("Hostname") + .withDescription("The network host to which files should be written.") + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto Share = core::PropertyDefinitionBuilder<>::createProperty("Share") + .withDescription(R"(The network share to which files should be written. This is the "first folder" after the hostname: \\hostname\[share]\dir1\dir2)") + .isRequired(true) + .build(); + EXTENSIONAPI static constexpr auto Domain = core::PropertyDefinitionBuilder<>::createProperty("Domain") + .withDescription("The domain used for authentication. Optional, in most cases username and password is sufficient.") + .isRequired(false) + .build(); + EXTENSIONAPI static constexpr auto Username = core::PropertyDefinitionBuilder<>::createProperty("Username") + .withDescription("The username used for authentication. If no username is set then anonymous authentication is attempted.") + .isRequired(false) + .build(); + EXTENSIONAPI static constexpr auto Password = core::PropertyDefinitionBuilder<>::createProperty("Password") + .withDescription("The password used for authentication. Required if Username is set.") + .isRequired(false) + .build(); + + static constexpr auto Properties = std::array{ + Hostname, + Share, + Domain, + Username, + Password + }; + + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; + ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_CONTROLLER_SERVICES + + using ControllerService::ControllerService; + + void initialize() override; + + void onEnable() override; + void notifyStop() override; + + void yield() override {} + bool isRunning() const override { return getState() == core::controller::ControllerServiceState::ENABLED; } + bool isWorkAvailable() override { return false; } + + virtual std::error_code validateConnection(); + virtual std::filesystem::path getPath() const { return server_path_; } + + private: + nonstd::expected connect(); + nonstd::expected disconnect(); + bool isConnected(); + + struct Credentials { + std::string username; + std::string password; + }; + + std::optional credentials_; + std::string server_path_; + NETRESOURCEA net_resource_; + std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); +}; +} // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/tests/CMakeLists.txt b/extensions/smb/tests/CMakeLists.txt new file mode 100644 index 0000000000..088d878c9a --- /dev/null +++ b/extensions/smb/tests/CMakeLists.txt @@ -0,0 +1,36 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +file(GLOB SMB_TESTS "*.cpp") + +SET(SMB_TEST_COUNT 0) +FOREACH(testfile ${SMB_TESTS}) + get_filename_component(testfilename "${testfile}" NAME_WE) + add_executable("${testfilename}" "${testfile}") + target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") + target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/smb") + target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/libminifi/test/") + createTests("${testfilename}") + target_link_libraries(${testfilename} Catch2WithMain) + target_link_libraries(${testfilename} minifi-smb) + target_link_libraries(${testfilename} minifi-standard-processors Netapi32) + MATH(EXPR SMB_TEST_COUNT "${SMB_TEST_COUNT}+1") + add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) +ENDFOREACH() +message("-- Finished building ${SMB_TEST_COUNT} SMB related test file(s)...") diff --git a/extensions/smb/tests/FetchSmbTests.cpp b/extensions/smb/tests/FetchSmbTests.cpp new file mode 100644 index 0000000000..27493766cc --- /dev/null +++ b/extensions/smb/tests/FetchSmbTests.cpp @@ -0,0 +1,105 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestBase.h" +#include "Catch.h" +#include "FetchSmb.h" +#include "SmbConnectionControllerService.h" +#include "utils/MockSmbConnectionControllerService.h" +#include "SingleProcessorTestController.h" +#include "OsUtils.h" +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::extensions::smb::test { + +REGISTER_RESOURCE(MockSmbConnectionControllerService, ControllerService); + +TEST_CASE("FetchSmb invalid network path") { + const auto fetch_smb = std::make_shared("FetchSmb"); + minifi::test::SingleProcessorTestController controller{fetch_smb}; + auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Hostname, utils::OsUtils::getHostName().value_or("localhost"))); + REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Share, "some_share_that_does_not_exists")); + REQUIRE(controller.plan->setProperty(fetch_smb, FetchSmb::ConnectionControllerService, "smb_connection_controller_service")); + const auto trigger_results = controller.trigger("", {{std::string(core::SpecialFlowAttribute::FILENAME), "a.foo"}, {std::string(core::SpecialFlowAttribute::PATH), ""}}); + CHECK(trigger_results.at(FetchSmb::Success).empty()); + CHECK(trigger_results.at(FetchSmb::Failure).empty()); + CHECK(fetch_smb->isYield()); +} + +TEST_CASE("FetchSmb tests") { + const auto fetch_smb = std::make_shared("FetchSmb"); + minifi::test::SingleProcessorTestController controller{fetch_smb}; + + auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + auto mock_smb_connection_controller_service = std::dynamic_pointer_cast(smb_connection_node->getControllerServiceImplementation()); + REQUIRE(mock_smb_connection_controller_service); + + constexpr std::string_view a_content = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus malesuada elit odio, sit amet viverra ante venenatis eget."; + constexpr std::string_view b_content = "Phasellus sed pharetra velit. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus."; + constexpr std::string_view original_content = "Morbi blandit tincidunt sem ac interdum. Aenean at mauris non augue rhoncus finibus quis vel augue."; + + mock_smb_connection_controller_service->setPath(controller.createTempDirectory()); + + auto a_expected_attributes = mock_smb_connection_controller_service->addFile("a.foo", a_content, 5min); + auto b_expected_attributes = mock_smb_connection_controller_service->addFile("subdir/b.foo", b_content, 5min); + + REQUIRE(controller.plan->setProperty(fetch_smb, FetchSmb::ConnectionControllerService, "smb_connection_controller_service")); + + SECTION("Without Remote File property") { + } + SECTION("Remote File Property with expression language") { + REQUIRE(controller.plan->setProperty(fetch_smb, FetchSmb::RemoteFile, "${path}/${filename}")); + } + + { + const auto trigger_results = controller.trigger(original_content, {{std::string(core::SpecialFlowAttribute::FILENAME), "a.foo"}, {std::string(core::SpecialFlowAttribute::PATH), ""}}); + + CHECK(trigger_results.at(FetchSmb::Failure).empty()); + REQUIRE(trigger_results.at(FetchSmb::Success).size() == 1); + auto succeeded_flow_file = trigger_results.at(FetchSmb::Success)[0]; + + CHECK(controller.plan->getContent(succeeded_flow_file) == a_content); + CHECK_FALSE(succeeded_flow_file->getAttribute(FetchSmb::ErrorCode.name)); + CHECK_FALSE(succeeded_flow_file->getAttribute(FetchSmb::ErrorMessage.name)); + } + { + const auto trigger_results = controller.trigger(original_content, {{std::string(core::SpecialFlowAttribute::FILENAME), "b.foo"}, {std::string(core::SpecialFlowAttribute::PATH), "subdir"}}); + + CHECK(trigger_results.at(FetchSmb::Failure).empty()); + REQUIRE(trigger_results.at(FetchSmb::Success).size() == 1); + auto succeeded_flow_file = trigger_results.at(FetchSmb::Success)[0]; + + CHECK(controller.plan->getContent(succeeded_flow_file) == b_content); + CHECK_FALSE(succeeded_flow_file->getAttribute(FetchSmb::ErrorCode.name)); + CHECK_FALSE(succeeded_flow_file->getAttribute(FetchSmb::ErrorMessage.name)); + } + { + const auto trigger_results = controller.trigger(original_content, {{std::string(core::SpecialFlowAttribute::FILENAME), "c.foo"}, {std::string(core::SpecialFlowAttribute::PATH), "subdir"}}); + + CHECK(trigger_results.at(FetchSmb::Success).empty()); + REQUIRE(trigger_results.at(FetchSmb::Failure).size() == 1); + auto failed_flow_file = trigger_results.at(FetchSmb::Failure)[0]; + + CHECK(controller.plan->getContent(failed_flow_file) == original_content); + CHECK(failed_flow_file->getAttribute(FetchSmb::ErrorCode.name) == "2"); + CHECK(failed_flow_file->getAttribute(FetchSmb::ErrorMessage.name) == "Error opening file: No such file or directory"); + } +} + +} // namespace org::apache::nifi::minifi::extensions::smb::test diff --git a/extensions/smb/tests/ListAndFetchSmbTests.cpp b/extensions/smb/tests/ListAndFetchSmbTests.cpp new file mode 100644 index 0000000000..76990e2e41 --- /dev/null +++ b/extensions/smb/tests/ListAndFetchSmbTests.cpp @@ -0,0 +1,65 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestBase.h" +#include "Catch.h" +#include "ListSmb.h" +#include "FetchSmb.h" +#include "utils/MockSmbConnectionControllerService.h" +#include "ReadFromFlowFileTestProcessor.h" +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::extensions::smb::test { + +REGISTER_RESOURCE(MockSmbConnectionControllerService, ControllerService); + +using minifi::processors::ReadFromFlowFileTestProcessor; + +TEST_CASE("ListSmb and FetchSmb work together") { + TestController controller; + auto plan = controller.createPlan(); + auto list_smb = std::dynamic_pointer_cast(plan->addProcessor("ListSmb", "list_smb")); + auto fetch_smb = std::dynamic_pointer_cast(plan->addProcessor("FetchSmb", "fetch_smb")); + auto read_from_success_relationship = std::dynamic_pointer_cast(plan->addProcessor("ReadFromFlowFileTestProcessor", "read_from_success_relationship")); + auto read_from_failure_relationship = std::dynamic_pointer_cast(plan->addProcessor("ReadFromFlowFileTestProcessor", "read_from_failure_relationship")); + + plan->addConnection(list_smb, ListSmb::Success, fetch_smb); + + plan->addConnection(fetch_smb, FetchSmb::Success, read_from_success_relationship); + plan->addConnection(fetch_smb, FetchSmb::Failure, read_from_failure_relationship); + + auto smb_connection_node = plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + auto mock_smb_connection_controller_service = std::dynamic_pointer_cast(smb_connection_node->getControllerServiceImplementation()); + REQUIRE(mock_smb_connection_controller_service); + + plan->setProperty(list_smb, ListSmb::ConnectionControllerService, "smb_connection_controller_service"); + plan->setProperty(fetch_smb, FetchSmb::ConnectionControllerService, "smb_connection_controller_service"); + + read_from_success_relationship->setAutoTerminatedRelationships(std::array{ReadFromFlowFileTestProcessor::Success}); + read_from_failure_relationship->setAutoTerminatedRelationships(std::array{ReadFromFlowFileTestProcessor::Success}); + + mock_smb_connection_controller_service->setPath(controller.createTempDirectory()); + constexpr std::string_view content = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus malesuada elit odio, sit amet viverra ante venenatis eget."; + mock_smb_connection_controller_service->addFile("subdir/b.foo", content, 5min); + + controller.runSession(plan); + CHECK(read_from_success_relationship->numberOfFlowFilesRead() == 1); + CHECK(read_from_failure_relationship->numberOfFlowFilesRead() == 0); +} + +} // namespace org::apache::nifi::minifi::extensions::smb::test diff --git a/extensions/smb/tests/ListSmbTests.cpp b/extensions/smb/tests/ListSmbTests.cpp new file mode 100644 index 0000000000..6efc5d40e9 --- /dev/null +++ b/extensions/smb/tests/ListSmbTests.cpp @@ -0,0 +1,121 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestBase.h" +#include "Catch.h" +#include "ListSmb.h" +#include "utils/MockSmbConnectionControllerService.h" +#include "SingleProcessorTestController.h" +#include "range/v3/algorithm/count_if.hpp" +#include "range/v3/algorithm/find_if.hpp" +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::extensions::smb::test { + +REGISTER_RESOURCE(MockSmbConnectionControllerService, ControllerService); + +TEST_CASE("ListSmb invalid network path") { + const auto list_smb = std::make_shared("ListSmb"); + minifi::test::SingleProcessorTestController controller{list_smb}; + auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Hostname, utils::OsUtils::getHostName().value_or("localhost"))); + REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Share, "some_share_that_does_not_exists")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::ConnectionControllerService, "smb_connection_controller_service")); + const auto trigger_results = controller.trigger(); + CHECK(trigger_results.at(ListSmb::Success).empty()); + CHECK(list_smb->isYield()); +} + +bool checkForFlowFileWithAttributes(const std::vector>& result, ListSmbExpectedAttributes expected_attributes) { + auto matching_flow_file = ranges::find_if(result, [&](const auto& flow_file) { return flow_file->getAttribute(core::SpecialFlowAttribute::FILENAME) == expected_attributes.expected_filename; }); + if (matching_flow_file == result.end()) { + return false; + } + expected_attributes.checkAttributes(**matching_flow_file); + return true; +} + +TEST_CASE("ListSmb tests") { + const auto list_smb = std::make_shared("ListSmb"); + minifi::test::SingleProcessorTestController controller{list_smb}; + + auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + auto mock_smb_connection_controller_service = std::dynamic_pointer_cast(smb_connection_node->getControllerServiceImplementation()); + REQUIRE(mock_smb_connection_controller_service); + mock_smb_connection_controller_service->setPath(controller.createTempDirectory()); + + auto a_expected_attributes = mock_smb_connection_controller_service->addFile("a.foo", std::string(10_KiB, 'a'), 5min); + auto b_expected_attributes = mock_smb_connection_controller_service->addFile("b.foo", std::string(13_KiB, 'b'), 1h); + auto c_expected_attributes = mock_smb_connection_controller_service->addFile("c.bar", std::string(1_KiB, 'c'), 2h); + auto d_expected_attributes = mock_smb_connection_controller_service->addFile("subdir/d.foo", std::string(100, 'd'), 10min); + auto e_expected_attributes = mock_smb_connection_controller_service->addFile("subdir2/e.foo", std::string(1, 'e'), 0s); + auto f_expected_attributes = mock_smb_connection_controller_service->addFile("third/f.bar", std::string(50_KiB, 'f'), 30min); + + REQUIRE((a_expected_attributes && b_expected_attributes && c_expected_attributes && d_expected_attributes && e_expected_attributes && f_expected_attributes)); + + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::ConnectionControllerService, "smb_connection_controller_service")); + + SECTION("FileFilter without subdirs") { + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::FileFilter, ".*\\.foo")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "false")); + const auto trigger_results = controller.trigger(); + CHECK(trigger_results.at(ListSmb::Success).size() == 2); + CHECK_FALSE(list_smb->isYield()); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *a_expected_attributes)); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *b_expected_attributes)); + } + + SECTION("PathFilter and FileFilter") { + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::FileFilter, ".*\\.foo")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "true")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::PathFilter, "subdir.*")); + const auto trigger_results = controller.trigger(); + CHECK(trigger_results.at(ListSmb::Success).size() == 2); + CHECK_FALSE(list_smb->isYield()); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *d_expected_attributes)); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *e_expected_attributes)); + } + + SECTION("Subdirs with age restriction") { + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::MinimumFileAge, "3min")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::MaximumFileAge, "59min")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "true")); + const auto trigger_results = controller.trigger(); + CHECK(trigger_results.at(ListSmb::Success).size() == 3); + CHECK_FALSE(list_smb->isYield()); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *a_expected_attributes)); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *d_expected_attributes)); + } + + SECTION("Subdirs with size restriction") { + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::MinimumFileSize, "2 KB")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::MaximumFileSize, "20 KB")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "true")); + const auto trigger_results = controller.trigger(); + CHECK(trigger_results.at(ListSmb::Success).size() == 2); + CHECK_FALSE(list_smb->isYield()); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *a_expected_attributes)); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *b_expected_attributes)); + } + + const auto second_trigger = controller.trigger(); + CHECK(second_trigger.at(ListSmb::Success).empty()); + CHECK(list_smb->isYield()); +} + +} // namespace org::apache::nifi::minifi::extensions::smb::test diff --git a/extensions/smb/tests/PutSmbTests.cpp b/extensions/smb/tests/PutSmbTests.cpp new file mode 100644 index 0000000000..14f5ed6490 --- /dev/null +++ b/extensions/smb/tests/PutSmbTests.cpp @@ -0,0 +1,184 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestBase.h" +#include "Catch.h" +#include "PutSmb.h" +#include "utils/MockSmbConnectionControllerService.h" +#include "SingleProcessorTestController.h" +#include "range/v3/algorithm/count_if.hpp" +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::extensions::smb::test { + +REGISTER_RESOURCE(MockSmbConnectionControllerService, ControllerService); + +std::string checkFileContent(const std::filesystem::path& path) { + gsl_Expects(std::filesystem::exists(path)); + std::ifstream if_stream(path); + return {std::istreambuf_iterator(if_stream), std::istreambuf_iterator()}; +} + +TEST_CASE("PutSmb invalid network path") { + const auto put_smb = std::make_shared("PutSmb"); + minifi::test::SingleProcessorTestController controller{put_smb}; + auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Hostname, utils::OsUtils::getHostName().value_or("localhost"))); + REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Share, "some_share_that_does_not_exists")); + REQUIRE(controller.plan->setProperty(put_smb, PutSmb::ConnectionControllerService, "smb_connection_controller_service")); + const auto trigger_results = controller.trigger("", {{std::string(core::SpecialFlowAttribute::FILENAME), "a.foo"}, {std::string(core::SpecialFlowAttribute::PATH), ""}}); + CHECK(trigger_results.at(PutSmb::Success).empty()); + CHECK(trigger_results.at(PutSmb::Failure).empty()); + CHECK(put_smb->isYield()); +} + +TEST_CASE("PutSmb conflict resolution test") { + const auto put_smb = std::make_shared("PutSmb"); + minifi::test::SingleProcessorTestController controller{put_smb}; + + auto temp_directory = controller.createTempDirectory(); + auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + auto mock_smb_connection_controller_service = std::dynamic_pointer_cast(smb_connection_node->getControllerServiceImplementation()); + REQUIRE(mock_smb_connection_controller_service); + mock_smb_connection_controller_service->setPath(temp_directory); + + controller.plan->setProperty(put_smb, PutSmb::ConnectionControllerService, "smb_connection_controller_service"); + + SECTION("Replace") { + controller.plan->setProperty(put_smb, PutSmb::ConflictResolution, magic_enum::enum_name(PutSmb::FileExistsResolutionStrategy::replace)); + + std::string file_name = "my_file.txt"; + + CHECK_FALSE(std::filesystem::exists(temp_directory / file_name)); + + const auto first_trigger_results = controller.trigger("alpha", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(first_trigger_results.at(PutSmb::Failure).empty()); + CHECK(first_trigger_results.at(PutSmb::Success).size() == 1); + + CHECK(std::filesystem::exists(temp_directory / file_name)); + CHECK(checkFileContent(temp_directory / file_name) == "alpha"); + + const auto second_trigger_results = controller.trigger("beta", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(second_trigger_results.at(PutSmb::Failure).empty()); + CHECK(second_trigger_results.at(PutSmb::Success).size() == 1); + + CHECK(std::filesystem::exists(temp_directory / file_name)); + CHECK(checkFileContent(temp_directory / file_name) == "beta"); + } + + SECTION("Ignore") { + controller.plan->setProperty(put_smb, PutSmb::ConflictResolution, magic_enum::enum_name(PutSmb::FileExistsResolutionStrategy::ignore)); + + std::string file_name = "my_file.txt"; + + CHECK_FALSE(std::filesystem::exists(temp_directory / file_name)); + + const auto first_trigger_results = controller.trigger("alpha", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(first_trigger_results.at(PutSmb::Failure).empty()); + CHECK(first_trigger_results.at(PutSmb::Success).size() == 1); + + CHECK(std::filesystem::exists(temp_directory / file_name)); + CHECK(checkFileContent(temp_directory / file_name) == "alpha"); + + const auto second_trigger_results = controller.trigger("beta", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(second_trigger_results.at(PutSmb::Failure).empty()); + CHECK(second_trigger_results.at(PutSmb::Success).size() == 1); + + CHECK(std::filesystem::exists(temp_directory / file_name)); + CHECK(checkFileContent(temp_directory / file_name) == "alpha"); + } + + SECTION("Fail") { + controller.plan->setProperty(put_smb, PutSmb::ConflictResolution, magic_enum::enum_name(PutSmb::FileExistsResolutionStrategy::fail)); + + std::string file_name = "my_file.txt"; + + CHECK_FALSE(std::filesystem::exists(temp_directory / file_name)); + + const auto first_trigger_results = controller.trigger("alpha", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(first_trigger_results.at(PutSmb::Failure).empty()); + CHECK(first_trigger_results.at(PutSmb::Success).size() == 1); + + CHECK(std::filesystem::exists(temp_directory / file_name)); + CHECK(checkFileContent(temp_directory / file_name) == "alpha"); + + const auto second_trigger_results = controller.trigger("beta", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(second_trigger_results.at(PutSmb::Failure).size() == 1); + CHECK(second_trigger_results.at(PutSmb::Success).empty()); + + CHECK(std::filesystem::exists(temp_directory / file_name)); + CHECK(checkFileContent(temp_directory / file_name) == "alpha"); + } +} + +TEST_CASE("PutSmb create missing dirs test") { + const auto put_smb = std::make_shared("PutSmb"); + minifi::test::SingleProcessorTestController controller{put_smb}; + + auto temp_directory = controller.createTempDirectory(); + auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); + auto mock_smb_connection_controller_service = std::dynamic_pointer_cast(smb_connection_node->getControllerServiceImplementation()); + REQUIRE(mock_smb_connection_controller_service); + mock_smb_connection_controller_service->setPath(temp_directory); + + controller.plan->setProperty(put_smb, PutSmb::ConnectionControllerService, "smb_connection_controller_service"); + controller.plan->setProperty(put_smb, PutSmb::Directory, "a/b"); + + SECTION("Create missing dirs") { + controller.plan->setProperty(put_smb, PutSmb::CreateMissingDirectories, "true"); + std::string file_name = "my_file.txt"; + + auto expected_path = temp_directory / "a" / "b" / file_name; + + CHECK_FALSE(std::filesystem::exists(expected_path)); + + const auto first_trigger_results = controller.trigger("alpha", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(first_trigger_results.at(PutSmb::Failure).empty()); + CHECK(first_trigger_results.at(PutSmb::Success).size() == 1); + + REQUIRE(std::filesystem::exists(expected_path)); + CHECK(checkFileContent(expected_path) == "alpha"); + } + + SECTION("Don't create missing dirs") { + controller.plan->setProperty(put_smb, PutSmb::CreateMissingDirectories, "false"); + std::string file_name = "my_file.txt"; + + auto expected_path = temp_directory / "a" / "b" / file_name; + + CHECK_FALSE(std::filesystem::exists(expected_path)); + + const auto first_trigger_results = controller.trigger("alpha", {{std::string(core::SpecialFlowAttribute::FILENAME), file_name}}); + + CHECK(first_trigger_results.at(PutSmb::Failure).size() == 1); + CHECK(first_trigger_results.at(PutSmb::Success).empty()); + + CHECK_FALSE(std::filesystem::exists(expected_path)); + } +} + + + +} // namespace org::apache::nifi::minifi::extensions::smb::test diff --git a/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp b/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp new file mode 100644 index 0000000000..70b7e8d2e1 --- /dev/null +++ b/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp @@ -0,0 +1,72 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "TestBase.h" +#include "Catch.h" +#include "SmbConnectionControllerService.h" +#include "utils/TempSmbShare.h" + +namespace org::apache::nifi::minifi::extensions::smb::test { + +struct SmbConnectionControllerServiceFixture { + SmbConnectionControllerServiceFixture() = default; + + TestController test_controller_{}; + std::shared_ptr plan_ = test_controller_.createPlan(); + std::shared_ptr smb_connection_node_ = plan_->addController("SmbConnectionControllerService", "smb_connection_controller_service"); + std::shared_ptr smb_connection_ = std::dynamic_pointer_cast(smb_connection_node_->getControllerServiceImplementation()); +}; + + +TEST_CASE_METHOD(SmbConnectionControllerServiceFixture, "SmbConnectionControllerService onEnable throws when empty") { + REQUIRE_THROWS(plan_->finalize()); +} + +TEST_CASE_METHOD(SmbConnectionControllerServiceFixture, "SmbConnectionControllerService anonymous connection") { + auto temp_directory = test_controller_.createTempDirectory(); + auto share_local_name = temp_directory.filename().wstring(); + + auto temp_smb_share = TempSmbShare::create(share_local_name, temp_directory.wstring()); + if (!temp_smb_share && temp_smb_share.error() == std::error_code(5, std::system_category())) { + SKIP("SmbConnectionControllerService tests needs administrator privileges"); + return; + } + + + SECTION("Valid share") { + plan_->setProperty(smb_connection_node_, SmbConnectionControllerService::Hostname, "localhost"); + plan_->setProperty(smb_connection_node_, SmbConnectionControllerService::Share, minifi::utils::OsUtils::wideStringToString(share_local_name)); + + REQUIRE_NOTHROW(plan_->finalize()); + + auto connection_error = smb_connection_->validateConnection(); + CHECK_FALSE(connection_error); + } + + SECTION("Invalid share") { + plan_->setProperty(smb_connection_node_, SmbConnectionControllerService::Hostname, "localhost"); + plan_->setProperty(smb_connection_node_, SmbConnectionControllerService::Share, "invalid_share_name"); + + REQUIRE_NOTHROW(plan_->finalize()); + + auto connection_error = smb_connection_->validateConnection(); + CHECK(connection_error); + } +} + +} // namespace org::apache::nifi::minifi::extensions::smb::test diff --git a/extensions/smb/tests/utils/MockSmbConnectionControllerService.h b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h new file mode 100644 index 0000000000..a3ad615e53 --- /dev/null +++ b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include "../../SmbConnectionControllerService.h" +#include "core/controller/ControllerService.h" +#include "utils/OsUtils.h" +#include "ListSmb.h" +#include "Catch.h" + +using namespace std::literals::chrono_literals; + +namespace org::apache::nifi::minifi::extensions::smb::test { + +struct ListSmbExpectedAttributes { + std::string expected_filename; + std::string expected_path; + std::string expected_service_location; + std::chrono::system_clock::time_point expected_last_modified_time; + std::chrono::system_clock::time_point expected_creation_time; + std::chrono::system_clock::time_point expected_last_access_time; + std::string expected_size; + + void checkAttributes(core::FlowFile& flow_file) { + CHECK(flow_file.getAttribute(ListSmb::Filename.name) == expected_filename); + CHECK(flow_file.getAttribute(ListSmb::Path.name) == expected_path); + CHECK(flow_file.getAttribute(ListSmb::ServiceLocation.name) == expected_service_location); + auto last_modified_time_from_attribute = utils::timeutils::parseDateTimeStr(*flow_file.getAttribute(ListSmb::LastModifiedTime.name)); + auto creation_time_from_attribute = utils::timeutils::parseDateTimeStr(*flow_file.getAttribute(ListSmb::CreationTime.name)); + auto last_access_time_from_attribute = utils::timeutils::parseDateTimeStr(*flow_file.getAttribute(ListSmb::LastAccessTime.name)); + + CHECK(std::chrono::abs(expected_last_modified_time - *last_modified_time_from_attribute) < 1s); + CHECK(std::chrono::abs(expected_creation_time - *creation_time_from_attribute) < 1s); + CHECK(std::chrono::abs(expected_last_access_time - *last_access_time_from_attribute) < 1s); + CHECK(flow_file.getAttribute(ListSmb::Size.name) == expected_size); + } +}; + +class MockSmbConnectionControllerService : public SmbConnectionControllerService { + public: + using SmbConnectionControllerService::SmbConnectionControllerService; + EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; + ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_CONTROLLER_SERVICES + + void onEnable() override {} + void notifyStop() override {} + + std::error_code validateConnection() override { + if (server_path_) + return {}; + return std::make_error_code(std::errc::not_connected); + } + std::filesystem::path getPath() const override { + gsl_Expects(server_path_); + return *server_path_; + } + + void setPath(std::filesystem::path path) { server_path_ = std::move(path);} + + nonstd::expected addFile(const std::filesystem::path& relative_path, + std::string_view content, + std::chrono::file_clock::duration age) { + auto full_path = getPath() / relative_path; + std::filesystem::create_directories(full_path.parent_path()); + { + std::ofstream out_file(full_path, std::ios::binary | std::ios::out); + if (!out_file.is_open()) + return nonstd::make_unexpected(std::make_error_code(std::errc::bad_file_descriptor)); + out_file << content; + } + auto last_write_time_error = utils::file::set_last_write_time(full_path, std::chrono::file_clock::now() - age); + if (!last_write_time_error) + return nonstd::make_unexpected(std::make_error_code(std::errc::bad_file_descriptor)); + auto current_time = std::chrono::system_clock::now(); + auto path = relative_path.parent_path().empty() ? (std::filesystem::path(".") / "").string() : (relative_path.parent_path() / "").string(); + return ListSmbExpectedAttributes{ + .expected_filename = relative_path.filename().string(), + .expected_path = path, + .expected_service_location = server_path_->string(), + .expected_last_modified_time = current_time-age, + .expected_creation_time = current_time, + .expected_last_access_time = current_time, + .expected_size = fmt::format("{}", content.size())}; + } + + private: + std::optional server_path_ = std::nullopt; +}; +} // namespace org::apache::nifi::minifi::extensions::smb::test diff --git a/extensions/smb/tests/utils/TempSmbShare.h b/extensions/smb/tests/utils/TempSmbShare.h new file mode 100644 index 0000000000..3c6aa98c79 --- /dev/null +++ b/extensions/smb/tests/utils/TempSmbShare.h @@ -0,0 +1,76 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include "windows.h" +#include "lm.h" +#include "utils/OsUtils.h" +#include "utils/expected.h" +#include "TestUtils.h" +#include "ListSmb.h" + +using namespace std::literals::chrono_literals; + +namespace org::apache::nifi::minifi::extensions::smb::test { + +class TempSmbShare { + public: + TempSmbShare(TempSmbShare&& other) = default; + + ~TempSmbShare() { + if (!net_name_.empty()) + NetShareDel(nullptr, net_name_.data(), 0); + } + + static nonstd::expected create(std::wstring net_name, std::wstring path) { + SHARE_INFO_502 shareInfo{}; + std::wstring remark = L"SMB share to test SMB capabilities of minifi"; + shareInfo.shi502_netname = net_name.data(); + shareInfo.shi502_type = STYPE_DISKTREE; + shareInfo.shi502_remark = remark.data(); + shareInfo.shi502_permissions = ACCESS_ALL; + shareInfo.shi502_max_uses = -1; + shareInfo.shi502_current_uses = 0; + shareInfo.shi502_path = path.data(); + shareInfo.shi502_passwd = nullptr; + shareInfo.shi502_reserved = 0; + shareInfo.shi502_security_descriptor = nullptr; + + DWORD netshare_result = NetShareAdd(nullptr, 502, reinterpret_cast(&shareInfo), nullptr); + if (netshare_result == NERR_Success) { + auto asd = TempSmbShare(std::move(net_name), std::move(path)); + return asd; + } + return nonstd::make_unexpected(utils::OsUtils::windowsErrorToErrorCode(netshare_result)); + } + + std::filesystem::path getPath() const { + return path_; + } + + private: + TempSmbShare(std::wstring net_name, std::wstring path) : net_name_(std::move(net_name)), path_(std::move(path)) {} + + std::wstring net_name_; + std::wstring path_; +}; + +} // namespace org::apache::nifi::minifi::extensions::smb::test diff --git a/extensions/standard-processors/processors/FetchFile.cpp b/extensions/standard-processors/processors/FetchFile.cpp index c8d991669c..8b0e75005d 100644 --- a/extensions/standard-processors/processors/FetchFile.cpp +++ b/extensions/standard-processors/processors/FetchFile.cpp @@ -21,7 +21,7 @@ #include #include "utils/ProcessorConfigUtils.h" -#include "utils/FileReaderCallback.h" +#include "utils/file/FileReaderCallback.h" #include "utils/file/FileUtils.h" #include "core/Resource.h" @@ -52,9 +52,9 @@ std::filesystem::path FetchFile::getFileToFetch(core::ProcessContext& context, c return file_to_fetch_path; } - flow_file->getAttribute("absolute.path", file_to_fetch_path); + flow_file->getAttribute(core::SpecialFlowAttribute::ABSOLUTE_PATH, file_to_fetch_path); std::string filename; - flow_file->getAttribute("filename", filename); + flow_file->getAttribute(core::SpecialFlowAttribute::FILENAME, filename); return std::filesystem::path(file_to_fetch_path) / filename; } diff --git a/extensions/standard-processors/processors/GetFile.cpp b/extensions/standard-processors/processors/GetFile.cpp index 8cee464694..75b7d7cd9b 100644 --- a/extensions/standard-processors/processors/GetFile.cpp +++ b/extensions/standard-processors/processors/GetFile.cpp @@ -28,7 +28,7 @@ #include "core/ProcessSession.h" #include "core/Resource.h" #include "core/TypedValues.h" -#include "utils/FileReaderCallback.h" +#include "utils/file/FileReaderCallback.h" #include "utils/RegexUtils.h" using namespace std::literals::chrono_literals; diff --git a/extensions/standard-processors/processors/ListFile.cpp b/extensions/standard-processors/processors/ListFile.cpp index ee22a07817..130926572f 100644 --- a/extensions/standard-processors/processors/ListFile.cpp +++ b/extensions/standard-processors/processors/ListFile.cpp @@ -47,122 +47,67 @@ void ListFile::onSchedule(const std::shared_ptr &context, context->getProperty(RecurseSubdirectories, recurse_subdirectories_); std::string value; - if (context->getProperty(FileFilter, value) && !value.empty()) { - file_filter_ = std::regex(value); + if (context->getProperty(FileFilter.name, value) && !value.empty()) { + file_filter_.filename_filter = std::regex(value); } - if (recurse_subdirectories_ && context->getProperty(PathFilter, value) && !value.empty()) { - path_filter_ = std::regex(value); + if (recurse_subdirectories_ && context->getProperty(PathFilter.name, value) && !value.empty()) { + file_filter_.path_filter = std::regex(value); } if (auto minimum_file_age = context->getProperty(MinimumFileAge)) { - minimum_file_age_ = minimum_file_age->getMilliseconds(); + file_filter_.minimum_file_age = minimum_file_age->getMilliseconds(); } - if (auto maximum_file_age = context->getProperty(MaximumFileAge) | utils::flatMap(&core::TimePeriodValue::fromString)) { - maximum_file_age_ = maximum_file_age->getMilliseconds(); + if (auto maximum_file_age = context->getProperty(MaximumFileAge)) { + file_filter_.maximum_file_age = maximum_file_age->getMilliseconds(); } - uint64_t int_value = 0; - if (context->getProperty(MinimumFileSize, value) && !value.empty() && core::Property::StringToInt(value, int_value)) { - minimum_file_size_ = int_value; + if (auto minimum_file_size = context->getProperty(MinimumFileSize)) { + file_filter_.minimum_file_size = minimum_file_size->getValue(); } - if (context->getProperty(MaximumFileSize, value) && !value.empty() && core::Property::StringToInt(value, int_value)) { - maximum_file_size_ = int_value; + if (auto maximum_file_size = context->getProperty(MaximumFileSize)) { + file_filter_.maximum_file_size = maximum_file_size->getValue(); } - context->getProperty(IgnoreHiddenFiles, ignore_hidden_files_); + context->getProperty(IgnoreHiddenFiles.name, file_filter_.ignore_hidden_files); } -bool ListFile::fileMatchesFilters(const ListedFile& listed_file) { - if (ignore_hidden_files_ && utils::file::FileUtils::is_hidden(listed_file.full_file_path)) { - logger_->log_debug("File '%s' is hidden so it will not be listed", listed_file.full_file_path.string()); - return false; - } - - if (file_filter_) { - const auto file_name = listed_file.full_file_path.filename(); - - if (!std::regex_match(file_name.string(), *file_filter_)) { - logger_->log_debug("File '%s' does not match file filter so it will not be listed", listed_file.full_file_path.string()); - return false; - } - } - - if (path_filter_) { - const auto relative_path = std::filesystem::relative(listed_file.full_file_path.parent_path(), input_directory_); - if (!std::regex_match(relative_path.string(), *path_filter_)) { - logger_->log_debug("Relative path '%s' does not match path filter so file '%s' will not be listed", relative_path.string(), listed_file.full_file_path.string()); - return false; - } - } - - if (minimum_file_age_ || maximum_file_age_) { - const auto file_age = std::chrono::duration_cast(std::chrono::system_clock::now() - listed_file.getLastModified()); - - if (minimum_file_age_ && file_age < *minimum_file_age_) { - logger_->log_debug("File '%s' does not meet the minimum file age requirement so it will not be listed", listed_file.full_file_path.string()); - return false; - } - - if (maximum_file_age_ && file_age > *maximum_file_age_) { - logger_->log_debug("File '%s' does not meet the maximum file age requirement so it will not be listed", listed_file.full_file_path.string()); - return false; - } - } - - if (minimum_file_size_ || maximum_file_size_) { - const auto file_size = utils::file::file_size(listed_file.full_file_path); - - if (minimum_file_size_ && file_size < *minimum_file_size_) { - logger_->log_debug("File '%s' does not meet the minimum file size requirement so it will not be listed", listed_file.full_file_path.string()); - return false; - } - - if (maximum_file_size_ && *maximum_file_size_ < file_size) { - logger_->log_debug("File '%s' does not meet the maximum file size requirement so it will not be listed", listed_file.full_file_path.string()); - return false; - } - } - - return true; -} - -std::shared_ptr ListFile::createFlowFile(core::ProcessSession& session, const ListedFile& listed_file) { +std::shared_ptr ListFile::createFlowFile(core::ProcessSession& session, const utils::ListedFile& listed_file) { auto flow_file = session.create(); - session.putAttribute(flow_file, core::SpecialFlowAttribute::FILENAME, listed_file.full_file_path.filename().string()); - session.putAttribute(flow_file, core::SpecialFlowAttribute::ABSOLUTE_PATH, (listed_file.full_file_path.parent_path() / "").string()); + session.putAttribute(flow_file, core::SpecialFlowAttribute::FILENAME, listed_file.getPath().filename().string()); + session.putAttribute(flow_file, core::SpecialFlowAttribute::ABSOLUTE_PATH, (listed_file.getPath().parent_path() / "").string()); - auto relative_path = std::filesystem::relative(listed_file.full_file_path.parent_path(), input_directory_); + auto relative_path = std::filesystem::relative(listed_file.getPath().parent_path(), listed_file.getDirectory()); session.putAttribute(flow_file, core::SpecialFlowAttribute::PATH, (relative_path / "").string()); - session.putAttribute(flow_file, "file.size", std::to_string(utils::file::file_size(listed_file.full_file_path))); - session.putAttribute(flow_file, "file.lastModifiedTime", utils::timeutils::getDateTimeStr(std::chrono::time_point_cast(listed_file.last_modified_time))); + session.putAttribute(flow_file, ListFile::FileSize.name, std::to_string(utils::file::file_size(listed_file.getPath()))); + session.putAttribute(flow_file, ListFile::FileLastModifiedTime.name, utils::timeutils::getDateTimeStr(std::chrono::time_point_cast(listed_file.getLastModified()))); - if (auto permission_string = utils::file::FileUtils::get_permission_string(listed_file.full_file_path)) { - session.putAttribute(flow_file, "file.permissions", *permission_string); + if (auto permission_string = utils::file::FileUtils::get_permission_string(listed_file.getPath())) { + session.putAttribute(flow_file, ListFile::FilePermissions.name, *permission_string); } else { - logger_->log_warn("Failed to get permissions of file '%s'", listed_file.full_file_path.string()); - session.putAttribute(flow_file, "file.permissions", ""); + logger_->log_warn("Failed to get permissions of file '%s'", listed_file.getPath().string()); + session.putAttribute(flow_file, ListFile::FilePermissions.name, ""); } - if (auto owner = utils::file::FileUtils::get_file_owner(listed_file.full_file_path)) { - session.putAttribute(flow_file, "file.owner", *owner); + if (auto owner = utils::file::FileUtils::get_file_owner(listed_file.getPath())) { + session.putAttribute(flow_file, ListFile::FileOwner.name, *owner); } else { - logger_->log_warn("Failed to get owner of file '%s'", listed_file.full_file_path.string()); - session.putAttribute(flow_file, "file.owner", ""); + logger_->log_warn("Failed to get owner of file '%s'", listed_file.getPath().string()); + session.putAttribute(flow_file, ListFile::FileOwner.name, ""); } #ifndef WIN32 - if (auto group = utils::file::FileUtils::get_file_group(listed_file.full_file_path)) { - session.putAttribute(flow_file, "file.group", *group); + if (auto group = utils::file::FileUtils::get_file_group(listed_file.getPath())) { + session.putAttribute(flow_file, ListFile::FileGroup.name, *group); } else { - logger_->log_warn("Failed to get group of file '%s'", listed_file.full_file_path.string()); - session.putAttribute(flow_file, "file.group", ""); + logger_->log_warn("Failed to get group of file '%s'", listed_file.getPath().string()); + session.putAttribute(flow_file, ListFile::FileGroup.name, ""); } #else - session.putAttribute(flow_file, "file.group", ""); + session.putAttribute(flow_file, ListFile::FileGroup.name, ""); #endif return flow_file; @@ -170,33 +115,19 @@ std::shared_ptr ListFile::createFlowFile(core::ProcessSession& s void ListFile::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { gsl_Expects(context && session); - logger_->log_trace("ListFile onTrigger"); auto stored_listing_state = state_manager_->getCurrentState(); auto latest_listing_state = stored_listing_state; uint32_t files_listed = 0; auto process_files = [&](const std::filesystem::path& path, const std::filesystem::path& filename) { - ListedFile listed_file; - listed_file.full_file_path = path / filename; - if (auto last_modified_time = utils::file::last_write_time(listed_file.full_file_path)) { - listed_file.last_modified_time = std::chrono::time_point_cast(utils::file::to_sys(*last_modified_time)); - } else { - logger_->log_warn("Could not get last modification time of file '%s'", listed_file.full_file_path.string()); - listed_file.last_modified_time = {}; - } - - if (stored_listing_state.wasObjectListedAlready(listed_file)) { - logger_->log_debug("File '%s' was already listed.", listed_file.full_file_path.string()); - return true; - } + auto listed_file = utils::ListedFile(path / filename, input_directory_); - if (!fileMatchesFilters(listed_file)) { + if (stored_listing_state.wasObjectListedAlready(listed_file) || !listed_file.matches(file_filter_)) { return true; } - auto flow_file = createFlowFile(*session, listed_file); - session->transfer(flow_file, Success); + session->transfer(createFlowFile(*session, listed_file), Success); ++files_listed; latest_listing_state.updateState(listed_file); return true; diff --git a/extensions/standard-processors/processors/ListFile.h b/extensions/standard-processors/processors/ListFile.h index 49de33adf3..40dabf98fe 100644 --- a/extensions/standard-processors/processors/ListFile.h +++ b/extensions/standard-processors/processors/ListFile.h @@ -31,6 +31,7 @@ #include "core/logging/LoggerConfiguration.h" #include "utils/Enum.h" #include "utils/ListingStateManager.h" +#include "utils/file/ListedFile.h" #include "utils/file/FileUtils.h" namespace org::apache::nifi::minifi::processors { @@ -144,33 +145,13 @@ class ListFile : public core::Processor { void onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) override; private: - struct ListedFile : public utils::ListedObject { - [[nodiscard]] std::chrono::time_point getLastModified() const override { - return last_modified_time; - } - - [[nodiscard]] std::string getKey() const override { - return full_file_path.string(); - } - - std::chrono::time_point last_modified_time; - std::filesystem::path full_file_path; - }; - - bool fileMatchesFilters(const ListedFile& listed_file); - std::shared_ptr createFlowFile(core::ProcessSession& session, const ListedFile& listed_file); + std::shared_ptr createFlowFile(core::ProcessSession& session, const utils::ListedFile& listed_file); std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); std::filesystem::path input_directory_; std::unique_ptr state_manager_; bool recurse_subdirectories_ = true; - std::optional file_filter_; - std::optional path_filter_; - std::optional minimum_file_age_; - std::optional maximum_file_age_; - std::optional minimum_file_size_; - std::optional maximum_file_size_; - bool ignore_hidden_files_ = true; + utils::FileFilter file_filter_{}; }; } // namespace org::apache::nifi::minifi::processors diff --git a/extensions/standard-processors/processors/PutFile.cpp b/extensions/standard-processors/processors/PutFile.cpp index c109a66d95..33046feaf8 100644 --- a/extensions/standard-processors/processors/PutFile.cpp +++ b/extensions/standard-processors/processors/PutFile.cpp @@ -26,10 +26,9 @@ #include #include #include -#ifdef WIN32 -#include -#endif #include "utils/file/FileUtils.h" +#include "utils/file/FileWriterCallback.h" +#include "utils/ProcessorConfigUtils.h" #include "utils/gsl.h" #include "core/Resource.h" @@ -43,16 +42,10 @@ void PutFile::initialize() { } void PutFile::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory* /*sessionFactory*/) { - if (!context->getProperty(ConflictResolution, conflict_resolution_)) { - logger_->log_error("Conflict Resolution Strategy attribute is missing or invalid"); - } - - std::string value; - context->getProperty(CreateDirs, value); - try_mkdirs_ = utils::StringUtils::toBool(value).value_or(true); - - if (context->getProperty(MaxDestFiles, value)) { - core::Property::StringToInt(value, max_dest_files_); + conflict_resolution_strategy_ = utils::parseEnumProperty(*context, ConflictResolution); + try_mkdirs_ = context->getProperty(CreateDirs).value_or(true); + if (auto max_dest_files = context->getProperty(MaxDestFiles); max_dest_files && *max_dest_files > 0) { + max_dest_files_ = gsl::narrow_cast(*max_dest_files); } #ifndef WIN32 @@ -61,132 +54,91 @@ void PutFile::onSchedule(core::ProcessContext *context, core::ProcessSessionFact #endif } -void PutFile::onTrigger(core::ProcessContext *context, core::ProcessSession *session) { - if (IsNullOrEmpty(conflict_resolution_)) { - logger_->log_error("Conflict resolution value is invalid"); - context->yield(); - return; - } - - std::shared_ptr flowFile = session->get(); - - // Do nothing if there are no incoming files - if (!flowFile) { - return; - } - - session->remove(flowFile); - +std::optional PutFile::getDestinationPath(core::ProcessContext& context, const std::shared_ptr& flow_file) { std::filesystem::path directory; - - if (auto directory_str = context->getProperty(Directory, flowFile)) { + if (auto directory_str = context.getProperty(Directory, flow_file); directory_str && !directory_str->empty()) { directory = *directory_str; } else { - logger_->log_error("Directory attribute is missing or invalid"); - } - - if (IsNullOrEmpty(directory)) { logger_->log_error("Directory attribute evaluated to invalid value"); - session->transfer(flowFile, Failure); - return; + return std::nullopt; } + auto file_name_str = flow_file->getAttribute(core::SpecialFlowAttribute::FILENAME).value_or(flow_file->getUUIDStr()); - std::string filename; - flowFile->getAttribute(core::SpecialFlowAttribute::FILENAME, filename); - auto tmpFile = tmpWritePath(filename, directory); - - logger_->log_debug("PutFile using temporary file %s", tmpFile.string()); + return directory / file_name_str; +} - // Determine dest full file paths - auto destFile = directory / filename; +bool PutFile::directoryIsFull(const std::filesystem::path& directory) const { + return max_dest_files_ && utils::file::is_directory(directory) && utils::file::countNumberOfFiles(directory) >= *max_dest_files_; +} - logger_->log_debug("PutFile writing file %s into directory %s", filename, directory.string()); +void PutFile::onTrigger(core::ProcessContext *context, core::ProcessSession *session) { + std::shared_ptr flow_file = session->get(); - if ((max_dest_files_ != -1) && utils::file::is_directory(directory)) { - int64_t count = 0; + // Do nothing if there are no incoming files + if (!flow_file) { + return; + } - // Callback, called for each file entry in the listed directory - // Return value is used to break (false) or continue (true) listing - auto lambda = [&count, this](const std::filesystem::path&, const std::filesystem::path&) -> bool { - return ++count < max_dest_files_; - }; + auto dest_path = getDestinationPath(*context, flow_file); + if (!dest_path) { + return session->transfer(flow_file, Failure); + } - utils::file::list_dir(directory, lambda, logger_, false); + logger_->log_debug("PutFile writing file %s into directory %s", dest_path->filename().string(), dest_path->parent_path().string()); - if (count >= max_dest_files_) { - logger_->log_warn("Routing to failure because the output directory %s has at least %u files, which exceeds the " - "configured max number of files", directory.string(), max_dest_files_); - session->transfer(flowFile, Failure); - return; - } + if (directoryIsFull(dest_path->parent_path())) { + logger_->log_warn("Routing to failure because the output directory %s has at least %u files, which exceeds the " + "configured max number of files", dest_path->parent_path().string(), *max_dest_files_); + return session->transfer(flow_file, Failure); } - if (utils::file::exists(destFile)) { - logger_->log_warn("Destination file %s exists; applying Conflict Resolution Strategy: %s", destFile.string(), conflict_resolution_); - - if (conflict_resolution_ == CONFLICT_RESOLUTION_STRATEGY_REPLACE) { - putFile(session, flowFile, tmpFile, destFile, directory); - } else if (conflict_resolution_ == CONFLICT_RESOLUTION_STRATEGY_IGNORE) { - session->transfer(flowFile, Success); - } else { - session->transfer(flowFile, Failure); + if (utils::file::exists(*dest_path)) { + logger_->log_warn("Destination file %s exists; applying Conflict Resolution Strategy: %s", dest_path->string(), std::string(magic_enum::enum_name(conflict_resolution_strategy_))); + if (conflict_resolution_strategy_ == FileExistsResolutionStrategy::fail) { + return session->transfer(flow_file, Failure); + } else if (conflict_resolution_strategy_ == FileExistsResolutionStrategy::ignore) { + return session->transfer(flow_file, Success); } - } else { - putFile(session, flowFile, tmpFile, destFile, directory); } -} -std::filesystem::path PutFile::tmpWritePath(const std::filesystem::path& filename, const std::filesystem::path& directory) { - utils::Identifier tmpFileUuid = id_generator_->generate(); - auto new_filename = std::filesystem::path("." + filename.filename().string()); - new_filename += "." + tmpFileUuid.to_string(); - return (directory / filename.parent_path() / new_filename); + putFile(*session, flow_file, *dest_path); } -bool PutFile::putFile(core::ProcessSession *session, - const std::shared_ptr& flowFile, - const std::filesystem::path& tmpFile, - const std::filesystem::path& destFile, - const std::filesystem::path& destDir) { - if (!utils::file::exists(destDir) && try_mkdirs_) { - logger_->log_debug("Destination directory does not exist; will attempt to create: %s", destDir.string()); - utils::file::create_dir(destDir, true); +void PutFile::prepareDirectory(const std::filesystem::path& directory_path) const { + if (!utils::file::exists(directory_path) && try_mkdirs_) { + logger_->log_debug("Destination directory does not exist; will attempt to create: %s", directory_path.string()); + utils::file::create_dir(directory_path, true); #ifndef WIN32 if (directory_permissions_.valid()) { - utils::file::set_permissions(destDir, directory_permissions_.getValue()); + utils::file::set_permissions(directory_path, directory_permissions_.getValue()); } #endif } +} + +void PutFile::putFile(core::ProcessSession& session, + const std::shared_ptr& flow_file, + const std::filesystem::path& dest_file) { + prepareDirectory(dest_file.parent_path()); bool success = false; - if (flowFile->getSize() > 0) { - ReadCallback cb(tmpFile, destFile); - session->read(flowFile, std::ref(cb)); - logger_->log_debug("Committing %s", destFile.string()); - success = cb.commit(); + utils::FileWriterCallback file_writer_callback(dest_file); + auto read_result = session.read(flow_file, std::ref(file_writer_callback)); + if (io::isError(read_result)) { + logger_->log_error("Failed to write to %s", dest_file.string()); + success = false; } else { - std::ofstream outfile(destFile, std::ios::out | std::ios::binary); - if (!outfile.good()) { - logger_->log_error("Failed to create empty file: %s", destFile.string()); - } else { - success = true; - } + success = file_writer_callback.commit(); } #ifndef WIN32 if (permissions_.valid()) { - utils::file::set_permissions(destFile, permissions_.getValue()); + utils::file::set_permissions(dest_file, permissions_.getValue()); } #endif - if (success) { - session->transfer(flowFile, Success); - return true; - } else { - session->transfer(flowFile, Failure); - } - return false; + session.transfer(flow_file, success ? Success : Failure); } #ifndef WIN32 @@ -227,66 +179,6 @@ void PutFile::getDirectoryPermissions(core::ProcessContext *context) { } #endif -PutFile::ReadCallback::ReadCallback(std::filesystem::path tmp_file, std::filesystem::path dest_file) - : tmp_file_(std::move(tmp_file)), - dest_file_(std::move(dest_file)) { -} - -// Copy the entire file contents to the temporary file -int64_t PutFile::ReadCallback::operator()(const std::shared_ptr& stream) { - // Copy file contents into tmp file - write_succeeded_ = false; - size_t size = 0; - std::array buffer{}; - - std::ofstream tmp_file_os(tmp_file_, std::ios::out | std::ios::binary); - - do { - const auto read = stream->read(buffer); - if (io::isError(read)) return -1; - if (read == 0) break; - tmp_file_os.write(reinterpret_cast(buffer.data()), gsl::narrow(read)); - size += read; - } while (size < stream->size()); - - tmp_file_os.close(); - - if (tmp_file_os) { - write_succeeded_ = true; - } - - return gsl::narrow(size); -} - -// Renames tmp file to final destination -// Returns true if commit succeeded -bool PutFile::ReadCallback::commit() { - bool success = false; - - logger_->log_info("PutFile committing put file operation to %s", dest_file_.string()); - - if (write_succeeded_) { - std::error_code rename_error; - std::filesystem::rename(tmp_file_, dest_file_, rename_error); - if (rename_error) { - logger_->log_info("PutFile commit put file operation to %s failed because std::filesystem::rename call failed", dest_file_.string()); - } else { - success = true; - logger_->log_info("PutFile commit put file operation to %s succeeded", dest_file_.string()); - } - } else { - logger_->log_error("PutFile commit put file operation to %s failed because write failed", dest_file_.string()); - } - - return success; -} - -// Clean up resources -PutFile::ReadCallback::~ReadCallback() { - // Clean up tmp file, if necessary - std::filesystem::remove(tmp_file_); -} - REGISTER_RESOURCE(PutFile, Processor); } // namespace org::apache::nifi::minifi::processors diff --git a/extensions/standard-processors/processors/PutFile.h b/extensions/standard-processors/processors/PutFile.h index b9eff26d33..3dc985158d 100644 --- a/extensions/standard-processors/processors/PutFile.h +++ b/extensions/standard-processors/processors/PutFile.h @@ -1,7 +1,5 @@ + /** - * @file PutFile.h - * PutFile class declaration - * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -34,21 +32,24 @@ #include "core/logging/LoggerConfiguration.h" #include "utils/Id.h" #include "utils/Export.h" +#include "utils/Enum.h" namespace org::apache::nifi::minifi::processors { class PutFile : public core::Processor { public: - static constexpr std::string_view CONFLICT_RESOLUTION_STRATEGY_REPLACE = "replace"; - static constexpr std::string_view CONFLICT_RESOLUTION_STRATEGY_IGNORE = "ignore"; - static constexpr std::string_view CONFLICT_RESOLUTION_STRATEGY_FAIL = "fail"; - explicit PutFile(std::string_view name, const utils::Identifier& uuid = {}) : core::Processor(name, uuid) { } ~PutFile() override = default; + enum class FileExistsResolutionStrategy { + fail, + replace, + ignore + }; + EXTENSIONAPI static constexpr const char* Description = "Writes the contents of a FlowFile to the local file system"; #ifndef WIN32 @@ -68,8 +69,8 @@ class PutFile : public core::Processor { .build(); EXTENSIONAPI static constexpr auto ConflictResolution = core::PropertyDefinitionBuilder<3>::createProperty("Conflict Resolution Strategy") .withDescription("Indicates what should happen when a file with the same name already exists in the output directory") - .withAllowedValues({CONFLICT_RESOLUTION_STRATEGY_FAIL, CONFLICT_RESOLUTION_STRATEGY_IGNORE, CONFLICT_RESOLUTION_STRATEGY_REPLACE}) - .withDefaultValue(CONFLICT_RESOLUTION_STRATEGY_FAIL) + .withDefaultValue(magic_enum::enum_name(FileExistsResolutionStrategy::fail)) + .withAllowedValues(magic_enum::enum_names()) .build(); EXTENSIONAPI static constexpr auto CreateDirs = core::PropertyDefinitionBuilder<0, 1>::createProperty("Create Missing Directories") .withDescription("If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure.") @@ -111,38 +112,15 @@ class PutFile : public core::Processor { void onTrigger(core::ProcessContext *context, core::ProcessSession *session) override; void initialize() override; - class ReadCallback { - public: - ReadCallback(std::filesystem::path tmp_file, std::filesystem::path dest_file); - ~ReadCallback(); - int64_t operator()(const std::shared_ptr& stream); - bool commit(); - - private: - std::shared_ptr logger_{ core::logging::LoggerFactory::getLogger() }; - bool write_succeeded_ = false; - std::filesystem::path tmp_file_; - std::filesystem::path dest_file_; - }; - - /** - * Generate a safe (universally-unique) temporary filename on the same partition - * - * @param filename from which to generate temporary write file path - * @return - */ - static std::filesystem::path tmpWritePath(const std::filesystem::path& filename, const std::filesystem::path& directory); - private: - std::string conflict_resolution_; + FileExistsResolutionStrategy conflict_resolution_strategy_ = FileExistsResolutionStrategy::fail; bool try_mkdirs_ = true; - int64_t max_dest_files_ = -1; + std::optional max_dest_files_ = std::nullopt; - bool putFile(core::ProcessSession *session, - const std::shared_ptr& flowFile, - const std::filesystem::path& tmpFile, - const std::filesystem::path& destFile, - const std::filesystem::path& destDir); + void prepareDirectory(const std::filesystem::path& directory_path) const; + bool directoryIsFull(const std::filesystem::path& directory) const; + std::optional getDestinationPath(core::ProcessContext& context, const std::shared_ptr& flow_file); + void putFile(core::ProcessSession& session, const std::shared_ptr& flow_file, const std::filesystem::path& dest_file); std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); static std::shared_ptr id_generator_; diff --git a/extensions/standard-processors/tests/unit/PutFileTests.cpp b/extensions/standard-processors/tests/unit/PutFileTests.cpp index d793ac6ea7..da30d491d2 100644 --- a/extensions/standard-processors/tests/unit/PutFileTests.cpp +++ b/extensions/standard-processors/tests/unit/PutFileTests.cpp @@ -49,7 +49,6 @@ TEST_CASE("PutFileTest", "[getfileputpfile]") { LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); std::shared_ptr plan = testController.createPlan(); @@ -289,12 +288,6 @@ TEST_CASE("PutFileTestFileExistsReplace", "[getfileputpfile]") { LogTestController::getInstance().reset(); } -TEST_CASE("Test generation of temporary write path", "[putfileTmpWritePath]") { - auto processor = std::make_shared("processorname"); - std::filesystem::path path = std::filesystem::path("a") / std::string("b") / ""; - CHECK(processor->tmpWritePath(path, "").string().starts_with(path.string())); -} - TEST_CASE("PutFileMaxFileCountTest", "[getfileputpfilemaxcount]") { TestController testController; @@ -436,7 +429,6 @@ TEST_CASE("PutFileCreateDirectoryTest", "[PutFileProperties]") { LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); LogTestController::getInstance().setDebug(); std::shared_ptr plan = testController.createPlan(); @@ -479,8 +471,6 @@ TEST_CASE("PutFileCreateDirectoryTest", "[PutFileProperties]") { REQUIRE_FALSE(utils::file::exists(putfiledir)); REQUIRE_FALSE(utils::file::exists(path)); - std::string check = "Failed to create empty file: " + path.string(); - REQUIRE(LogTestController::getInstance().contains(check)); } SECTION("with a non-empty file and create directory property set to true") { @@ -512,8 +502,6 @@ TEST_CASE("PutFileCreateDirectoryTest", "[PutFileProperties]") { REQUIRE_FALSE(utils::file::exists(putfiledir)); REQUIRE_FALSE(utils::file::exists(path)); - std::string check = "PutFile commit put file operation to " + path.string() + " failed because write failed"; - REQUIRE(LogTestController::getInstance().contains(check)); } } diff --git a/extensions/windows-event-log/wel/MetadataWalker.cpp b/extensions/windows-event-log/wel/MetadataWalker.cpp index 11c2fc75ca..d0942c4012 100644 --- a/extensions/windows-event-log/wel/MetadataWalker.cpp +++ b/extensions/windows-event-log/wel/MetadataWalker.cpp @@ -159,10 +159,6 @@ std::map MetadataWalker::getIdentifiers() const { return replaced_identifiers_; } -std::string MetadataWalker::to_string(const wchar_t* pChar) { - return std::wstring_convert>().to_bytes(pChar); -} - template requires std::is_convertible_v, std::string> void MetadataWalker::updateText(pugi::xml_node &node, const std::string &field_name, Fn &&fn) { diff --git a/libminifi/include/c2/triggers/FileUpdateTrigger.h b/libminifi/include/c2/triggers/FileUpdateTrigger.h index 7458410551..1c75325fbb 100644 --- a/libminifi/include/c2/triggers/FileUpdateTrigger.h +++ b/libminifi/include/c2/triggers/FileUpdateTrigger.h @@ -103,9 +103,9 @@ class FileUpdateTrigger : public C2Trigger { return true; } - std::optional getLastUpdate() const; + std::optional getLastUpdate() const; - void setLastUpdate(const std::optional &last_update); + void setLastUpdate(const std::optional &last_update); protected: std::string file_; @@ -114,7 +114,7 @@ class FileUpdateTrigger : public C2Trigger { private: std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(); mutable std::mutex last_update_lock; - std::optional last_update_; + std::optional last_update_; }; } // namespace org::apache::nifi::minifi::c2 diff --git a/libminifi/include/core/logging/Logger.h b/libminifi/include/core/logging/Logger.h index fc0c71081d..e35d155555 100644 --- a/libminifi/include/core/logging/Logger.h +++ b/libminifi/include/core/logging/Logger.h @@ -153,6 +153,16 @@ class Logger : public BaseLogger { Logger(Logger const&) = delete; Logger& operator=(Logger const&) = delete; + + /** + * @brief Log critical message + * @param format format string ('man printf' for syntax) + * @warning does not check @p log or @p format for null. Caller must ensure parameters and format string lengths match + */ + template + void log_critical(const char * const format, Args&& ...args) { + log(spdlog::level::critical, format, std::forward(args)...); + } /** * @brief Log error message * @param format format string ('man printf' for syntax) @@ -252,4 +262,5 @@ class Logger : public BaseLogger { #define LOG_WARN(x) LogBuilder((x).get(), org::apache::nifi::minifi::core::logging::LOG_LEVEL::warn) +#define LOG_CRITICAL(x) LogBuilder((x).get(), org::apache::nifi::minifi::core::logging::LOG_LEVEL::critical) } // namespace org::apache::nifi::minifi::core::logging diff --git a/libminifi/include/utils/LogUtils.h b/libminifi/include/utils/LogUtils.h index 3244653953..6dc8df015e 100644 --- a/libminifi/include/utils/LogUtils.h +++ b/libminifi/include/utils/LogUtils.h @@ -29,6 +29,7 @@ enum class LogLevelOption { LOGGING_INFO, LOGGING_WARN, LOGGING_ERROR, + LOGGING_CRITICAL, LOGGING_OFF }; @@ -50,6 +51,9 @@ void logWithLevel(const std::shared_ptr& logger, LogLevel case LogLevelOption::LOGGING_ERROR: logger->log_error(std::forward(args)...); break; + case LogLevelOption::LOGGING_CRITICAL: + logger->log_critical(std::forward(args)...); + break; case LogLevelOption::LOGGING_OFF: default: break; @@ -74,6 +78,8 @@ constexpr customize_t enum_name(LogLevelOption value) noexcept { return "WARN"; case LogLevelOption::LOGGING_ERROR: return "ERROR"; + case LogLevelOption::LOGGING_CRITICAL: + return "CRITICAL"; case LogLevelOption::LOGGING_OFF: return "OFF"; } diff --git a/libminifi/include/utils/OsUtils.h b/libminifi/include/utils/OsUtils.h index 8378067493..0f73052795 100644 --- a/libminifi/include/utils/OsUtils.h +++ b/libminifi/include/utils/OsUtils.h @@ -51,6 +51,10 @@ std::string getMachineArchitecture(); #ifdef WIN32 /// Resolves common identifiers extern std::string resolve_common_identifiers(const std::string &id); + +std::wstring stringToWideString(const std::string& string); + +std::string wideStringToString(const std::wstring& wide_string); #endif std::optional getHostName(); diff --git a/libminifi/include/utils/FileReaderCallback.h b/libminifi/include/utils/file/FileReaderCallback.h similarity index 100% rename from libminifi/include/utils/FileReaderCallback.h rename to libminifi/include/utils/file/FileReaderCallback.h diff --git a/libminifi/include/utils/file/FileUtils.h b/libminifi/include/utils/file/FileUtils.h index 98a0af5843..207ef08778 100644 --- a/libminifi/include/utils/file/FileUtils.h +++ b/libminifi/include/utils/file/FileUtils.h @@ -16,6 +16,7 @@ */ #pragma once +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include "utils/expected.h" #ifndef WIN32 #include @@ -84,9 +86,9 @@ namespace org::apache::nifi::minifi::utils::file { namespace FileUtils = ::org::apache::nifi::minifi::utils::file; -std::chrono::system_clock::time_point to_sys(std::filesystem::file_time_type file_time); +std::chrono::system_clock::time_point to_sys(std::chrono::file_clock::time_point file_time); -std::filesystem::file_time_type from_sys(std::chrono::system_clock::time_point sys_time); +std::chrono::file_clock::time_point from_sys(std::chrono::system_clock::time_point sys_time); inline int64_t delete_dir(const std::filesystem::path& path, bool delete_files_recursively = true) { // Empty path is interpreted as the root of the current partition on Windows, which should not be allowed @@ -118,7 +120,7 @@ inline std::chrono::time_point{}; } -inline std::optional last_write_time(const std::filesystem::path& path) { +inline std::optional last_write_time(const std::filesystem::path& path) { std::error_code ec; auto result = std::filesystem::last_write_time(path, ec); if (ec.value() == 0) { @@ -127,10 +129,12 @@ inline std::optional last_write_time(const std: return std::nullopt; } -inline bool set_last_write_time(const std::filesystem::path& path, std::filesystem::file_time_type new_time) { +inline nonstd::expected set_last_write_time(const std::filesystem::path& path, std::chrono::file_clock::time_point new_time) { std::error_code ec; std::filesystem::last_write_time(path, new_time, ec); - return ec.value() == 0; + if (ec) + return nonstd::make_unexpected(ec); + return {}; } inline uint64_t file_size(const std::filesystem::path& path) { @@ -570,4 +574,23 @@ inline std::optional get_relative_path(const std::filesys return std::filesystem::relative(path, base_path); } +inline size_t countNumberOfFiles(const std::filesystem::path& path) { + size_t counter = 0; + for (const auto& entry : std::filesystem::directory_iterator(path)) { + if (entry.is_regular_file()) + ++counter; + } + return counter; +} + +#ifdef WIN32 +struct WindowsFileTimes { + std::chrono::file_clock::time_point creation_time; + std::chrono::file_clock::time_point last_access_time; + std::chrono::file_clock::time_point last_write_time; +}; + +nonstd::expected getWindowsFileTimes(const std::filesystem::path& path); +#endif + } // namespace org::apache::nifi::minifi::utils::file diff --git a/libminifi/include/utils/file/FileWriterCallback.h b/libminifi/include/utils/file/FileWriterCallback.h new file mode 100644 index 0000000000..c284d9ab01 --- /dev/null +++ b/libminifi/include/utils/file/FileWriterCallback.h @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include "io/StreamPipe.h" +#include "utils/expected.h" +#include "core/logging/LoggerConfiguration.h" + +namespace org::apache::nifi::minifi::utils { + +class FileWriterCallback { + public: + explicit FileWriterCallback(std::filesystem::path dest_path); + ~FileWriterCallback(); + int64_t operator()(const std::shared_ptr& stream); + bool commit(); + + + private: + bool write_succeeded_ = false; + std::filesystem::path temp_path_; + std::filesystem::path dest_path_; +}; +} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/include/utils/file/ListedFile.h b/libminifi/include/utils/file/ListedFile.h new file mode 100644 index 0000000000..9961f7e01a --- /dev/null +++ b/libminifi/include/utils/file/ListedFile.h @@ -0,0 +1,104 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "../ListingStateManager.h" + +namespace org::apache::nifi::minifi::utils { + +struct FileFilter { + std::optional filename_filter; + std::optional path_filter; + std::optional minimum_file_age; + std::optional maximum_file_age; + std::optional minimum_file_size; + std::optional maximum_file_size; + bool ignore_hidden_files = true; +}; + +class ListedFile : public utils::ListedObject { + public: + explicit ListedFile(std::filesystem::path full_file_path, std::filesystem::path input_directory) : full_file_path_(std::move(full_file_path)), input_directory_(std::move(input_directory)) { + if (auto last_write_time = utils::file::last_write_time(full_file_path_)) { + last_modified_time_ = utils::file::to_sys(*last_write_time); + } + } + + [[nodiscard]] std::chrono::system_clock::time_point getLastModified() const override { + return std::chrono::time_point_cast(last_modified_time_); + } + + [[nodiscard]] std::string getKey() const override { + return full_file_path_.string(); + } + + [[nodiscard]] const std::filesystem::path& getPath() const { + return full_file_path_; + } + + [[nodiscard]] const std::filesystem::path& getDirectory() const { + return input_directory_; + } + + [[nodiscard]] bool matches(const FileFilter& file_filter) { + if (file_filter.ignore_hidden_files && utils::file::FileUtils::is_hidden(full_file_path_)) + return false; + + return fileAgeIsBetween(file_filter.minimum_file_age, file_filter.maximum_file_age) && + fileSizeIsBetween(file_filter.minimum_file_size, file_filter.maximum_file_size) && + matchesRegex(file_filter.filename_filter, file_filter.path_filter); + } + + private: + [[nodiscard]] bool matchesRegex(const std::optional& file_regex, const std::optional& path_regex) const { + if (file_regex && !std::regex_match(full_file_path_.filename().string(), *file_regex)) + return false; + if (path_regex && !std::regex_match(std::filesystem::relative(full_file_path_.parent_path(), input_directory_).string(), *path_regex)) + return false; + return true; + } + + [[nodiscard]] bool fileAgeIsBetween(const std::optional minimum_age, const std::optional maximum_age) const { + auto file_age = getAge(); + if (minimum_age && minimum_age > file_age) + return false; + if (maximum_age && maximum_age < file_age) + return false; + return true; + } + + [[nodiscard]] bool fileSizeIsBetween(const std::optional minimum_size, const std::optional maximum_size) const { + if (minimum_size && minimum_size > getSize()) + return false; + if (maximum_size && maximum_size < getSize()) + return false; + return true; + } + + [[nodiscard]] std::chrono::system_clock::duration getAge() const { return std::chrono::system_clock::now() - last_modified_time_;} + [[nodiscard]] size_t getSize() const { return utils::file::file_size(full_file_path_); } + std::chrono::system_clock::time_point last_modified_time_; + std::filesystem::path full_file_path_; + std::filesystem::path input_directory_; +}; + +} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/src/c2/triggers/FileUpdateTrigger.cpp b/libminifi/src/c2/triggers/FileUpdateTrigger.cpp index 3ed74dfee1..c3e94daa2f 100644 --- a/libminifi/src/c2/triggers/FileUpdateTrigger.cpp +++ b/libminifi/src/c2/triggers/FileUpdateTrigger.cpp @@ -39,12 +39,12 @@ C2Payload FileUpdateTrigger::getAction() { return response_payload; } -std::optional FileUpdateTrigger::getLastUpdate() const { +std::optional FileUpdateTrigger::getLastUpdate() const { std::lock_guard lock(last_update_lock); return last_update_; } -void FileUpdateTrigger::setLastUpdate(const std::optional &last_update) { +void FileUpdateTrigger::setLastUpdate(const std::optional &last_update) { std::lock_guard lock(last_update_lock); last_update_ = last_update; } diff --git a/libminifi/src/core/logging/Logger.cpp b/libminifi/src/core/logging/Logger.cpp index 8a751890f4..aafd820236 100644 --- a/libminifi/src/core/logging/Logger.cpp +++ b/libminifi/src/core/logging/Logger.cpp @@ -100,7 +100,7 @@ bool Logger::should_log(const LOG_LEVEL &level) { void Logger::log_string(LOG_LEVEL level, std::string str) { switch (level) { case critical: - log_warn(str.c_str()); + log_critical(str.c_str()); break; case err: log_error(str.c_str()); diff --git a/libminifi/src/utils/NetworkInterfaceInfo.cpp b/libminifi/src/utils/NetworkInterfaceInfo.cpp index fc9dc5753d..e86bb327fa 100644 --- a/libminifi/src/utils/NetworkInterfaceInfo.cpp +++ b/libminifi/src/utils/NetworkInterfaceInfo.cpp @@ -18,11 +18,9 @@ #include "utils/net/Socket.h" #include "core/logging/LoggerConfiguration.h" #ifdef WIN32 -#include -#include #include -#include #pragma comment(lib, "IPHLPAPI.lib") +#include "utils/OsUtils.h" #else #include #include @@ -37,19 +35,9 @@ namespace org::apache::nifi::minifi::utils { std::shared_ptr NetworkInterfaceInfo::logger_ = core::logging::LoggerFactory::getLogger(); #ifdef WIN32 -namespace { -std::string utf8_encode(const std::wstring& wstr) { - if (wstr.empty()) - return std::string(); - int size_needed = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, nullptr, 0, nullptr, nullptr); - std::string result_string(size_needed, 0); - WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, result_string.data(), size_needed, nullptr, nullptr); - return result_string; -} -} NetworkInterfaceInfo::NetworkInterfaceInfo(const IP_ADAPTER_ADDRESSES* adapter) { - name_ = utf8_encode(adapter->FriendlyName); + name_ = OsUtils::wideStringToString(adapter->FriendlyName); for (auto unicast_address = adapter->FirstUnicastAddress; unicast_address != nullptr; unicast_address = unicast_address->Next) { if (unicast_address->Address.lpSockaddr->sa_family == AF_INET) { ip_v4_addresses_.push_back(net::sockaddr_ntop(unicast_address->Address.lpSockaddr)); @@ -94,7 +82,7 @@ std::vector NetworkInterfaceInfo::getNetworkInterfaceInfos return network_adapters; } std::vector bytes(buffer_length, 0); - IP_ADAPTER_ADDRESSES* adapter = reinterpret_cast(bytes.data()); + auto* adapter = reinterpret_cast(bytes.data()); get_adapters_err = GetAdaptersAddresses(0, 0, nullptr, adapter, &buffer_length); if (NO_ERROR != get_adapters_err) { logger_->log_error("GetAdaptersAddresses failed: %lu", get_adapters_err); diff --git a/libminifi/src/utils/OsUtils.cpp b/libminifi/src/utils/OsUtils.cpp index e1c03a947a..670e41aa35 100644 --- a/libminifi/src/utils/OsUtils.cpp +++ b/libminifi/src/utils/OsUtils.cpp @@ -89,31 +89,32 @@ std::string OsUtils::userIdToUsername(const std::string &uid) { name = uid; if (!name.empty()) { #ifdef _WIN32 - const auto resolved_name = resolve_common_identifiers(name); + auto resolved_name = resolve_common_identifiers(name); if (!resolved_name.empty()) { return resolved_name; } // First call to LookupAccountSid to get the buffer sizes. - PSID pSidOwner = NULL; - const auto guard_pSidOwner = gsl::finally([&pSidOwner]() { if (pSidOwner != NULL) { LocalFree(pSidOwner); } }); + PSID pSidOwner = nullptr; + const auto guard_pSidOwner = gsl::finally([&pSidOwner]() { if (pSidOwner != nullptr) { LocalFree(pSidOwner); } }); if (ConvertStringSidToSidA(name.c_str(), &pSidOwner)) { SID_NAME_USE sidType = SidTypeUnknown; - DWORD windowsAccountNameSize = 0, dwwindowsDomainSize = 0; + DWORD windowsAccountNameSize = 0; + DWORD dwwindowsDomainSize = 0; /* We can use a unique ptr with a deleter here but some of the calls below require we use global alloc -- so a global deleter to call GlobalFree won't buy us a ton unless we anticipate requiring more of this. If we do I suggest we break this out until a subset of OsUtils into our own convenience functions. */ - LPTSTR windowsDomain = NULL; - LPTSTR windowsAccount = NULL; + LPTSTR windowsDomain = nullptr; + LPTSTR windowsAccount = nullptr; /* The first call will be to obtain sizes for domain and account, after which we will allocate the memory and free it after. In some cases youc an replace GlobalAlloc with */ - LookupAccountSid(NULL /** local computer **/, pSidOwner, + LookupAccountSid(nullptr /** local computer **/, pSidOwner, windowsAccount, (LPDWORD)&windowsAccountNameSize, windowsDomain, @@ -132,7 +133,7 @@ std::string OsUtils::userIdToUsername(const std::string &uid) { } if (LookupAccountSid( - NULL, + nullptr, pSidOwner, windowsAccount, (LPDWORD)&windowsAccountNameSize, @@ -337,6 +338,36 @@ std::optional OsUtils::getHostName() { return {hostname}; } +#ifdef WIN32 +std::wstring OsUtils::stringToWideString(const std::string& string) { + if (string.empty()) + return {}; + + const auto size_needed = MultiByteToWideChar(CP_UTF8, 0, &string.at(0), static_cast(string.size()), nullptr, 0); + if (size_needed <= 0) { + throw std::runtime_error("MultiByteToWideChar() failed: " + std::to_string(size_needed)); + } + + std::wstring result(size_needed, 0); + MultiByteToWideChar(CP_UTF8, 0, &string.at(0), static_cast(string.size()), &result.at(0), size_needed); + return result; +} + +std::string OsUtils::wideStringToString(const std::wstring& wide_string) { + if (wide_string.empty()) + return {}; + + const auto size_needed = WideCharToMultiByte(CP_UTF8, 0, &wide_string.at(0), static_cast(wide_string.size()), nullptr, 0, nullptr, nullptr); + if (size_needed <= 0) { + throw std::runtime_error("WideCharToMultiByte() failed: " + std::to_string(size_needed)); + } + + std::string result(size_needed, 0); + WideCharToMultiByte(CP_UTF8, 0, &wide_string.at(0), static_cast(wide_string.size()), &result.at(0), size_needed, nullptr, nullptr); + return result; +} +#endif + std::optional OsUtils::getSystemLoadAverage() { #ifndef WIN32 double load_avg[1]; diff --git a/libminifi/src/utils/FileReaderCallback.cpp b/libminifi/src/utils/file/FileReaderCallback.cpp similarity index 98% rename from libminifi/src/utils/FileReaderCallback.cpp rename to libminifi/src/utils/file/FileReaderCallback.cpp index aa4fb6c693..cbfec6c3b7 100644 --- a/libminifi/src/utils/FileReaderCallback.cpp +++ b/libminifi/src/utils/file/FileReaderCallback.cpp @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "utils/FileReaderCallback.h" +#include "utils/file/FileReaderCallback.h" #include #include diff --git a/libminifi/src/utils/file/FileUtils.cpp b/libminifi/src/utils/file/FileUtils.cpp index 79f8df9a64..a96aa2840f 100644 --- a/libminifi/src/utils/file/FileUtils.cpp +++ b/libminifi/src/utils/file/FileUtils.cpp @@ -25,6 +25,10 @@ #include "utils/Literals.h" #include "utils/Searcher.h" +#ifdef WIN32 +#include "utils/OsUtils.h" +#endif + namespace org::apache::nifi::minifi::utils::file { uint64_t computeChecksum(const std::filesystem::path& file_name, uint64_t up_to_position) { @@ -66,7 +70,7 @@ bool contains(const std::filesystem::path& file_path, std::string_view text_to_s return std::search(view.begin(), view.end(), searcher) != view.end(); } -std::chrono::system_clock::time_point to_sys(std::filesystem::file_time_type file_time) { +std::chrono::system_clock::time_point to_sys(std::chrono::file_clock::time_point file_time) { using namespace std::chrono; // NOLINT(build/namespaces) #if defined(WIN32) // workaround for https://github.com/microsoft/STL/issues/2446 @@ -81,7 +85,7 @@ std::chrono::system_clock::time_point to_sys(std::filesystem::file_time_type fil #endif } -std::filesystem::file_time_type from_sys(std::chrono::system_clock::time_point sys_time) { +std::chrono::file_clock::time_point from_sys(std::chrono::system_clock::time_point sys_time) { using namespace std::chrono; // NOLINT(build/namespaces) #if defined(WIN32) // workaround for https://github.com/microsoft/STL/issues/2446 @@ -96,4 +100,22 @@ std::filesystem::file_time_type from_sys(std::chrono::system_clock::time_point s #endif } +#ifdef WIN32 +std::chrono::file_clock::time_point fileTimePointFromFileTime(const FILETIME& filetime) { + static_assert(std::ratio_equal_v>, "file_clock duration must be 100 nanoseconds"); + std::chrono::file_clock::duration duration{(static_cast(filetime.dwHighDateTime) << 32) | filetime.dwLowDateTime}; + return std::chrono::file_clock::time_point{duration}; +} + +nonstd::expected getWindowsFileTimes(const std::filesystem::path& path) { + WIN32_FILE_ATTRIBUTE_DATA file_attributes; + auto get_file_attributes_result = GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &file_attributes); + if (!get_file_attributes_result) + return nonstd::make_unexpected(utils::OsUtils::windowsErrorToErrorCode(GetLastError())); + return WindowsFileTimes{.creation_time = fileTimePointFromFileTime(file_attributes.ftCreationTime), + .last_access_time = fileTimePointFromFileTime(file_attributes.ftLastAccessTime), + .last_write_time = fileTimePointFromFileTime(file_attributes.ftLastWriteTime)}; +} +#endif // WIN32 + } // namespace org::apache::nifi::minifi::utils::file diff --git a/libminifi/src/utils/file/FileWriterCallback.cpp b/libminifi/src/utils/file/FileWriterCallback.cpp new file mode 100644 index 0000000000..3a1542efc1 --- /dev/null +++ b/libminifi/src/utils/file/FileWriterCallback.cpp @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/file/FileWriterCallback.h" +#include + +namespace org::apache::nifi::minifi::utils { + +FileWriterCallback::FileWriterCallback(std::filesystem::path dest_path) + : dest_path_(std::move(dest_path)) { + auto new_filename = std::filesystem::path("." + dest_path_.filename().string() + "." + utils::IdGenerator::getIdGenerator()->generate().to_string()); + temp_path_ = dest_path_.parent_path() / new_filename; +} + +FileWriterCallback::~FileWriterCallback() { + std::error_code remove_error; + std::filesystem::remove(temp_path_, remove_error); +} + +int64_t FileWriterCallback::operator()(const std::shared_ptr& stream) { + write_succeeded_ = false; + size_t size = 0; + std::array buffer{}; + + std::ofstream tmp_file_os(temp_path_, std::ios::out | std::ios::binary); + + do { + const auto read = stream->read(buffer); + if (io::isError(read)) return -1; + if (read == 0) break; + tmp_file_os.write(reinterpret_cast(buffer.data()), gsl::narrow(read)); + size += read; + } while (size < stream->size()); + + tmp_file_os.close(); + + if (tmp_file_os) { + write_succeeded_ = true; + } + + return gsl::narrow(size); +} + +bool FileWriterCallback::commit() { + if (!write_succeeded_) + return false; + + std::error_code rename_error; + std::filesystem::rename(temp_path_, dest_path_, rename_error); + return !rename_error; +} +} // namespace org::apache::nifi::minifi::utils diff --git a/libminifi/test/Utils.h b/libminifi/test/Utils.h index b3ed11854a..5fbc581785 100644 --- a/libminifi/test/Utils.h +++ b/libminifi/test/Utils.h @@ -60,7 +60,7 @@ void matchJSON(const rapidjson::Value& actual, const rapidjson::Value& expected) } else if (expected.IsArray()) { REQUIRE(actual.IsArray()); REQUIRE(actual.Size() == expected.Size()); - for (size_t idx{0}; idx < expected.Size(); ++idx) { + for (rapidjson::SizeType idx{0}; idx < expected.Size(); ++idx) { matchJSON(actual[idx], expected[idx]); } } else { diff --git a/libminifi/test/archive-tests/FocusArchiveTests.cpp b/libminifi/test/archive-tests/FocusArchiveTests.cpp index 51d8a7f363..9ea80b8073 100644 --- a/libminifi/test/archive-tests/FocusArchiveTests.cpp +++ b/libminifi/test/archive-tests/FocusArchiveTests.cpp @@ -45,86 +45,87 @@ const char* FILE_CONTENT[NUM_FILES] = {"Test file 1\n", "Test file 2\n"}; const char* FOCUSED_FILE = FILE_NAMES[0]; const char* FOCUSED_CONTENT = FILE_CONTENT[0]; +namespace org::apache::nifi::minifi::processors::test { TEST_CASE("Test Creation of FocusArchiveEntry", "[getfileCreate]") { TestController testController; - std::shared_ptr processor = std::make_shared("processorname"); + std::shared_ptr processor = std::make_shared("processorname"); REQUIRE(processor->getName() == "processorname"); } TEST_CASE("Test Creation of UnfocusArchiveEntry", "[getfileCreate]") { - TestController testController; - std::shared_ptr processor = std::make_shared("processorname"); - REQUIRE(processor->getName() == "processorname"); - REQUIRE(processor->getUUID()); + TestController testController; + std::shared_ptr processor = std::make_shared("processorname"); + REQUIRE(processor->getName() == "processorname"); + REQUIRE(processor->getUUID()); } TEST_CASE("FocusArchive", "[testFocusArchive]") { - TestController testController; - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); + TestController testController; + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); - std::shared_ptr plan = testController.createPlan(); - std::shared_ptr repo = std::make_shared(); + std::shared_ptr plan = testController.createPlan(); + std::shared_ptr repo = std::make_shared(); - auto dir1 = testController.createTempDirectory(); - auto dir2 = testController.createTempDirectory(); - auto dir3 = testController.createTempDirectory(); + auto dir1 = testController.createTempDirectory(); + auto dir2 = testController.createTempDirectory(); + auto dir3 = testController.createTempDirectory(); - REQUIRE(!dir1.empty()); - REQUIRE(!dir2.empty()); - REQUIRE(!dir3.empty()); - std::shared_ptr getfile = plan->addProcessor("GetFile", "getfileCreate2"); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::Directory, dir1.string()); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::KeepSourceFile, "true"); + REQUIRE(!dir1.empty()); + REQUIRE(!dir2.empty()); + REQUIRE(!dir3.empty()); + std::shared_ptr getfile = plan->addProcessor("GetFile", "getfileCreate2"); + plan->setProperty(getfile, GetFile::Directory, dir1.string()); + plan->setProperty(getfile, GetFile::KeepSourceFile, "true"); - std::shared_ptr fprocessor = plan->addProcessor("FocusArchiveEntry", "focusarchiveCreate", core::Relationship("success", "description"), true); - plan->setProperty(fprocessor, org::apache::nifi::minifi::processors::FocusArchiveEntry::Path, FOCUSED_FILE); + std::shared_ptr fprocessor = plan->addProcessor("FocusArchiveEntry", "focusarchiveCreate", core::Relationship("success", "description"), true); + plan->setProperty(fprocessor, FocusArchiveEntry::Path, FOCUSED_FILE); - std::shared_ptr putfile1 = plan->addProcessor("PutFile", "PutFile1", core::Relationship("success", "description"), true); - plan->setProperty(putfile1, org::apache::nifi::minifi::processors::PutFile::Directory, dir2.string()); - plan->setProperty(putfile1, org::apache::nifi::minifi::processors::PutFile::ConflictResolution, - org::apache::nifi::minifi::processors::PutFile::CONFLICT_RESOLUTION_STRATEGY_REPLACE); + std::shared_ptr putfile1 = plan->addProcessor("PutFile", "PutFile1", core::Relationship("success", "description"), true); + plan->setProperty(putfile1, PutFile::Directory, dir2.string()); + plan->setProperty(putfile1, PutFile::ConflictResolution, magic_enum::enum_name(PutFile::FileExistsResolutionStrategy::replace)); - std::shared_ptr ufprocessor = plan->addProcessor("UnfocusArchiveEntry", "unfocusarchiveCreate", core::Relationship("success", "description"), true); + std::shared_ptr ufprocessor = plan->addProcessor("UnfocusArchiveEntry", "unfocusarchiveCreate", core::Relationship("success", "description"), true); - std::shared_ptr putfile2 = plan->addProcessor("PutFile", "PutFile2", core::Relationship("success", "description"), true); - plan->setProperty(putfile2, org::apache::nifi::minifi::processors::PutFile::Directory, dir3.string()); - plan->setProperty(putfile2, org::apache::nifi::minifi::processors::PutFile::ConflictResolution, - org::apache::nifi::minifi::processors::PutFile::CONFLICT_RESOLUTION_STRATEGY_REPLACE); + std::shared_ptr putfile2 = plan->addProcessor("PutFile", "PutFile2", core::Relationship("success", "description"), true); + plan->setProperty(putfile2, PutFile::Directory, dir3.string()); + plan->setProperty(putfile2, PutFile::ConflictResolution, magic_enum::enum_name(PutFile::FileExistsResolutionStrategy::replace)); - auto archive_path_1 = dir1 / TEST_ARCHIVE_NAME; + auto archive_path_1 = dir1 / TEST_ARCHIVE_NAME; - TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); - build_test_archive(archive_path_1, test_archive_map); + TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); + build_test_archive(archive_path_1, test_archive_map); - REQUIRE(check_archive_contents(archive_path_1, test_archive_map)); + REQUIRE(check_archive_contents(archive_path_1, test_archive_map)); - plan->runNextProcessor(); // GetFile - plan->runNextProcessor(); // FocusArchive - plan->runNextProcessor(); // PutFile 1 (focused) + plan->runNextProcessor(); // GetFile + plan->runNextProcessor(); // FocusArchive + plan->runNextProcessor(); // PutFile 1 (focused) - std::ifstream ifs(dir2 / FOCUSED_FILE, std::ios::in | std::ios::binary | std::ios::ate); + std::ifstream ifs(dir2 / FOCUSED_FILE, std::ios::in | std::ios::binary | std::ios::ate); - auto size = gsl::narrow(ifs.tellg()); - ifs.seekg(0, std::ios::beg); - char *content = new char[size]; - ifs.read(content, size); + auto size = gsl::narrow(ifs.tellg()); + ifs.seekg(0, std::ios::beg); + char* content = new char[size]; + ifs.read(content, size); - REQUIRE(size == strlen(FOCUSED_CONTENT)); - REQUIRE(memcmp(content, FOCUSED_CONTENT, size) == 0); + REQUIRE(size == strlen(FOCUSED_CONTENT)); + REQUIRE(memcmp(content, FOCUSED_CONTENT, size) == 0); - plan->runNextProcessor(); // UnfocusArchive - plan->runNextProcessor(); // PutFile 2 (unfocused) + plan->runNextProcessor(); // UnfocusArchive + plan->runNextProcessor(); // PutFile 2 (unfocused) - auto archive_path_2 = dir3 / TEST_ARCHIVE_NAME; - REQUIRE(check_archive_contents(archive_path_2, test_archive_map)); + auto archive_path_2 = dir3 / TEST_ARCHIVE_NAME; + REQUIRE(check_archive_contents(archive_path_2, test_archive_map)); } + +} // namespace org::apache::nifi::minifi::processors::test diff --git a/libminifi/test/archive-tests/ManipulateArchiveTests.cpp b/libminifi/test/archive-tests/ManipulateArchiveTests.cpp index 0f905674ec..e4db56c9fd 100644 --- a/libminifi/test/archive-tests/ManipulateArchiveTests.cpp +++ b/libminifi/test/archive-tests/ManipulateArchiveTests.cpp @@ -45,55 +45,56 @@ const char* MODIFY_DEST = "modified"; using PROP_MAP_T = std::vector>; +namespace org::apache::nifi::minifi::processors::test { + bool run_archive_test(OrderedTestArchive& input_archive, const OrderedTestArchive& output_archive, const PROP_MAP_T& properties, bool check_attributes = true) { - TestController testController; - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - LogTestController::getInstance().setTrace(); - - std::shared_ptr plan = testController.createPlan(); - std::shared_ptr repo = std::make_shared(); - - auto dir1 = testController.createTempDirectory(); - auto dir2 = testController.createTempDirectory(); - - REQUIRE(!dir1.empty()); - REQUIRE(!dir2.empty()); - - std::shared_ptr getfile = plan->addProcessor("GetFile", "getfileCreate2"); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::Directory, dir1.string()); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::KeepSourceFile, "true"); - - std::shared_ptr maprocessor = plan->addProcessor("ManipulateArchive", "testManipulateArchive", core::Relationship("success", "description"), true); - - for (const auto& [name, value] : properties) { - plan->setProperty(maprocessor, name, value); - } - - std::shared_ptr putfile2 = plan->addProcessor("PutFile", "PutFile2", core::Relationship("success", "description"), true); - plan->setProperty(putfile2, org::apache::nifi::minifi::processors::PutFile::Directory, dir2.string()); - plan->setProperty(putfile2, org::apache::nifi::minifi::processors::PutFile::ConflictResolution, - org::apache::nifi::minifi::processors::PutFile::CONFLICT_RESOLUTION_STRATEGY_REPLACE); - - auto archive_path_1 = dir1 / TEST_ARCHIVE_NAME; - - build_test_archive(archive_path_1, input_archive); - REQUIRE(check_archive_contents(archive_path_1, input_archive, true)); - - plan->runNextProcessor(); // GetFile - plan->runNextProcessor(); // ManipulateArchive - plan->runNextProcessor(); // PutFile 2 (manipulated) - - auto output_path = dir2 / TEST_ARCHIVE_NAME; - return check_archive_contents(output_path, output_archive, check_attributes); + TestController testController; + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + LogTestController::getInstance().setTrace(); + + std::shared_ptr plan = testController.createPlan(); + std::shared_ptr repo = std::make_shared(); + + auto dir1 = testController.createTempDirectory(); + auto dir2 = testController.createTempDirectory(); + + REQUIRE(!dir1.empty()); + REQUIRE(!dir2.empty()); + + std::shared_ptr getfile = plan->addProcessor("GetFile", "getfileCreate2"); + plan->setProperty(getfile, GetFile::Directory, dir1.string()); + plan->setProperty(getfile, GetFile::KeepSourceFile, "true"); + + std::shared_ptr maprocessor = plan->addProcessor("ManipulateArchive", "testManipulateArchive", core::Relationship("success", "description"), true); + + for (const auto& [name, value] : properties) { + plan->setProperty(maprocessor, name, value); + } + + std::shared_ptr putfile2 = plan->addProcessor("PutFile", "PutFile2", core::Relationship("success", "description"), true); + plan->setProperty(putfile2, PutFile::Directory, dir2.string()); + plan->setProperty(putfile2, PutFile::ConflictResolution, magic_enum::enum_name(PutFile::FileExistsResolutionStrategy::replace)); + + auto archive_path_1 = dir1 / TEST_ARCHIVE_NAME; + + build_test_archive(archive_path_1, input_archive); + REQUIRE(check_archive_contents(archive_path_1, input_archive, true)); + + plan->runNextProcessor(); // GetFile + plan->runNextProcessor(); // ManipulateArchive + plan->runNextProcessor(); // PutFile 2 (manipulated) + + auto output_path = dir2 / TEST_ARCHIVE_NAME; + return check_archive_contents(output_path, output_archive, check_attributes); } bool run_archive_test(TAE_MAP_T input_map, TAE_MAP_T output_map, const PROP_MAP_T& properties, bool check_attributes = true) { @@ -109,235 +110,236 @@ bool run_archive_test(TAE_MAP_T input_map, TAE_MAP_T output_map, const PROP_MAP_ TEST_CASE("Test creation of ManipulateArchive", "[manipulatearchiveCreate]") { TestController testController; - std::shared_ptr processor = std::make_shared("processorname"); + std::shared_ptr processor = std::make_shared("processorname"); REQUIRE(processor->getName() == "processorname"); REQUIRE(processor->getUUID()); } TEST_CASE("Test ManipulateArchive Touch", "[testManipulateArchiveTouch]") { - TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); - - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_TOUCH} - }; - - // The other attributes aren't checked, so we can leave them uninitialized - TestArchiveEntry touched_entry; - touched_entry.name = MODIFY_DEST; - touched_entry.content = ""; - touched_entry.size = 0; - touched_entry.type = AE_IFREG; - - // Copy original map and append touched entry - TAE_MAP_T mod_archive_map(test_archive_map); - mod_archive_map[MODIFY_DEST] = touched_entry; - - REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties, false)); + TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); + + PROP_MAP_T properties{ + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_TOUCH} + }; + + // The other attributes aren't checked, so we can leave them uninitialized + TestArchiveEntry touched_entry; + touched_entry.name = MODIFY_DEST; + touched_entry.content = ""; + touched_entry.size = 0; + touched_entry.type = AE_IFREG; + + // Copy original map and append touched entry + TAE_MAP_T mod_archive_map(test_archive_map); + mod_archive_map[MODIFY_DEST] = touched_entry; + + REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties, false)); } TEST_CASE("Test ManipulateArchive Copy", "[testManipulateArchiveCopy]") { - TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); + TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Target, MODIFY_SRC}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_COPY} - }; + PROP_MAP_T properties{ + {ManipulateArchive::Target, MODIFY_SRC}, + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_COPY} + }; - TAE_MAP_T mod_archive_map(test_archive_map); - mod_archive_map[MODIFY_DEST] = test_archive_map[MODIFY_SRC]; + TAE_MAP_T mod_archive_map(test_archive_map); + mod_archive_map[MODIFY_DEST] = test_archive_map[MODIFY_SRC]; - REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties)); + REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties)); } TEST_CASE("Test ManipulateArchive Move", "[testManipulateArchiveMove]") { - TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); + TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Target, MODIFY_SRC}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_MOVE} - }; + PROP_MAP_T properties{ + {ManipulateArchive::Target, MODIFY_SRC}, + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_MOVE} + }; - TAE_MAP_T mod_archive_map(test_archive_map); + TAE_MAP_T mod_archive_map(test_archive_map); - mod_archive_map[MODIFY_DEST] = test_archive_map[MODIFY_SRC]; - mod_archive_map[MODIFY_DEST].name = MODIFY_DEST; + mod_archive_map[MODIFY_DEST] = test_archive_map[MODIFY_SRC]; + mod_archive_map[MODIFY_DEST].name = MODIFY_DEST; - auto it = mod_archive_map.find(MODIFY_SRC); - mod_archive_map.erase(it); + auto it = mod_archive_map.find(MODIFY_SRC); + mod_archive_map.erase(it); - REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties)); + REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties)); } TEST_CASE("Test ManipulateArchive Remove", "[testManipulateArchiveRemove]") { - TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); + TAE_MAP_T test_archive_map = build_test_archive_map(NUM_FILES, FILE_NAMES, FILE_CONTENT); - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Target, MODIFY_SRC}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_REMOVE} - }; + PROP_MAP_T properties{ + {ManipulateArchive::Target, MODIFY_SRC}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_REMOVE} + }; - TAE_MAP_T mod_archive_map(test_archive_map); + TAE_MAP_T mod_archive_map(test_archive_map); - auto it = mod_archive_map.find(MODIFY_SRC); - mod_archive_map.erase(it); + auto it = mod_archive_map.find(MODIFY_SRC); + mod_archive_map.erase(it); - REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties)); + REQUIRE(run_archive_test(test_archive_map, mod_archive_map, properties)); } TEST_CASE("Test ManipulateArchive Ordered Touch (before)", "[testManipulateArchiveOrderedTouchBefore]") { - OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); - - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_TOUCH}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Before, ORDER_ANCHOR} - }; - - // The other attributes aren't checked, so we can leave them uninitialized - TestArchiveEntry touched_entry; - touched_entry.name = MODIFY_DEST; - touched_entry.content = ""; - touched_entry.size = 0; - touched_entry.type = AE_IFREG; - - // Copy original map and append touched entry - OrderedTestArchive mod_archive = test_archive; - mod_archive.map[MODIFY_DEST] = touched_entry; - - auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); - mod_archive.order.insert(it, MODIFY_DEST); - - REQUIRE(run_archive_test(test_archive, mod_archive, properties, false)); + OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); + + PROP_MAP_T properties{ + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_TOUCH}, + {ManipulateArchive::Before, ORDER_ANCHOR} + }; + + // The other attributes aren't checked, so we can leave them uninitialized + TestArchiveEntry touched_entry; + touched_entry.name = MODIFY_DEST; + touched_entry.content = ""; + touched_entry.size = 0; + touched_entry.type = AE_IFREG; + + // Copy original map and append touched entry + OrderedTestArchive mod_archive = test_archive; + mod_archive.map[MODIFY_DEST] = touched_entry; + + auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); + mod_archive.order.insert(it, MODIFY_DEST); + + REQUIRE(run_archive_test(test_archive, mod_archive, properties, false)); } TEST_CASE("Test ManipulateArchive Ordered Copy (before)", "[testManipulateArchiveOrderedCopyBefore]") { - OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); - - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Target, MODIFY_SRC}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_COPY}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Before, ORDER_ANCHOR} - }; - - OrderedTestArchive mod_archive = test_archive; - mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; - auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); - mod_archive.order.insert(it, MODIFY_DEST); - - REQUIRE(run_archive_test(test_archive, mod_archive, properties)); + OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); + + PROP_MAP_T properties{ + {ManipulateArchive::Target, MODIFY_SRC}, + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_COPY}, + {ManipulateArchive::Before, ORDER_ANCHOR} + }; + + OrderedTestArchive mod_archive = test_archive; + mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; + auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); + mod_archive.order.insert(it, MODIFY_DEST); + + REQUIRE(run_archive_test(test_archive, mod_archive, properties)); } TEST_CASE("Test ManipulateArchive Ordered Move (before)", "[testManipulateArchiveOrderedMoveBefore]") { - OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); - - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Target, MODIFY_SRC}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_MOVE}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Before, ORDER_ANCHOR} - }; - - OrderedTestArchive mod_archive = test_archive; - - // Update map - mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; - mod_archive.map[MODIFY_DEST].name = MODIFY_DEST; - auto m_it = mod_archive.map.find(MODIFY_SRC); - mod_archive.map.erase(m_it); - - // Update order - auto o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), MODIFY_SRC); - mod_archive.order.erase(o_it); - o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); - mod_archive.order.insert(o_it, MODIFY_DEST); - - REQUIRE(run_archive_test(test_archive, mod_archive, properties)); + OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); + + PROP_MAP_T properties{ + {ManipulateArchive::Target, MODIFY_SRC}, + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_MOVE}, + {ManipulateArchive::Before, ORDER_ANCHOR} + }; + + OrderedTestArchive mod_archive = test_archive; + + // Update map + mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; + mod_archive.map[MODIFY_DEST].name = MODIFY_DEST; + auto m_it = mod_archive.map.find(MODIFY_SRC); + mod_archive.map.erase(m_it); + + // Update order + auto o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), MODIFY_SRC); + mod_archive.order.erase(o_it); + o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); + mod_archive.order.insert(o_it, MODIFY_DEST); + + REQUIRE(run_archive_test(test_archive, mod_archive, properties)); } TEST_CASE("Test ManipulateArchive Ordered Touch (after)", "[testManipulateArchiveOrderedTouchAfter]") { - OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); - - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_TOUCH}, - {org::apache::nifi::minifi::processors::ManipulateArchive::After, ORDER_ANCHOR} - }; - - // The other attributes aren't checked, so we can leave them uninitialized - TestArchiveEntry touched_entry; - touched_entry.name = MODIFY_DEST; - touched_entry.content = ""; - touched_entry.size = 0; - touched_entry.type = AE_IFREG; - - // Copy original map and append touched entry - OrderedTestArchive mod_archive = test_archive; - mod_archive.map[MODIFY_DEST] = touched_entry; - - auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); - it++; - mod_archive.order.insert(it, MODIFY_DEST); - - REQUIRE(run_archive_test(test_archive, mod_archive, properties, false)); + OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); + + PROP_MAP_T properties{ + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_TOUCH}, + {ManipulateArchive::After, ORDER_ANCHOR} + }; + + // The other attributes aren't checked, so we can leave them uninitialized + TestArchiveEntry touched_entry; + touched_entry.name = MODIFY_DEST; + touched_entry.content = ""; + touched_entry.size = 0; + touched_entry.type = AE_IFREG; + + // Copy original map and append touched entry + OrderedTestArchive mod_archive = test_archive; + mod_archive.map[MODIFY_DEST] = touched_entry; + + auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); + it++; + mod_archive.order.insert(it, MODIFY_DEST); + + REQUIRE(run_archive_test(test_archive, mod_archive, properties, false)); } TEST_CASE("Test ManipulateArchive Ordered Copy (after)", "[testManipulateArchiveOrderedCopyAfter]") { - OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); - - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Target, MODIFY_SRC}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_COPY}, - {org::apache::nifi::minifi::processors::ManipulateArchive::After, ORDER_ANCHOR} - }; - - OrderedTestArchive mod_archive = test_archive; - mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; - auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); - it++; - mod_archive.order.insert(it, MODIFY_DEST); - - REQUIRE(run_archive_test(test_archive, mod_archive, properties)); + OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); + + PROP_MAP_T properties{ + {ManipulateArchive::Target, MODIFY_SRC}, + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, + ManipulateArchive::OPERATION_COPY}, + {ManipulateArchive::After, ORDER_ANCHOR} + }; + + OrderedTestArchive mod_archive = test_archive; + mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; + auto it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); + it++; + mod_archive.order.insert(it, MODIFY_DEST); + + REQUIRE(run_archive_test(test_archive, mod_archive, properties)); } TEST_CASE("Test ManipulateArchive Ordered Move (after)", "[testManipulateArchiveOrderedMoveAfter]") { - OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); - - PROP_MAP_T properties { - {org::apache::nifi::minifi::processors::ManipulateArchive::Target, MODIFY_SRC}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Destination, MODIFY_DEST}, - {org::apache::nifi::minifi::processors::ManipulateArchive::Operation, - org::apache::nifi::minifi::processors::ManipulateArchive::OPERATION_MOVE}, - {org::apache::nifi::minifi::processors::ManipulateArchive::After, ORDER_ANCHOR} - }; - - OrderedTestArchive mod_archive = test_archive; - - // Update map - mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; - mod_archive.map[MODIFY_DEST].name = MODIFY_DEST; - auto m_it = mod_archive.map.find(MODIFY_SRC); - mod_archive.map.erase(m_it); - - // Update order - auto o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), MODIFY_SRC); - mod_archive.order.erase(o_it); - o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); - o_it++; - mod_archive.order.insert(o_it, MODIFY_DEST); - - REQUIRE(run_archive_test(test_archive, mod_archive, properties)); + OrderedTestArchive test_archive = build_ordered_test_archive(NUM_FILES, FILE_NAMES, FILE_CONTENT); + + PROP_MAP_T properties{ + {ManipulateArchive::Target, MODIFY_SRC}, + {ManipulateArchive::Destination, MODIFY_DEST}, + {ManipulateArchive::Operation, ManipulateArchive::OPERATION_MOVE}, + {ManipulateArchive::After, ORDER_ANCHOR} + }; + + OrderedTestArchive mod_archive = test_archive; + + // Update map + mod_archive.map[MODIFY_DEST] = test_archive.map[MODIFY_SRC]; + mod_archive.map[MODIFY_DEST].name = MODIFY_DEST; + auto m_it = mod_archive.map.find(MODIFY_SRC); + mod_archive.map.erase(m_it); + + // Update order + auto o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), MODIFY_SRC); + mod_archive.order.erase(o_it); + o_it = std::find(mod_archive.order.begin(), mod_archive.order.end(), ORDER_ANCHOR); + o_it++; + mod_archive.order.insert(o_it, MODIFY_DEST); + + REQUIRE(run_archive_test(test_archive, mod_archive, properties)); } + +} // namespace org::apache::nifi::minifi::processors::test diff --git a/libminifi/test/unit/OsUtilTests.cpp b/libminifi/test/unit/OsUtilTests.cpp index a7777d1770..15c4628cc3 100644 --- a/libminifi/test/unit/OsUtilTests.cpp +++ b/libminifi/test/unit/OsUtilTests.cpp @@ -39,6 +39,26 @@ TEST_CASE("Test userIdToUsername for well-known SIDs", "[OsUtils]") { CHECK_FALSE(minifi::utils::OsUtils::userIdToUsername("S-1-3-4").empty()); CHECK_FALSE(minifi::utils::OsUtils::userIdToUsername("S-1-5-80-0").empty()); } + +TEST_CASE("OsUtils::stringToWideString tests") { + using org::apache::nifi::minifi::utils::OsUtils::stringToWideString; + + CHECK(stringToWideString("árvíztűrő tükörfúrógép") == L"árvíztűrő tükörfúrógép"); + CHECK(stringToWideString("Falsches Üben von Xylophonmusik quält jeden größeren Zwerg.") == L"Falsches Üben von Xylophonmusik quält jeden größeren Zwerg."); + CHECK(stringToWideString("가나다라마바사아자차카타파하") == L"가나다라마바사아자차카타파하"); + CHECK(stringToWideString("العربية تجربة") == L"العربية تجربة"); + CHECK(stringToWideString("פטכןצימסעואבגדהוזחטייכלמנסעפצקרשת") == L"פטכןצימסעואבגדהוזחטייכלמנסעפצקרשת"); +} + +TEST_CASE("OsUtils::wideStringToString tests") { + using org::apache::nifi::minifi::utils::OsUtils::wideStringToString; + + CHECK(wideStringToString(L"árvíztűrő tükörfúrógép") == "árvíztűrő tükörfúrógép"); + CHECK(wideStringToString(L"Falsches Üben von Xylophonmusik quält jeden größeren Zwerg.") == "Falsches Üben von Xylophonmusik quält jeden größeren Zwerg."); + CHECK(wideStringToString(L"가나다라마바사아자차카타파하") == "가나다라마바사아자차카타파하"); + CHECK(wideStringToString(L"العربية تجربة") == "العربية تجربة"); + CHECK(wideStringToString(L"פטכןצימסעואבגדהוזחטייכלמנסעפצקרשת") == "פטכןצימסעואבגדהוזחטייכלמנסעפצקרשת"); +} #endif TEST_CASE("Machine architecture is supported") { diff --git a/run_clang_tidy.sh b/run_clang_tidy.sh index 79bab757d8..9600227e44 100755 --- a/run_clang_tidy.sh +++ b/run_clang_tidy.sh @@ -4,7 +4,7 @@ set -uo pipefail FILE=$1 -EXCLUDED_DIRECTORY=("extensions/pdh" "extensions/windows-event-log" "nanofi") +EXCLUDED_DIRECTORY=("extensions/pdh" "extensions/windows-event-log" "extensions/smb" "nanofi") EXCLUDED_FILES=("WindowsCertStoreLocationTests.cpp") for excluded_file in "${EXCLUDED_FILES[@]}"; do From d1e8d4ce82981a51a7fe2242280029919c69482d Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 23 Aug 2023 10:32:20 +0200 Subject: [PATCH 02/25] Update libminifi/src/utils/file/FileUtils.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Márton Szász --- libminifi/src/utils/file/FileUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libminifi/src/utils/file/FileUtils.cpp b/libminifi/src/utils/file/FileUtils.cpp index a96aa2840f..eaa09fb89f 100644 --- a/libminifi/src/utils/file/FileUtils.cpp +++ b/libminifi/src/utils/file/FileUtils.cpp @@ -102,7 +102,7 @@ std::chrono::file_clock::time_point from_sys(std::chrono::system_clock::time_poi #ifdef WIN32 std::chrono::file_clock::time_point fileTimePointFromFileTime(const FILETIME& filetime) { - static_assert(std::ratio_equal_v>, "file_clock duration must be 100 nanoseconds"); + static_assert(std::ratio_equal_v>, "file_clock duration tick period must be 100 nanoseconds"); std::chrono::file_clock::duration duration{(static_cast(filetime.dwHighDateTime) << 32) | filetime.dwLowDateTime}; return std::chrono::file_clock::time_point{duration}; } From 3ab67dfa7729019c7126297935933fffad2a9eaf Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 23 Aug 2023 11:50:33 +0200 Subject: [PATCH 03/25] added comment and a test windows file_clock epoch --- libminifi/src/utils/file/FileUtils.cpp | 1 + libminifi/test/unit/TimeUtilTests.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/libminifi/src/utils/file/FileUtils.cpp b/libminifi/src/utils/file/FileUtils.cpp index eaa09fb89f..ef22dcbd0b 100644 --- a/libminifi/src/utils/file/FileUtils.cpp +++ b/libminifi/src/utils/file/FileUtils.cpp @@ -102,6 +102,7 @@ std::chrono::file_clock::time_point from_sys(std::chrono::system_clock::time_poi #ifdef WIN32 std::chrono::file_clock::time_point fileTimePointFromFileTime(const FILETIME& filetime) { + // FILETIME contains a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 (UTC). static_assert(std::ratio_equal_v>, "file_clock duration tick period must be 100 nanoseconds"); std::chrono::file_clock::duration duration{(static_cast(filetime.dwHighDateTime) << 32) | filetime.dwLowDateTime}; return std::chrono::file_clock::time_point{duration}; diff --git a/libminifi/test/unit/TimeUtilTests.cpp b/libminifi/test/unit/TimeUtilTests.cpp index 2e56ed0966..9b332a3383 100644 --- a/libminifi/test/unit/TimeUtilTests.cpp +++ b/libminifi/test/unit/TimeUtilTests.cpp @@ -117,6 +117,30 @@ TEST_CASE("Test system_clock epoch", "[systemclockepoch]") { REQUIRE(unix_epoch_plus_3e9_sec.time_since_epoch() == 3000000000s); } +#ifdef WIN32 +TEST_CASE("Test windows file_clock duration period and epoch") { + using namespace std::chrono; + static_assert(std::ratio_equal_v>, "file_clock duration tick period must be 100 nanoseconds"); + auto file_clock_epoch = std::chrono::file_clock::time_point{}; + auto file_clock_epoch_as_sys_time = utils::file::to_sys(file_clock_epoch); + system_clock::time_point expected_windows_file_epoch = date::sys_days(date::January / 1 / 1601); + CHECK(file_clock_epoch_as_sys_time == expected_windows_file_epoch); +} + +TEST_CASE("Test windows FILETIME epoch") { + SYSTEMTIME system_time; + FILETIME file_time{.dwLowDateTime=0, .dwHighDateTime=0}; + FileTimeToSystemTime(&file_time, &system_time); + CHECK(system_time.wYear == 1601); + CHECK(system_time.wMonth == 1); + CHECK(system_time.wDay == 1); + CHECK(system_time.wHour == 0); + CHECK(system_time.wMinute == 0); + CHECK(system_time.wSecond == 0); + CHECK(system_time.wMilliseconds == 0); +} +#endif + TEST_CASE("Test clock resolutions", "[clockresolutiontests]") { using namespace std::chrono; CHECK(std::is_constructible::value); // The resolution of the system_clock is at least microseconds From 19cde5ca42517ce8a8f0f0fba33fda2b41c6fbe9 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 24 Aug 2023 13:21:14 +0200 Subject: [PATCH 04/25] review changes --- extensions/smb/FetchSmb.cpp | 16 ++++------------ extensions/smb/ListSmb.cpp | 11 ++--------- extensions/smb/PutSmb.cpp | 11 ++--------- .../smb/SmbConnectionControllerService.cpp | 17 +++++++++++++++-- extensions/smb/SmbConnectionControllerService.h | 10 +++------- .../standard-processors/processors/PutFile.h | 1 - 6 files changed, 26 insertions(+), 40 deletions(-) diff --git a/extensions/smb/FetchSmb.cpp b/extensions/smb/FetchSmb.cpp index bb199c36a4..5488929411 100644 --- a/extensions/smb/FetchSmb.cpp +++ b/extensions/smb/FetchSmb.cpp @@ -28,16 +28,11 @@ void FetchSmb::initialize() { void FetchSmb::onSchedule(const std::shared_ptr& context, const std::shared_ptr&) { gsl_Expects(context); - if (auto connection_controller_name = context->getProperty(FetchSmb::ConnectionControllerService)) { - smb_connection_controller_service_ = std::dynamic_pointer_cast(context->getControllerService(*connection_controller_name)); - } - if (!smb_connection_controller_service_) { - throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); - } + smb_connection_controller_service_ = SmbConnectionControllerService::getFromProperty(*context, FetchSmb::ConnectionControllerService); } namespace { -std::filesystem::path getPath(core::ProcessContext& context, const std::shared_ptr& flow_file) { +std::filesystem::path getTargetRelativePath(core::ProcessContext& context, const std::shared_ptr& flow_file) { auto remote_file = context.getProperty(FetchSmb::RemoteFile, flow_file); if (remote_file && !remote_file->empty()) { if (remote_file->starts_with('/')) @@ -53,8 +48,7 @@ std::filesystem::path getPath(core::ProcessContext& context, const std::shared_p void FetchSmb::onTrigger(const std::shared_ptr& context, const std::shared_ptr& session) { gsl_Expects(context && session && smb_connection_controller_service_); - auto connection_error = smb_connection_controller_service_->validateConnection(); - if (connection_error) { + if (auto connection_error = smb_connection_controller_service_->validateConnection()) { logger_->log_error("Couldn't establish connection to the specified network location due to %s", connection_error.message()); context->yield(); return; @@ -66,10 +60,8 @@ void FetchSmb::onTrigger(const std::shared_ptr& context, c return; } - auto path = getPath(*context, flow_file); - try { - session->write(flow_file, utils::FileReaderCallback{smb_connection_controller_service_->getPath() / path}); + session->write(flow_file, utils::FileReaderCallback{smb_connection_controller_service_->getPath() / getTargetRelativePath(*context, flow_file)}); session->transfer(flow_file, Success); } catch (const utils::FileReaderCallbackIOError& io_error) { flow_file->addAttribute(ErrorCode.name, fmt::format("{}", io_error.error_code)); diff --git a/extensions/smb/ListSmb.cpp b/extensions/smb/ListSmb.cpp index 33487135b5..6c67e54b5d 100644 --- a/extensions/smb/ListSmb.cpp +++ b/extensions/smb/ListSmb.cpp @@ -33,13 +33,7 @@ void ListSmb::initialize() { void ListSmb::onSchedule(const std::shared_ptr &context, const std::shared_ptr &/*sessionFactory*/) { gsl_Expects(context); - - if (auto connection_controller_name = context->getProperty(ListSmb::ConnectionControllerService)) { - smb_connection_controller_service_ = std::dynamic_pointer_cast(context->getControllerService(*connection_controller_name)); - } - if (!smb_connection_controller_service_) { - throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); - } + smb_connection_controller_service_ = SmbConnectionControllerService::getFromProperty(*context, ListSmb::ConnectionControllerService); auto state_manager = context->getStateManager(); if (state_manager == nullptr) { @@ -113,8 +107,7 @@ std::shared_ptr ListSmb::createFlowFile(core::ProcessSession& se void ListSmb::onTrigger(const std::shared_ptr &context, const std::shared_ptr &session) { gsl_Expects(context && session && smb_connection_controller_service_); - auto connection_error = smb_connection_controller_service_->validateConnection(); - if (connection_error) { + if (auto connection_error = smb_connection_controller_service_->validateConnection()) { logger_->log_error("Couldn't establish connection to the specified network location due to %s", connection_error.message()); context->yield(); return; diff --git a/extensions/smb/PutSmb.cpp b/extensions/smb/PutSmb.cpp index 36864947f3..d8639ad30d 100644 --- a/extensions/smb/PutSmb.cpp +++ b/extensions/smb/PutSmb.cpp @@ -32,13 +32,7 @@ void PutSmb::initialize() { void PutSmb::onSchedule(core::ProcessContext* context, core::ProcessSessionFactory*) { gsl_Expects(context); - if (auto connection_controller_name = context->getProperty(PutSmb::ConnectionControllerService)) { - smb_connection_controller_service_ = std::dynamic_pointer_cast(context->getControllerService(*connection_controller_name)); - } - if (!smb_connection_controller_service_) { - throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); - } - + smb_connection_controller_service_ = SmbConnectionControllerService::getFromProperty(*context, PutSmb::ConnectionControllerService); create_missing_dirs_ = context->getProperty(PutSmb::CreateMissingDirectories).value_or(true); conflict_resolution_strategy_ = utils::parseEnumProperty(*context, ConflictResolution); } @@ -51,8 +45,7 @@ std::filesystem::path PutSmb::getFilePath(core::ProcessContext& context, const s void PutSmb::onTrigger(core::ProcessContext* context, core::ProcessSession* session) { gsl_Expects(context && session && smb_connection_controller_service_); - auto connection_error = smb_connection_controller_service_->validateConnection(); - if (connection_error) { + if (auto connection_error = smb_connection_controller_service_->validateConnection()) { logger_->log_error("Couldn't establish connection to the specified network location due to %s", connection_error.message()); context->yield(); return; diff --git a/extensions/smb/SmbConnectionControllerService.cpp b/extensions/smb/SmbConnectionControllerService.cpp index ad8ab5fa6e..5e700a4470 100644 --- a/extensions/smb/SmbConnectionControllerService.cpp +++ b/extensions/smb/SmbConnectionControllerService.cpp @@ -44,7 +44,7 @@ void SmbConnectionControllerService::onEnable() { auto username = getProperty(Username); if (password.has_value() != username.has_value()) - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Either a username and password must be provided, or neither of them should be provided."); + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Either both a username and a password, or neither of them should be provided."); if (username.has_value()) credentials_.emplace(Credentials{.username = *username, .password = *password}); @@ -64,6 +64,17 @@ void SmbConnectionControllerService::notifyStop() { logger_->log_error("Error while disconnecting from SMB: %s", disconnection_result.error().message()); } +std::shared_ptr SmbConnectionControllerService::getFromProperty(const core::ProcessContext& context, const core::PropertyReference& property) { + std::shared_ptr smb_connection_controller_service; + if (auto connection_controller_name = context.getProperty(property)) { + smb_connection_controller_service = std::dynamic_pointer_cast(context.getControllerService(*connection_controller_name)); + } + if (!smb_connection_controller_service) { + minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); + } + return smb_connection_controller_service; +} + nonstd::expected SmbConnectionControllerService::connect() { auto connection_result = WNetAddConnection2A(&net_resource_, credentials_ ? credentials_->password.c_str() : nullptr, @@ -86,8 +97,10 @@ nonstd::expected SmbConnectionControllerService::disconne bool SmbConnectionControllerService::isConnected() { std::error_code error_code; auto exists = std::filesystem::exists(server_path_, error_code); - if (error_code) + if (error_code) { + logger_->log_debug("std::filesystem::exists(%s) failed due to %s", server_path_, error_code.message()); return false; + } return exists; } diff --git a/extensions/smb/SmbConnectionControllerService.h b/extensions/smb/SmbConnectionControllerService.h index 4a8f046786..52155b3510 100644 --- a/extensions/smb/SmbConnectionControllerService.h +++ b/extensions/smb/SmbConnectionControllerService.h @@ -20,6 +20,7 @@ #include #include +#include "ProcessContext.h" #include "core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "core/controller/ControllerService.h" @@ -30,13 +31,6 @@ namespace org::apache::nifi::minifi::extensions::smb { -class SmbConnectionController { - public: - virtual nonstd::expected connect() = 0; - virtual nonstd::expected disconnect() = 0; - virtual bool isConnected() const = 0; -}; - class SmbConnectionControllerService : public core::controller::ControllerService { public: EXTENSIONAPI static constexpr const char* Description = "SMB Connection Controller Service"; @@ -87,6 +81,8 @@ class SmbConnectionControllerService : public core::controller::ControllerServic virtual std::error_code validateConnection(); virtual std::filesystem::path getPath() const { return server_path_; } + static std::shared_ptr getFromProperty(const core::ProcessContext& context, const core::PropertyReference& property); + private: nonstd::expected connect(); nonstd::expected disconnect(); diff --git a/extensions/standard-processors/processors/PutFile.h b/extensions/standard-processors/processors/PutFile.h index 3dc985158d..4e4514615f 100644 --- a/extensions/standard-processors/processors/PutFile.h +++ b/extensions/standard-processors/processors/PutFile.h @@ -1,4 +1,3 @@ - /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with From 1d5fb295717321aad98dfd1b774419db933958be Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 24 Aug 2023 14:00:59 +0200 Subject: [PATCH 05/25] review changes --- cmake/MiNiFiOptions.cmake | 2 +- extensions/smb/FetchSmb.cpp | 1 - extensions/smb/SmbConnectionControllerService.cpp | 1 - win_build_vs.bat | 6 +++++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index c020820450..22568ddb83 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -85,7 +85,7 @@ if (WIN32) add_minifi_option(MSI_REDISTRIBUTE_UCRT_NONASL "Redistribute Universal C Runtime DLLs with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) add_minifi_option(ENABLE_WEL "Enables the suite of Windows Event Log extensions." OFF) add_minifi_option(ENABLE_PDH "Enables PDH support." OFF) - add_minifi_option(ENABLE_SMB "Enables SMB support." ON) + add_minifi_option(ENABLE_SMB "Enables SMB support." OFF) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/extensions/smb/FetchSmb.cpp b/extensions/smb/FetchSmb.cpp index 5488929411..29330f9095 100644 --- a/extensions/smb/FetchSmb.cpp +++ b/extensions/smb/FetchSmb.cpp @@ -67,7 +67,6 @@ void FetchSmb::onTrigger(const std::shared_ptr& context, c flow_file->addAttribute(ErrorCode.name, fmt::format("{}", io_error.error_code)); flow_file->addAttribute(ErrorMessage.name, io_error.what()); session->transfer(flow_file, Failure); - return; } } diff --git a/extensions/smb/SmbConnectionControllerService.cpp b/extensions/smb/SmbConnectionControllerService.cpp index 5e700a4470..511032554e 100644 --- a/extensions/smb/SmbConnectionControllerService.cpp +++ b/extensions/smb/SmbConnectionControllerService.cpp @@ -37,7 +37,6 @@ void SmbConnectionControllerService::onEnable() { if (!getProperty(Share, share)) throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Missing share"); - server_path_ = "\\\\" + hostname + "\\" + share; auto password = getProperty(Password); diff --git a/win_build_vs.bat b/win_build_vs.bat index fada62d85f..2f935e300b 100755 --- a/win_build_vs.bat +++ b/win_build_vs.bat @@ -37,6 +37,9 @@ set enable_encrypt_config=ON set enable_lua_scripting=ON set enable_mqtt=ON set enable_opc=ON +set enable_pdh=OFF +set enable_splunk=ON +set enable_smb=ON set enable_openwsman=OFF set enable_ops=ON set enable_pcap=OFF @@ -70,6 +73,7 @@ for %%x in (%*) do ( if [%%~x] EQU [/SFTP] set enable_sftp=ON if [%%~x] EQU [/PDH] set enable_pdh=ON if [%%~x] EQU [/NO_SPLUNK] set enable_splunk=OFF + if [%%~x] EQU [/NO_SMB] set enable_smb=OFF if [%%~x] EQU [/NO_GCP] set enable_gcp=OFF if [%%~x] EQU [/NO_ELASTIC] set enable_elastic=OFF if [%%~x] EQU [/M] set installer_merge_modules=ON @@ -113,7 +117,7 @@ cmake -G %generator% %build_platform_cmd% -DINSTALLER_MERGE_MODULES=%installer_m -DCMAKE_BUILD_TYPE_INIT=%cmake_build_type% -DCMAKE_BUILD_TYPE=%cmake_build_type% -DWIN32=WIN32 -DENABLE_LIBRDKAFKA=%enable_kafka% -DENABLE_JNI=%enable_jni% -DOPENSSL_OFF=OFF ^ -DENABLE_COAP=%enable_coap% -DENABLE_AWS=%enable_aws% -DENABLE_PDH=%enable_pdh% -DENABLE_AZURE=%enable_azure% -DENABLE_SFTP=%enable_sftp% -DENABLE_SPLUNK=%enable_splunk% -DENABLE_GCP=%enable_gcp% ^ -DENABLE_NANOFI=%enable_nanofi% -DENABLE_OPENCV=%enable_opencv% -DENABLE_PROMETHEUS=%enable_prometheus% -DENABLE_ELASTICSEARCH=%enable_elastic% -DUSE_SHARED_LIBS=OFF -DENABLE_CONTROLLER=OFF ^ - -DENABLE_BUSTACHE=%enable_bustache% -DENABLE_ENCRYPT_CONFIG=%enable_encrypt_config% -DENABLE_LUA_SCRIPTING=%enable_lua_scripting% ^ + -DENABLE_BUSTACHE=%enable_bustache% -DENABLE_ENCRYPT_CONFIG=%enable_encrypt_config% -DENABLE_LUA_SCRIPTING=%enable_lua_scripting% -DENABLE_SMB=%enable_smb% ^ -DENABLE_MQTT=%enable_mqtt% -DENABLE_OPC=%enable_opc% -DENABLE_OPENWSMAN=%enable_openwsman% -DENABLE_OPS=%enable_ops% -DENABLE_PCAP=%enable_pcap% ^ -DENABLE_PYTHON_SCRIPTING=%enable_python_scripting% -DENABLE_SENSORS=%enable_sensors% -DENABLE_USB_CAMERA=%enable_usb_camera% ^ -DBUILD_ROCKSDB=ON -DFORCE_WINDOWS=ON -DUSE_SYSTEM_UUID=OFF -DDISABLE_LIBARCHIVE=OFF -DENABLE_WEL=ON -DFAIL_ON_WARNINGS=OFF -DSKIP_TESTS=%skiptests% ^ From 8978f2998b2343740ad2c1f19136c72a2605fcde Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 24 Aug 2023 14:25:14 +0200 Subject: [PATCH 06/25] test ListSmb::IgnoreHiddenFiles --- extensions/smb/tests/ListSmbTests.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/extensions/smb/tests/ListSmbTests.cpp b/extensions/smb/tests/ListSmbTests.cpp index 6efc5d40e9..308579b929 100644 --- a/extensions/smb/tests/ListSmbTests.cpp +++ b/extensions/smb/tests/ListSmbTests.cpp @@ -19,6 +19,7 @@ #include "TestBase.h" #include "Catch.h" #include "ListSmb.h" +#include "Utils.h" #include "utils/MockSmbConnectionControllerService.h" #include "SingleProcessorTestController.h" #include "range/v3/algorithm/count_if.hpp" @@ -65,6 +66,8 @@ TEST_CASE("ListSmb tests") { auto d_expected_attributes = mock_smb_connection_controller_service->addFile("subdir/d.foo", std::string(100, 'd'), 10min); auto e_expected_attributes = mock_smb_connection_controller_service->addFile("subdir2/e.foo", std::string(1, 'e'), 0s); auto f_expected_attributes = mock_smb_connection_controller_service->addFile("third/f.bar", std::string(50_KiB, 'f'), 30min); + auto g_expected_attributes = mock_smb_connection_controller_service->addFile("g.foo", std::string(50_KiB, 'f'), 30min); + REQUIRE_FALSE(minifi::test::utils::hide_file(mock_smb_connection_controller_service->getPath() / "g.foo")); REQUIRE((a_expected_attributes && b_expected_attributes && c_expected_attributes && d_expected_attributes && e_expected_attributes && f_expected_attributes)); @@ -113,6 +116,18 @@ TEST_CASE("ListSmb tests") { CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *b_expected_attributes)); } + SECTION("Dont ignore hidden files") { + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::IgnoreHiddenFiles, "false")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::FileFilter, ".*\\.foo")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "false")); + const auto trigger_results = controller.trigger(); + CHECK(trigger_results.at(ListSmb::Success).size() == 3); + CHECK_FALSE(list_smb->isYield()); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *a_expected_attributes)); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *b_expected_attributes)); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *g_expected_attributes)); + } + const auto second_trigger = controller.trigger(); CHECK(second_trigger.at(ListSmb::Success).empty()); CHECK(list_smb->isYield()); From 6dc138dbadc004d02781df331280110ecf29ef7b Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 24 Aug 2023 15:26:06 +0200 Subject: [PATCH 07/25] add /bigobj to WIN32 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f602449c3e..486555b387 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -108,7 +108,7 @@ endif() # Enable usage of the VERSION specifier if (WIN32) add_compile_definitions(WIN32_LEAN_AND_MEAN _CRT_SECURE_NO_WARNINGS NOMINMAX) - add_compile_options(/W3 /utf-8) + add_compile_options(/W3 /utf-8 /bigobj) endif() if (NOT PORTABLE) From 97bf67a05ff432fb7473ec2cb88df6475a26edab Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 24 Aug 2023 15:33:09 +0200 Subject: [PATCH 08/25] add SMB to ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 265a5d3390..e9de954dfe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,7 +108,7 @@ jobs: - name: build run: | for /f "usebackq delims=" %%i in (`vswhere.exe -latest -property installationPath`) do if exist "%%i\Common7\Tools\vsdevcmd.bat" call "%%i\Common7\Tools\vsdevcmd.bat" -arch=x64 -host_arch=x64 - win_build_vs.bat ..\b /64 /CI /S /A /PDH /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA + win_build_vs.bat ..\b /64 /CI /S /A /PDH /SMB /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA sccache --show-stats shell: cmd - name: sccache cache save From 4c503a069e2fb967e9ae6f281dff53e0fd42a58d Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Thu, 24 Aug 2023 15:36:59 +0200 Subject: [PATCH 09/25] removed unnecessary newline --- extensions/smb/PutSmb.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/extensions/smb/PutSmb.cpp b/extensions/smb/PutSmb.cpp index d8639ad30d..d191906ad8 100644 --- a/extensions/smb/PutSmb.cpp +++ b/extensions/smb/PutSmb.cpp @@ -29,7 +29,6 @@ void PutSmb::initialize() { setSupportedRelationships(Relationships); } - void PutSmb::onSchedule(core::ProcessContext* context, core::ProcessSessionFactory*) { gsl_Expects(context); smb_connection_controller_service_ = SmbConnectionControllerService::getFromProperty(*context, PutSmb::ConnectionControllerService); From a4c378723f8e4f07d37f1e40e55c38beb34f890b Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 25 Aug 2023 13:17:13 +0200 Subject: [PATCH 10/25] linter fix --- libminifi/test/unit/TimeUtilTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libminifi/test/unit/TimeUtilTests.cpp b/libminifi/test/unit/TimeUtilTests.cpp index 9b332a3383..9b9eb727fb 100644 --- a/libminifi/test/unit/TimeUtilTests.cpp +++ b/libminifi/test/unit/TimeUtilTests.cpp @@ -129,7 +129,7 @@ TEST_CASE("Test windows file_clock duration period and epoch") { TEST_CASE("Test windows FILETIME epoch") { SYSTEMTIME system_time; - FILETIME file_time{.dwLowDateTime=0, .dwHighDateTime=0}; + FILETIME file_time{.dwLowDateTime = 0, .dwHighDateTime = 0}; FileTimeToSystemTime(&file_time, &system_time); CHECK(system_time.wYear == 1601); CHECK(system_time.wMonth == 1); From 64a93252a92ff0cb2ea9abd3bae2a3e26c7cea19 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 5 Sep 2023 09:04:59 +0200 Subject: [PATCH 11/25] review changes and rebase fixes --- .../bustache/tests/ApplyTemplateTests.cpp | 9 ++---- extensions/smb/FetchSmb.h | 2 +- extensions/smb/ListSmb.cpp | 9 +++--- extensions/smb/ListSmb.h | 14 +++++----- extensions/smb/PutSmb.cpp | 3 ++ extensions/smb/PutSmb.h | 2 +- .../smb/SmbConnectionControllerService.cpp | 20 +++++++------ .../smb/SmbConnectionControllerService.h | 4 ++- extensions/smb/tests/FetchSmbTests.cpp | 2 +- extensions/smb/tests/ListSmbTests.cpp | 7 +++-- .../SmbConnectionControllerServiceTests.cpp | 1 - .../MockSmbConnectionControllerService.h | 5 ++-- extensions/smb/tests/utils/TempSmbShare.h | 28 +++++++++---------- .../processors/PutFile.cpp | 2 +- .../standard-processors/processors/PutFile.h | 2 +- libminifi/include/utils/file/FileUtils.h | 2 +- libminifi/src/utils/OsUtils.cpp | 15 +++++----- 17 files changed, 65 insertions(+), 62 deletions(-) diff --git a/extensions/bustache/tests/ApplyTemplateTests.cpp b/extensions/bustache/tests/ApplyTemplateTests.cpp index 8f2f73001d..a0144f9452 100644 --- a/extensions/bustache/tests/ApplyTemplateTests.cpp +++ b/extensions/bustache/tests/ApplyTemplateTests.cpp @@ -17,11 +17,8 @@ */ #include -#include #include -#include #include -#include #include #include "TestBase.h" @@ -30,9 +27,7 @@ #include "core/FlowFile.h" #include "core/Processor.h" -#include "core/ProcessContext.h" #include "core/ProcessSession.h" -#include "core/ProcessorNode.h" #include "repository/VolatileContentRepository.h" #include "unit/ProvenanceTestHelper.h" @@ -81,7 +76,7 @@ TEST_CASE("Test usage of ApplyTemplate", "[ApplyTemplateTest]") { REQUIRE_FALSE(put_file_destination_dir.empty()); std::shared_ptr getfile = plan->addProcessor("GetFile", "getFile"); - plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::Directory, get_file_source_dir); + plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::Directory, get_file_source_dir.string()); plan->setProperty(getfile, org::apache::nifi::minifi::processors::GetFile::KeepSourceFile, "true"); std::shared_ptr extract_text = plan->addProcessor("ExtractText", "testExtractText", core::Relationship("success", "description"), true); @@ -90,7 +85,7 @@ TEST_CASE("Test usage of ApplyTemplate", "[ApplyTemplateTest]") { std::shared_ptr apply_template = plan->addProcessor("ApplyTemplate", "testApplyTemplate", core::Relationship("success", "description"), true); std::shared_ptr put_file = plan->addProcessor("PutFile", "put_file", core::Relationship("success", "description"), true); - plan->setProperty(put_file, org::apache::nifi::minifi::processors::PutFile::Directory, put_file_destination_dir); + plan->setProperty(put_file, org::apache::nifi::minifi::processors::PutFile::Directory, put_file_destination_dir.string()); plan->setProperty(put_file, org::apache::nifi::minifi::processors::PutFile::ConflictResolution, magic_enum::enum_name(minifi::processors::PutFile::FileExistsResolutionStrategy::replace)); // Write attribute value to file for GetFile->ExtractText diff --git a/extensions/smb/FetchSmb.h b/extensions/smb/FetchSmb.h index 462478be28..52d415545e 100644 --- a/extensions/smb/FetchSmb.h +++ b/extensions/smb/FetchSmb.h @@ -51,7 +51,7 @@ class FetchSmb : public core::Processor { .withAllowedTypes() .build(); EXTENSIONAPI static constexpr auto RemoteFile = core::PropertyDefinitionBuilder<>::createProperty("Input Directory") - .withDescription("The full path of the file to be retrieved from the remote server. Expression language supported. If left empty the path and filename attributes will be used.") + .withDescription("The full path of the file to be retrieved from the remote server. If left empty, the path and filename attributes of the incoming flow file will be used.") .isRequired(false) .supportsExpressionLanguage(true) .build(); diff --git a/extensions/smb/ListSmb.cpp b/extensions/smb/ListSmb.cpp index 6c67e54b5d..1d2dc08c53 100644 --- a/extensions/smb/ListSmb.cpp +++ b/extensions/smb/ListSmb.cpp @@ -62,13 +62,12 @@ void ListSmb::onSchedule(const std::shared_ptr &context, c file_filter_.maximum_file_age = maximum_file_age->getMilliseconds(); } - uint64_t int_value = 0; - if (context->getProperty(MinimumFileSize, value) && !value.empty() && core::Property::StringToInt(value, int_value)) { - file_filter_.minimum_file_size = int_value; + if (const auto minimum_file_size = context->getProperty(MinimumFileSize)) { + file_filter_.minimum_file_size = minimum_file_size->getValue(); } - if (context->getProperty(MaximumFileSize, value) && !value.empty() && core::Property::StringToInt(value, int_value)) { - file_filter_.maximum_file_size = int_value; + if (const auto maximum_file_size = context->getProperty(MaximumFileSize)) { + file_filter_.maximum_file_size = maximum_file_size->getValue(); } context->getProperty(IgnoreHiddenFiles, file_filter_.ignore_hidden_files); diff --git a/extensions/smb/ListSmb.h b/extensions/smb/ListSmb.h index 489329344a..732729269d 100644 --- a/extensions/smb/ListSmb.h +++ b/extensions/smb/ListSmb.h @@ -52,7 +52,7 @@ class ListSmb : public core::Processor { .withAllowedTypes() .build(); EXTENSIONAPI static constexpr auto InputDirectory = core::PropertyDefinitionBuilder<>::createProperty("Input Directory") - .withDescription("The input directory from which files to pull files") + .withDescription("The input directory to list the contents of") .isRequired(false) .build(); EXTENSIONAPI static constexpr auto RecurseSubdirectories = core::PropertyDefinitionBuilder<>::createProperty("Recurse Subdirectories") @@ -71,19 +71,19 @@ class ListSmb : public core::Processor { .withDescription("The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored") .isRequired(true) .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) - .withDefaultValue("0 sec") + .withDefaultValue("5 sec") .build(); EXTENSIONAPI static constexpr auto MaximumFileAge = core::PropertyDefinitionBuilder<>::createProperty("Maximum File Age") .withDescription("The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored") + .withPropertyType(core::StandardPropertyTypes::TIME_PERIOD_TYPE) .build(); EXTENSIONAPI static constexpr auto MinimumFileSize = core::PropertyDefinitionBuilder<>::createProperty("Minimum File Size") .withDescription("The minimum size that a file must be in order to be pulled") - .isRequired(true) .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) - .withDefaultValue("0 B") .build(); EXTENSIONAPI static constexpr auto MaximumFileSize = core::PropertyDefinitionBuilder<>::createProperty("Maximum File Size") .withDescription("The maximum size that a file can be in order to be pulled") + .withPropertyType(core::StandardPropertyTypes::DATA_SIZE_TYPE) .build(); EXTENSIONAPI static constexpr auto IgnoreHiddenFiles = core::PropertyDefinitionBuilder<>::createProperty("Ignore Hidden Files") .withDescription("Indicates whether or not hidden files should be ignored") @@ -111,8 +111,8 @@ class ListSmb : public core::Processor { EXTENSIONAPI static constexpr auto Filename = core::OutputAttributeDefinition<>{"filename", { Success }, "The name of the file that was read from filesystem."}; EXTENSIONAPI static constexpr auto Path = core::OutputAttributeDefinition<>{"path", { Success }, "The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. " - "For example, for a given remote locationsmb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file " - "then the path attribute will be set to \"DIRECTORY/sub/folder\"."}; + "For example, for a given remote location smb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file " + "then the path attribute will be set to \"sub/folder\"."}; EXTENSIONAPI static constexpr auto ServiceLocation = core::OutputAttributeDefinition<>{"serviceLocation", { Success }, "The SMB URL of the share."}; EXTENSIONAPI static constexpr auto LastModifiedTime = core::OutputAttributeDefinition<>{"lastModifiedTime", { Success }, @@ -123,7 +123,7 @@ class ListSmb : public core::Processor { "The timestamp of when the file was accessed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'."}; - EXTENSIONAPI static constexpr auto Size = core::OutputAttributeDefinition<>{"size", { Success }, "The size of the file in bytes.."}; + EXTENSIONAPI static constexpr auto Size = core::OutputAttributeDefinition<>{"size", { Success }, "The size of the file in bytes."}; EXTENSIONAPI static constexpr auto OutputAttributes = std::array {Filename, Path, ServiceLocation, LastModifiedTime, CreationTime, LastAccessTime, Size }; diff --git a/extensions/smb/PutSmb.cpp b/extensions/smb/PutSmb.cpp index d191906ad8..b2ad64986f 100644 --- a/extensions/smb/PutSmb.cpp +++ b/extensions/smb/PutSmb.cpp @@ -21,6 +21,7 @@ #include "utils/ProcessorConfigUtils.h" #include "utils/OsUtils.h" #include "utils/file/FileWriterCallback.h" +#include "core/Resource.h" namespace org::apache::nifi::minifi::extensions::smb { @@ -88,4 +89,6 @@ void PutSmb::onTrigger(core::ProcessContext* context, core::ProcessSession* sess session->transfer(flow_file, success ? Success : Failure); } +REGISTER_RESOURCE(PutSmb, Processor); + } // namespace org::apache::nifi::minifi::extensions::smb diff --git a/extensions/smb/PutSmb.h b/extensions/smb/PutSmb.h index f1fbe44144..a104ddc8e0 100644 --- a/extensions/smb/PutSmb.h +++ b/extensions/smb/PutSmb.h @@ -55,7 +55,7 @@ class PutSmb : public core::Processor { .supportsExpressionLanguage(true) .withDefaultValue(".") .build(); - EXTENSIONAPI static constexpr auto ConflictResolution = core::PropertyDefinitionBuilder<3>::createProperty("Conflict Resolution Strategy") + EXTENSIONAPI static constexpr auto ConflictResolution = core::PropertyDefinitionBuilder()>::createProperty("Conflict Resolution Strategy") .withDescription("Indicates what should happen when a file with the same name already exists in the output directory") .withDefaultValue(magic_enum::enum_name(FileExistsResolutionStrategy::fail)) .withAllowedValues(magic_enum::enum_names()) diff --git a/extensions/smb/SmbConnectionControllerService.cpp b/extensions/smb/SmbConnectionControllerService.cpp index 511032554e..f9a0857cec 100644 --- a/extensions/smb/SmbConnectionControllerService.cpp +++ b/extensions/smb/SmbConnectionControllerService.cpp @@ -50,11 +50,12 @@ void SmbConnectionControllerService::onEnable() { else credentials_.reset(); - ZeroMemory(&net_resource_, sizeof(net_resource_)); - net_resource_.dwType = RESOURCETYPE_DISK; - net_resource_.lpLocalName = nullptr; - net_resource_.lpRemoteName = server_path_.data(); - net_resource_.lpProvider = nullptr; + net_resource_ = { + .dwType = RESOURCETYPE_DISK, + .lpLocalName = nullptr, + .lpRemoteName = server_path_.data(), + .lpProvider = nullptr, + }; } void SmbConnectionControllerService::notifyStop() { @@ -63,15 +64,15 @@ void SmbConnectionControllerService::notifyStop() { logger_->log_error("Error while disconnecting from SMB: %s", disconnection_result.error().message()); } -std::shared_ptr SmbConnectionControllerService::getFromProperty(const core::ProcessContext& context, const core::PropertyReference& property) { +gsl::not_null> SmbConnectionControllerService::getFromProperty(const core::ProcessContext& context, const core::PropertyReference& property) { std::shared_ptr smb_connection_controller_service; if (auto connection_controller_name = context.getProperty(property)) { smb_connection_controller_service = std::dynamic_pointer_cast(context.getControllerService(*connection_controller_name)); } if (!smb_connection_controller_service) { - minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); + throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Missing SMB Connection Controller Service"); } - return smb_connection_controller_service; + return gsl::make_not_null(smb_connection_controller_service); } nonstd::expected SmbConnectionControllerService::connect() { @@ -104,8 +105,9 @@ bool SmbConnectionControllerService::isConnected() { } std::error_code SmbConnectionControllerService::validateConnection() { - if (isConnected()) + if (isConnected()) { return std::error_code(); + } auto connection_result = connect(); if (!connection_result) { return connection_result.error(); diff --git a/extensions/smb/SmbConnectionControllerService.h b/extensions/smb/SmbConnectionControllerService.h index 52155b3510..405d869e40 100644 --- a/extensions/smb/SmbConnectionControllerService.h +++ b/extensions/smb/SmbConnectionControllerService.h @@ -50,10 +50,12 @@ class SmbConnectionControllerService : public core::controller::ControllerServic EXTENSIONAPI static constexpr auto Username = core::PropertyDefinitionBuilder<>::createProperty("Username") .withDescription("The username used for authentication. If no username is set then anonymous authentication is attempted.") .isRequired(false) + .withDependentProperties({"Password"}) .build(); EXTENSIONAPI static constexpr auto Password = core::PropertyDefinitionBuilder<>::createProperty("Password") .withDescription("The password used for authentication. Required if Username is set.") .isRequired(false) + .withDependentProperties({"Username"}) .build(); static constexpr auto Properties = std::array{ @@ -81,7 +83,7 @@ class SmbConnectionControllerService : public core::controller::ControllerServic virtual std::error_code validateConnection(); virtual std::filesystem::path getPath() const { return server_path_; } - static std::shared_ptr getFromProperty(const core::ProcessContext& context, const core::PropertyReference& property); + static gsl::not_null> getFromProperty(const core::ProcessContext& context, const core::PropertyReference& property); private: nonstd::expected connect(); diff --git a/extensions/smb/tests/FetchSmbTests.cpp b/extensions/smb/tests/FetchSmbTests.cpp index 27493766cc..9e00c5f43c 100644 --- a/extensions/smb/tests/FetchSmbTests.cpp +++ b/extensions/smb/tests/FetchSmbTests.cpp @@ -34,7 +34,7 @@ TEST_CASE("FetchSmb invalid network path") { minifi::test::SingleProcessorTestController controller{fetch_smb}; auto smb_connection_node = controller.plan->addController("MockSmbConnectionControllerService", "smb_connection_controller_service"); REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Hostname, utils::OsUtils::getHostName().value_or("localhost"))); - REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Share, "some_share_that_does_not_exists")); + REQUIRE(controller.plan->setProperty(smb_connection_node, SmbConnectionControllerService::Share, "some_share_that_does_not_exist")); REQUIRE(controller.plan->setProperty(fetch_smb, FetchSmb::ConnectionControllerService, "smb_connection_controller_service")); const auto trigger_results = controller.trigger("", {{std::string(core::SpecialFlowAttribute::FILENAME), "a.foo"}, {std::string(core::SpecialFlowAttribute::PATH), ""}}); CHECK(trigger_results.at(FetchSmb::Success).empty()); diff --git a/extensions/smb/tests/ListSmbTests.cpp b/extensions/smb/tests/ListSmbTests.cpp index 308579b929..babe1c424b 100644 --- a/extensions/smb/tests/ListSmbTests.cpp +++ b/extensions/smb/tests/ListSmbTests.cpp @@ -64,12 +64,13 @@ TEST_CASE("ListSmb tests") { auto b_expected_attributes = mock_smb_connection_controller_service->addFile("b.foo", std::string(13_KiB, 'b'), 1h); auto c_expected_attributes = mock_smb_connection_controller_service->addFile("c.bar", std::string(1_KiB, 'c'), 2h); auto d_expected_attributes = mock_smb_connection_controller_service->addFile("subdir/d.foo", std::string(100, 'd'), 10min); - auto e_expected_attributes = mock_smb_connection_controller_service->addFile("subdir2/e.foo", std::string(1, 'e'), 0s); + auto e_expected_attributes = mock_smb_connection_controller_service->addFile("subdir2/e.foo", std::string(1, 'e'), 10s); auto f_expected_attributes = mock_smb_connection_controller_service->addFile("third/f.bar", std::string(50_KiB, 'f'), 30min); auto g_expected_attributes = mock_smb_connection_controller_service->addFile("g.foo", std::string(50_KiB, 'f'), 30min); - REQUIRE_FALSE(minifi::test::utils::hide_file(mock_smb_connection_controller_service->getPath() / "g.foo")); + auto hide_file_error = minifi::test::utils::hide_file(mock_smb_connection_controller_service->getPath() / "g.foo"); + REQUIRE_FALSE(hide_file_error); - REQUIRE((a_expected_attributes && b_expected_attributes && c_expected_attributes && d_expected_attributes && e_expected_attributes && f_expected_attributes)); + REQUIRE((a_expected_attributes && b_expected_attributes && c_expected_attributes && d_expected_attributes && e_expected_attributes && f_expected_attributes && g_expected_attributes)); REQUIRE(controller.plan->setProperty(list_smb, ListSmb::ConnectionControllerService, "smb_connection_controller_service")); diff --git a/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp b/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp index 70b7e8d2e1..d0aadd451c 100644 --- a/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp +++ b/extensions/smb/tests/SmbConnectionControllerServiceTests.cpp @@ -44,7 +44,6 @@ TEST_CASE_METHOD(SmbConnectionControllerServiceFixture, "SmbConnectionController auto temp_smb_share = TempSmbShare::create(share_local_name, temp_directory.wstring()); if (!temp_smb_share && temp_smb_share.error() == std::error_code(5, std::system_category())) { SKIP("SmbConnectionControllerService tests needs administrator privileges"); - return; } diff --git a/extensions/smb/tests/utils/MockSmbConnectionControllerService.h b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h index a3ad615e53..32c9eeb7be 100644 --- a/extensions/smb/tests/utils/MockSmbConnectionControllerService.h +++ b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h @@ -21,6 +21,7 @@ #include "../../SmbConnectionControllerService.h" #include "core/controller/ControllerService.h" #include "utils/OsUtils.h" +#include "utils/file/FileUtils.h" #include "ListSmb.h" #include "Catch.h" @@ -84,10 +85,10 @@ class MockSmbConnectionControllerService : public SmbConnectionControllerService return nonstd::make_unexpected(std::make_error_code(std::errc::bad_file_descriptor)); out_file << content; } - auto last_write_time_error = utils::file::set_last_write_time(full_path, std::chrono::file_clock::now() - age); + auto current_time = std::chrono::system_clock::now(); + auto last_write_time_error = utils::file::set_last_write_time(full_path, minifi::utils::file::from_sys(current_time) - age); if (!last_write_time_error) return nonstd::make_unexpected(std::make_error_code(std::errc::bad_file_descriptor)); - auto current_time = std::chrono::system_clock::now(); auto path = relative_path.parent_path().empty() ? (std::filesystem::path(".") / "").string() : (relative_path.parent_path() / "").string(); return ListSmbExpectedAttributes{ .expected_filename = relative_path.filename().string(), diff --git a/extensions/smb/tests/utils/TempSmbShare.h b/extensions/smb/tests/utils/TempSmbShare.h index 3c6aa98c79..5024a0d59c 100644 --- a/extensions/smb/tests/utils/TempSmbShare.h +++ b/extensions/smb/tests/utils/TempSmbShare.h @@ -41,23 +41,23 @@ class TempSmbShare { } static nonstd::expected create(std::wstring net_name, std::wstring path) { - SHARE_INFO_502 shareInfo{}; std::wstring remark = L"SMB share to test SMB capabilities of minifi"; - shareInfo.shi502_netname = net_name.data(); - shareInfo.shi502_type = STYPE_DISKTREE; - shareInfo.shi502_remark = remark.data(); - shareInfo.shi502_permissions = ACCESS_ALL; - shareInfo.shi502_max_uses = -1; - shareInfo.shi502_current_uses = 0; - shareInfo.shi502_path = path.data(); - shareInfo.shi502_passwd = nullptr; - shareInfo.shi502_reserved = 0; - shareInfo.shi502_security_descriptor = nullptr; + SHARE_INFO_502 share_info = { + .shi502_netname = net_name.data(), + .shi502_type = STYPE_DISKTREE, + .shi502_remark = remark.data(), + .shi502_permissions = ACCESS_ALL, + .shi502_max_uses = static_cast(-1), + .shi502_current_uses = 0, + .shi502_path = path.data(), + .shi502_passwd = nullptr, + .shi502_reserved = 0, + .shi502_security_descriptor = nullptr, + }; - DWORD netshare_result = NetShareAdd(nullptr, 502, reinterpret_cast(&shareInfo), nullptr); + DWORD netshare_result = NetShareAdd(nullptr, 502, reinterpret_cast(&share_info), nullptr); if (netshare_result == NERR_Success) { - auto asd = TempSmbShare(std::move(net_name), std::move(path)); - return asd; + return TempSmbShare(std::move(net_name), std::move(path)); } return nonstd::make_unexpected(utils::OsUtils::windowsErrorToErrorCode(netshare_result)); } diff --git a/extensions/standard-processors/processors/PutFile.cpp b/extensions/standard-processors/processors/PutFile.cpp index 33046feaf8..9db95e6213 100644 --- a/extensions/standard-processors/processors/PutFile.cpp +++ b/extensions/standard-processors/processors/PutFile.cpp @@ -68,7 +68,7 @@ std::optional PutFile::getDestinationPath(core::ProcessCo } bool PutFile::directoryIsFull(const std::filesystem::path& directory) const { - return max_dest_files_ && utils::file::is_directory(directory) && utils::file::countNumberOfFiles(directory) >= *max_dest_files_; + return max_dest_files_ && utils::file::countNumberOfFiles(directory) >= *max_dest_files_; } void PutFile::onTrigger(core::ProcessContext *context, core::ProcessSession *session) { diff --git a/extensions/standard-processors/processors/PutFile.h b/extensions/standard-processors/processors/PutFile.h index 4e4514615f..bb88f82a4a 100644 --- a/extensions/standard-processors/processors/PutFile.h +++ b/extensions/standard-processors/processors/PutFile.h @@ -66,7 +66,7 @@ class PutFile : public core::Processor { .supportsExpressionLanguage(true) .withDefaultValue(".") .build(); - EXTENSIONAPI static constexpr auto ConflictResolution = core::PropertyDefinitionBuilder<3>::createProperty("Conflict Resolution Strategy") + EXTENSIONAPI static constexpr auto ConflictResolution = core::PropertyDefinitionBuilder()>::createProperty("Conflict Resolution Strategy") .withDescription("Indicates what should happen when a file with the same name already exists in the output directory") .withDefaultValue(magic_enum::enum_name(FileExistsResolutionStrategy::fail)) .withAllowedValues(magic_enum::enum_names()) diff --git a/libminifi/include/utils/file/FileUtils.h b/libminifi/include/utils/file/FileUtils.h index 207ef08778..1295d9ee9d 100644 --- a/libminifi/include/utils/file/FileUtils.h +++ b/libminifi/include/utils/file/FileUtils.h @@ -26,7 +26,6 @@ #include #include #include -#include "utils/expected.h" #ifndef WIN32 #include @@ -74,6 +73,7 @@ #include "core/logging/LoggerFactory.h" #include "utils/StringUtils.h" +#include "utils/expected.h" #include "utils/file/PathUtils.h" #include "utils/gsl.h" diff --git a/libminifi/src/utils/OsUtils.cpp b/libminifi/src/utils/OsUtils.cpp index 670e41aa35..f865000083 100644 --- a/libminifi/src/utils/OsUtils.cpp +++ b/libminifi/src/utils/OsUtils.cpp @@ -21,6 +21,7 @@ #include #include +#include "fmt/format.h" #include "utils/gsl.h" #include "Exception.h" @@ -343,13 +344,13 @@ std::wstring OsUtils::stringToWideString(const std::string& string) { if (string.empty()) return {}; - const auto size_needed = MultiByteToWideChar(CP_UTF8, 0, &string.at(0), static_cast(string.size()), nullptr, 0); + const auto size_needed = MultiByteToWideChar(CP_UTF8, 0, &string.at(0), gsl::narrow(string.size()), nullptr, 0); if (size_needed <= 0) { - throw std::runtime_error("MultiByteToWideChar() failed: " + std::to_string(size_needed)); + throw std::runtime_error(fmt::format("MultiByteToWideChar() returned: {}, due to {}", std::to_string(size_needed), utils::OsUtils::windowsErrorToErrorCode(GetLastError()).message())); } - std::wstring result(size_needed, 0); - MultiByteToWideChar(CP_UTF8, 0, &string.at(0), static_cast(string.size()), &result.at(0), size_needed); + std::wstring result(size_needed, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, &string.at(0), gsl::narrow(string.size()), &result.at(0), size_needed); return result; } @@ -357,13 +358,13 @@ std::string OsUtils::wideStringToString(const std::wstring& wide_string) { if (wide_string.empty()) return {}; - const auto size_needed = WideCharToMultiByte(CP_UTF8, 0, &wide_string.at(0), static_cast(wide_string.size()), nullptr, 0, nullptr, nullptr); + const auto size_needed = WideCharToMultiByte(CP_UTF8, 0, &wide_string.at(0), gsl::narrow(wide_string.size()), nullptr, 0, nullptr, nullptr); if (size_needed <= 0) { - throw std::runtime_error("WideCharToMultiByte() failed: " + std::to_string(size_needed)); + throw std::runtime_error(fmt::format("WideCharToMultiByte() returned: {}, due to {}", std::to_string(size_needed), utils::OsUtils::windowsErrorToErrorCode(GetLastError()).message())); } std::string result(size_needed, 0); - WideCharToMultiByte(CP_UTF8, 0, &wide_string.at(0), static_cast(wide_string.size()), &result.at(0), size_needed, nullptr, nullptr); + WideCharToMultiByte(CP_UTF8, 0, &wide_string.at(0), gsl::narrow(wide_string.size()), &result.at(0), size_needed, nullptr, nullptr); return result; } #endif From 7a457d8c25bbc4db12dc455e618b418d3e952f42 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 8 Sep 2023 18:00:30 +0200 Subject: [PATCH 12/25] add missing documentation --- PROCESSORS.md | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 1 + 2 files changed, 101 insertions(+) diff --git a/PROCESSORS.md b/PROCESSORS.md index a865342dd0..6da9203df7 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -46,6 +46,7 @@ limitations under the License. - [FetchOPCProcessor](#fetchopcprocessor) - [FetchS3Object](#fetchs3object) - [FetchSFTP](#fetchsftp) +- [FetchSmb](#fetchsmb) - [FocusArchiveEntry](#focusarchiveentry) - [GenerateFlowFile](#generateflowfile) - [GetEnvironmentalSensors](#getenvironmentalsensors) @@ -66,6 +67,7 @@ limitations under the License. - [ListGCSBucket](#listgcsbucket) - [ListS3](#lists3) - [ListSFTP](#listsftp) +- [ListSmb](#listsmb) - [LogAttribute](#logattribute) - [ManipulateArchive](#manipulatearchive) - [MergeContent](#mergecontent) @@ -82,6 +84,7 @@ limitations under the License. - [PutOPCProcessor](#putopcprocessor) - [PutS3Object](#puts3object) - [PutSFTP](#putsftp) +- [PutSmb](#putsmb) - [PutSplunkHTTP](#putsplunkhttp) - [PutSQL](#putsql) - [PutTCP](#puttcp) @@ -1027,6 +1030,36 @@ In the list below, the names of required properties appear in bold. Any other pr | permission.denied | Any FlowFile that could not be fetched from the remote server due to insufficient permissions will be transferred to this Relationship. | +## FetchSmb + +### Description + +Fetches files from a SMB Share. Designed to be used in tandem with ListSmb. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | +| Input Directory | | | The full path of the file to be retrieved from the remote server. If left empty, the path and filename attributes of the incoming flow file will be used.
**Supports Expression Language: true** | + +### Relationships + +| Name | Description | +|---------|--------------------------------------------------------------------| +| success | A flowfile will be routed here for each successfully fetched file. | +| failure | A flowfile will be routed here when failed to fetch its content. | + +### Output Attributes + +| Attribute | Relationship | Description | +|---------------|--------------|-------------------------------------------------------------------| +| error.code | failure | The error code returned by SMB when the fetch of a file fails. | +| error.message | failure | The error message returned by SMB when the fetch of a file fails. | + + ## FocusArchiveEntry ### Description @@ -1689,6 +1722,48 @@ In the list below, the names of required properties appear in bold. Any other pr | success | All FlowFiles that are received are routed to success | +## ListSmb + +### Description + +Retrieves a listing of files from an SMB share. For each file that is listed, creates a FlowFile that represents the file so that it can be fetched in conjunction with FetchSmb. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------------------|---------------|------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | +| Input Directory | | | The input directory to list the contents of | +| **Recurse Subdirectories** | true | | Indicates whether to list files from subdirectories of the directory | +| File Filter | | | Only files whose names match the given regular expression will be picked up | +| Path Filter | | | When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned | +| **Minimum File Age** | 5 sec | | The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored | +| Maximum File Age | | | The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored | +| Minimum File Size | | | The minimum size that a file must be in order to be pulled | +| Maximum File Size | | | The maximum size that a file can be in order to be pulled | +| **Ignore Hidden Files** | true | | Indicates whether or not hidden files should be ignored | + +### Relationships + +| Name | Description | +|---------|-------------------------------------------------------| +| success | All FlowFiles that are received are routed to success | + +### Output Attributes + +| Attribute | Relationship | Description | +|------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| filename | success | The name of the file that was read from filesystem. | +| path | success | The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. For example, for a given remote location smb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file then the path attribute will be set to "sub/folder". | +| serviceLocation | success | The SMB URL of the share. | +| lastModifiedTime | success | The timestamp of when the file's content changed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| creationTime | success | The timestamp of when the file was created in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| lastAccessTime | success | The timestamp of when the file was accessed in the filesystem as 'yyyy-MM-dd'T'HH:mm:ss'. | +| size | success | The size of the file in bytes. | + + ## LogAttribute ### Description @@ -2360,6 +2435,31 @@ In the list below, the names of required properties appear in bold. Any other pr | failure | FlowFiles that failed to send to the remote system; failure is usually looped back to this processor | +## PutSmb + +### Description + +Writes the contents of a FlowFile to an smb network location + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|---------------------------------------|---------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------| +| **SMB Connection Controller Service** | | | Specifies the SMB connection controller service to use for connecting to the SMB server. | +| Directory | . | | The output directory to which to put files
**Supports Expression Language: true** | +| Conflict Resolution Strategy | fail | fail
replace
ignore | Indicates what should happen when a file with the same name already exists in the output directory | +| **Create Missing Directories** | true | | If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure. | + +### Relationships + +| Name | Description | +|---------|-------------------------------------------------------------------------| +| success | All files are routed to success | +| failure | Failed files (conflict, write failure, etc.) are transferred to failure | + + ## PutSplunkHTTP ### Description diff --git a/README.md b/README.md index b46bb34521..024d5556bc 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ Through JNI extensions you can run NiFi processors using NARs. The JNI extension | ProcFs (Linux) | [ProcFsMonitor](PROCESSORS.md#procfsmonitor) | -DENABLE_PROCFS=ON | | Python Scripting | [ExecuteScript](PROCESSORS.md#executescript)
[ExecutePythonProcessor](PROCESSORS.md#executepythonprocessor)
**Custom Python Processors** | -DENABLE_PYTHON_SCRIPTING=ON | | Sensors | GetEnvironmentalSensors
GetMovementSensors | -DENABLE_SENSORS=ON | +| SMB (Windows) | [FetchSmb](PROCESSORS.md#fetchsmb)
[ListSmb](PROCESSORS.md#listsmb)
[PutSmb](PROCESSORS.md#putsmb) | -DENABLE_SMB=ON | | SFTP | [FetchSFTP](PROCESSORS.md#fetchsftp)
[ListSFTP](PROCESSORS.md#listsftp)
[PutSFTP](PROCESSORS.md#putsftp) | -DENABLE_SFTP=ON | | SQL | [ExecuteSQL](PROCESSORS.md#executesql)
[PutSQL](PROCESSORS.md#putsql)
[QueryDatabaseTable](PROCESSORS.md#querydatabasetable)
| -DENABLE_SQL=ON | | Splunk | [PutSplunkHTTP](PROCESSORS.md#putsplunkhttp)
[QuerySplunkIndexingStatus](PROCESSORS.md#querysplunkindexingstatus) | -DENABLE_SPLUNK=ON | From 03015fc2211706a771dde26aa6694488659522bd Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 8 Sep 2023 18:45:39 +0200 Subject: [PATCH 13/25] remove ~PutSmb() override = default; --- extensions/smb/PutSmb.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/extensions/smb/PutSmb.h b/extensions/smb/PutSmb.h index a104ddc8e0..99144e16e8 100644 --- a/extensions/smb/PutSmb.h +++ b/extensions/smb/PutSmb.h @@ -35,8 +35,6 @@ class PutSmb : public core::Processor { : core::Processor(std::move(name), uuid) { } - ~PutSmb() override = default; - enum class FileExistsResolutionStrategy { fail, replace, From 5d063870184f677b7e4847ed771bc5bf4f9c74ac Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 12 Sep 2023 11:54:02 +0100 Subject: [PATCH 14/25] ListSmb path attribute to work like NiFi --- extensions/smb/ListSmb.cpp | 2 +- extensions/smb/ListSmb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/smb/ListSmb.cpp b/extensions/smb/ListSmb.cpp index 1d2dc08c53..99fb8f7761 100644 --- a/extensions/smb/ListSmb.cpp +++ b/extensions/smb/ListSmb.cpp @@ -84,7 +84,7 @@ std::shared_ptr ListSmb::createFlowFile(core::ProcessSession& se session.putAttribute(flow_file, core::SpecialFlowAttribute::FILENAME, listed_file.getPath().filename().string()); session.putAttribute(flow_file, core::SpecialFlowAttribute::ABSOLUTE_PATH, (listed_file.getPath().parent_path() / "").string()); - auto relative_path = std::filesystem::relative(listed_file.getPath().parent_path(), listed_file.getDirectory()); + auto relative_path = std::filesystem::relative(listed_file.getPath().parent_path(), smb_connection_controller_service_->getPath()); session.putAttribute(flow_file, core::SpecialFlowAttribute::PATH, (relative_path / "").string()); session.putAttribute(flow_file, Size.name, std::to_string(utils::file::file_size(listed_file.getPath()))); diff --git a/extensions/smb/ListSmb.h b/extensions/smb/ListSmb.h index 732729269d..a52ae69032 100644 --- a/extensions/smb/ListSmb.h +++ b/extensions/smb/ListSmb.h @@ -112,7 +112,7 @@ class ListSmb : public core::Processor { EXTENSIONAPI static constexpr auto Path = core::OutputAttributeDefinition<>{"path", { Success }, "The path is set to the relative path of the file's directory on the remote filesystem compared to the Share root directory. " "For example, for a given remote location smb://HOSTNAME:PORT/SHARE/DIRECTORY, and a file is being listed from smb://HOSTNAME:PORT/SHARE/DIRECTORY/sub/folder/file " - "then the path attribute will be set to \"sub/folder\"."}; + "then the path attribute will be set to \"DIRECTORY/sub/folder\"."}; EXTENSIONAPI static constexpr auto ServiceLocation = core::OutputAttributeDefinition<>{"serviceLocation", { Success }, "The SMB URL of the share."}; EXTENSIONAPI static constexpr auto LastModifiedTime = core::OutputAttributeDefinition<>{"lastModifiedTime", { Success }, From 0b8ed3d78df2ec004579e4cb0622edacb3538244 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 12 Sep 2023 12:14:31 +0100 Subject: [PATCH 15/25] fix tests --- extensions/smb/tests/ListSmbTests.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/extensions/smb/tests/ListSmbTests.cpp b/extensions/smb/tests/ListSmbTests.cpp index babe1c424b..a89cb00bc1 100644 --- a/extensions/smb/tests/ListSmbTests.cpp +++ b/extensions/smb/tests/ListSmbTests.cpp @@ -63,9 +63,9 @@ TEST_CASE("ListSmb tests") { auto a_expected_attributes = mock_smb_connection_controller_service->addFile("a.foo", std::string(10_KiB, 'a'), 5min); auto b_expected_attributes = mock_smb_connection_controller_service->addFile("b.foo", std::string(13_KiB, 'b'), 1h); auto c_expected_attributes = mock_smb_connection_controller_service->addFile("c.bar", std::string(1_KiB, 'c'), 2h); - auto d_expected_attributes = mock_smb_connection_controller_service->addFile("subdir/d.foo", std::string(100, 'd'), 10min); - auto e_expected_attributes = mock_smb_connection_controller_service->addFile("subdir2/e.foo", std::string(1, 'e'), 10s); - auto f_expected_attributes = mock_smb_connection_controller_service->addFile("third/f.bar", std::string(50_KiB, 'f'), 30min); + auto d_expected_attributes = mock_smb_connection_controller_service->addFile(std::filesystem::path("subdir") / "some" / "d.foo", std::string(100, 'd'), 10min); + auto e_expected_attributes = mock_smb_connection_controller_service->addFile(std::filesystem::path("subdir2") /"e.foo", std::string(1, 'e'), 10s); + auto f_expected_attributes = mock_smb_connection_controller_service->addFile(std::filesystem::path("third") / "f.bar", std::string(50_KiB, 'f'), 30min); auto g_expected_attributes = mock_smb_connection_controller_service->addFile("g.foo", std::string(50_KiB, 'f'), 30min); auto hide_file_error = minifi::test::utils::hide_file(mock_smb_connection_controller_service->getPath() / "g.foo"); REQUIRE_FALSE(hide_file_error); @@ -84,6 +84,15 @@ TEST_CASE("ListSmb tests") { CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *b_expected_attributes)); } + SECTION("Input directory") { + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::InputDirectory, "subdir")); + REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "true")); + const auto trigger_results = controller.trigger(); + CHECK(trigger_results.at(ListSmb::Success).size() == 1); + CHECK_FALSE(list_smb->isYield()); + CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *d_expected_attributes)); + } + SECTION("PathFilter and FileFilter") { REQUIRE(controller.plan->setProperty(list_smb, ListSmb::FileFilter, ".*\\.foo")); REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "true")); From c13cbaa75ae2cfb79ab025f8efe6c6824bf8050f Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 12 Sep 2023 12:18:54 +0100 Subject: [PATCH 16/25] enable SMB by default --- cmake/MiNiFiOptions.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 22568ddb83..c020820450 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -85,7 +85,7 @@ if (WIN32) add_minifi_option(MSI_REDISTRIBUTE_UCRT_NONASL "Redistribute Universal C Runtime DLLs with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) add_minifi_option(ENABLE_WEL "Enables the suite of Windows Event Log extensions." OFF) add_minifi_option(ENABLE_PDH "Enables PDH support." OFF) - add_minifi_option(ENABLE_SMB "Enables SMB support." OFF) + add_minifi_option(ENABLE_SMB "Enables SMB support." ON) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") From 0c60601294e8fb42052a68ba1c3842325fa4f095 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 12 Sep 2023 12:32:15 +0100 Subject: [PATCH 17/25] use count_if --- libminifi/include/utils/file/FileUtils.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/libminifi/include/utils/file/FileUtils.h b/libminifi/include/utils/file/FileUtils.h index 1295d9ee9d..440f578232 100644 --- a/libminifi/include/utils/file/FileUtils.h +++ b/libminifi/include/utils/file/FileUtils.h @@ -575,12 +575,8 @@ inline std::optional get_relative_path(const std::filesys } inline size_t countNumberOfFiles(const std::filesystem::path& path) { - size_t counter = 0; - for (const auto& entry : std::filesystem::directory_iterator(path)) { - if (entry.is_regular_file()) - ++counter; - } - return counter; + using std::filesystem::directory_iterator; + return std::count_if(directory_iterator(path), directory_iterator(), [](const auto& entry) { return entry.is_regular_file(); }); } #ifdef WIN32 From 8ad193c9e5a26c5821ecf19d875ab9f05ad3807a Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 12 Sep 2023 13:06:15 +0100 Subject: [PATCH 18/25] erase leading slashes --- extensions/smb/FetchSmb.cpp | 4 +--- extensions/smb/tests/FetchSmbTests.cpp | 5 ++++- extensions/smb/tests/ListAndFetchSmbTests.cpp | 9 ++++++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/extensions/smb/FetchSmb.cpp b/extensions/smb/FetchSmb.cpp index 29330f9095..7f70d2ad48 100644 --- a/extensions/smb/FetchSmb.cpp +++ b/extensions/smb/FetchSmb.cpp @@ -35,9 +35,7 @@ namespace { std::filesystem::path getTargetRelativePath(core::ProcessContext& context, const std::shared_ptr& flow_file) { auto remote_file = context.getProperty(FetchSmb::RemoteFile, flow_file); if (remote_file && !remote_file->empty()) { - if (remote_file->starts_with('/')) - remote_file->erase(remote_file->begin()); - return *remote_file; + return std::filesystem::path{*remote_file}.relative_path(); // We need to make sure that the path remains relative (e.g. ${path}/foo where ${path} is empty can lead to /foo) } std::filesystem::path path = flow_file->getAttribute(core::SpecialFlowAttribute::PATH).value_or(""); std::filesystem::path filename = flow_file->getAttribute(core::SpecialFlowAttribute::FILENAME).value_or(""); diff --git a/extensions/smb/tests/FetchSmbTests.cpp b/extensions/smb/tests/FetchSmbTests.cpp index 9e00c5f43c..9f6c870f41 100644 --- a/extensions/smb/tests/FetchSmbTests.cpp +++ b/extensions/smb/tests/FetchSmbTests.cpp @@ -63,9 +63,12 @@ TEST_CASE("FetchSmb tests") { SECTION("Without Remote File property") { } - SECTION("Remote File Property with expression language") { + SECTION("Remote File Property with expression language (unix separator)") { REQUIRE(controller.plan->setProperty(fetch_smb, FetchSmb::RemoteFile, "${path}/${filename}")); } + SECTION("Remote File Property with expression language (windows separator)") { + REQUIRE(controller.plan->setProperty(fetch_smb, FetchSmb::RemoteFile, "${path}\\${filename}")); + } { const auto trigger_results = controller.trigger(original_content, {{std::string(core::SpecialFlowAttribute::FILENAME), "a.foo"}, {std::string(core::SpecialFlowAttribute::PATH), ""}}); diff --git a/extensions/smb/tests/ListAndFetchSmbTests.cpp b/extensions/smb/tests/ListAndFetchSmbTests.cpp index 76990e2e41..a0481c1d9b 100644 --- a/extensions/smb/tests/ListAndFetchSmbTests.cpp +++ b/extensions/smb/tests/ListAndFetchSmbTests.cpp @@ -55,7 +55,14 @@ TEST_CASE("ListSmb and FetchSmb work together") { mock_smb_connection_controller_service->setPath(controller.createTempDirectory()); constexpr std::string_view content = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus malesuada elit odio, sit amet viverra ante venenatis eget."; - mock_smb_connection_controller_service->addFile("subdir/b.foo", content, 5min); + mock_smb_connection_controller_service->addFile("input_dir/sub_dir/b.foo", content, 5min); + + SECTION("With Input Directory") { + plan->setProperty(list_smb, ListSmb::InputDirectory, "input_dir"); + } + + SECTION("Without Input Directory") { + } controller.runSession(plan); CHECK(read_from_success_relationship->numberOfFlowFilesRead() == 1); From e4e1f1d7d053f7ced2dd020b451bf21a289805fa Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 12 Sep 2023 15:25:33 +0200 Subject: [PATCH 19/25] use std::to_string instead of fmt::format with simple {} --- extensions/smb/FetchSmb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/smb/FetchSmb.cpp b/extensions/smb/FetchSmb.cpp index 7f70d2ad48..19c7a848f1 100644 --- a/extensions/smb/FetchSmb.cpp +++ b/extensions/smb/FetchSmb.cpp @@ -62,7 +62,7 @@ void FetchSmb::onTrigger(const std::shared_ptr& context, c session->write(flow_file, utils::FileReaderCallback{smb_connection_controller_service_->getPath() / getTargetRelativePath(*context, flow_file)}); session->transfer(flow_file, Success); } catch (const utils::FileReaderCallbackIOError& io_error) { - flow_file->addAttribute(ErrorCode.name, fmt::format("{}", io_error.error_code)); + flow_file->addAttribute(ErrorCode.name, std::to_string(io_error.error_code)); flow_file->addAttribute(ErrorMessage.name, io_error.what()); session->transfer(flow_file, Failure); } From 6f5a832174023ecb441298d5fa6d63a0d81dee3b Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 12 Sep 2023 17:19:24 +0200 Subject: [PATCH 20/25] add Put/Fetch/ListFile recommendation in case of automounted SMB shared --- extensions/smb/FetchSmb.h | 3 ++- extensions/smb/ListSmb.h | 3 ++- extensions/smb/PutSmb.h | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/extensions/smb/FetchSmb.h b/extensions/smb/FetchSmb.h index 52d415545e..c1f0286805 100644 --- a/extensions/smb/FetchSmb.h +++ b/extensions/smb/FetchSmb.h @@ -46,7 +46,8 @@ class FetchSmb : public core::Processor { EXTENSIONAPI static constexpr const char* Description = "Fetches files from a SMB Share. Designed to be used in tandem with ListSmb."; EXTENSIONAPI static constexpr auto ConnectionControllerService = core::PropertyDefinitionBuilder<>::createProperty("SMB Connection Controller Service") - .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server.") + .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server. " + "If the SMB share is auto-mounted to a drive letter, its recommended to use FetchFile instead.") .isRequired(true) .withAllowedTypes() .build(); diff --git a/extensions/smb/ListSmb.h b/extensions/smb/ListSmb.h index a52ae69032..a2f42089ba 100644 --- a/extensions/smb/ListSmb.h +++ b/extensions/smb/ListSmb.h @@ -47,7 +47,8 @@ class ListSmb : public core::Processor { "creates a FlowFile that represents the file so that it can be fetched in conjunction with FetchSmb."; EXTENSIONAPI static constexpr auto ConnectionControllerService = core::PropertyDefinitionBuilder<>::createProperty("SMB Connection Controller Service") - .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server.") + .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server. " + "If the SMB share is auto-mounted to a drive letter, its recommended to use ListFile instead.") .isRequired(true) .withAllowedTypes() .build(); diff --git a/extensions/smb/PutSmb.h b/extensions/smb/PutSmb.h index 99144e16e8..72f256704f 100644 --- a/extensions/smb/PutSmb.h +++ b/extensions/smb/PutSmb.h @@ -44,7 +44,8 @@ class PutSmb : public core::Processor { EXTENSIONAPI static constexpr const char* Description = "Writes the contents of a FlowFile to an smb network location"; EXTENSIONAPI static constexpr auto ConnectionControllerService = core::PropertyDefinitionBuilder<>::createProperty("SMB Connection Controller Service") - .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server.") + .withDescription("Specifies the SMB connection controller service to use for connecting to the SMB server. " + "If the SMB share is auto-mounted to a drive letter, its recommended to use PutFile instead.") .isRequired(true) .withAllowedTypes() .build(); From 8f86d3dd63633cc77d6d009877bfa16e4cd64571 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 13 Sep 2023 13:08:04 +0200 Subject: [PATCH 21/25] only warn in case of missing PutFile::ConflictResolution --- .../processors/PutFile.cpp | 2 +- .../include/utils/ProcessorConfigUtils.h | 21 ++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/extensions/standard-processors/processors/PutFile.cpp b/extensions/standard-processors/processors/PutFile.cpp index 9db95e6213..d2147fb8c7 100644 --- a/extensions/standard-processors/processors/PutFile.cpp +++ b/extensions/standard-processors/processors/PutFile.cpp @@ -42,7 +42,7 @@ void PutFile::initialize() { } void PutFile::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory* /*sessionFactory*/) { - conflict_resolution_strategy_ = utils::parseEnumProperty(*context, ConflictResolution); + conflict_resolution_strategy_ = utils::parseEnumPropertyOrWarnAndFallbackToDefault(*context, ConflictResolution, *logger_, FileExistsResolutionStrategy::fail); try_mkdirs_ = context->getProperty(CreateDirs).value_or(true); if (auto max_dest_files = context->getProperty(MaxDestFiles); max_dest_files && *max_dest_files > 0) { max_dest_files_ = gsl::narrow_cast(*max_dest_files); diff --git a/libminifi/include/utils/ProcessorConfigUtils.h b/libminifi/include/utils/ProcessorConfigUtils.h index 1b9f81c90c..8708a36b68 100644 --- a/libminifi/include/utils/ProcessorConfigUtils.h +++ b/libminifi/include/utils/ProcessorConfigUtils.h @@ -42,18 +42,33 @@ bool parseBooleanPropertyOrThrow(const core::ProcessContext& context, std::strin std::chrono::milliseconds parseTimePropertyMSOrThrow(const core::ProcessContext& context, std::string_view property_name); template -T parseEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { +nonstd::expected parseEnumPropertyNoThrow(const core::ProcessContext& context, const core::PropertyReference& prop) { std::string value; if (!context.getProperty(prop.name, value)) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' is missing"); + return nonstd::make_unexpected(fmt::format("Property '{}' is missing'", prop.name)); } auto result = magic_enum::enum_cast(value); if (!result) { - throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + value + "'"); + return nonstd::make_unexpected(fmt::format("Property '{}' has invalid value: '{}'", prop.name, value)); } return result.value(); } +template +T parseEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { + auto parsed_enum = parseEnumPropertyNoThrow(context, prop); + if (!parsed_enum) + throw Exception(PROCESS_SCHEDULE_EXCEPTION, parsed_enum.error()); + return parsed_enum.value(); +} + +template +T parseEnumPropertyOrWarnAndFallbackToDefault(const core::ProcessContext& context, const core::PropertyReference& prop, core::logging::Logger& logger, T&& default_value) { + auto parsed_enum = parseEnumPropertyNoThrow(context, prop); + if (!parsed_enum) { logger.log_warn("%s", parsed_enum.error()); } + return parsed_enum.value_or(default_value); +} + template std::optional parseOptionalEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { std::string value; From 4dbb2fd08f73b007a73b3926f5b62a4ba0fb9279 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 13 Sep 2023 15:34:45 +0200 Subject: [PATCH 22/25] linter fix --- extensions/smb/tests/ListSmbTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/smb/tests/ListSmbTests.cpp b/extensions/smb/tests/ListSmbTests.cpp index a89cb00bc1..d94f40bb0f 100644 --- a/extensions/smb/tests/ListSmbTests.cpp +++ b/extensions/smb/tests/ListSmbTests.cpp @@ -92,7 +92,7 @@ TEST_CASE("ListSmb tests") { CHECK_FALSE(list_smb->isYield()); CHECK(checkForFlowFileWithAttributes(trigger_results.at(ListSmb::Success), *d_expected_attributes)); } - + SECTION("PathFilter and FileFilter") { REQUIRE(controller.plan->setProperty(list_smb, ListSmb::FileFilter, ".*\\.foo")); REQUIRE(controller.plan->setProperty(list_smb, ListSmb::RecurseSubdirectories, "true")); From 99f9af271e923e4ac14ade0c1bdf6a010805f0ca Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Wed, 13 Sep 2023 16:28:40 +0200 Subject: [PATCH 23/25] remove unused /SMB flag from ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9de954dfe..265a5d3390 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,7 +108,7 @@ jobs: - name: build run: | for /f "usebackq delims=" %%i in (`vswhere.exe -latest -property installationPath`) do if exist "%%i\Common7\Tools\vsdevcmd.bat" call "%%i\Common7\Tools\vsdevcmd.bat" -arch=x64 -host_arch=x64 - win_build_vs.bat ..\b /64 /CI /S /A /PDH /SMB /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA + win_build_vs.bat ..\b /64 /CI /S /A /PDH /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA sccache --show-stats shell: cmd - name: sccache cache save From eef66039b99d1cccc1424e7ea3f79e360f8a3e6e Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Fri, 15 Sep 2023 10:03:21 +0200 Subject: [PATCH 24/25] Revert "only warn in case of missing PutFile::ConflictResolution" This reverts commit 09c396a1c13c9e2877a8f6942cd5395a25f5deed. --- .../processors/PutFile.cpp | 2 +- .../include/utils/ProcessorConfigUtils.h | 21 +++---------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/extensions/standard-processors/processors/PutFile.cpp b/extensions/standard-processors/processors/PutFile.cpp index d2147fb8c7..9db95e6213 100644 --- a/extensions/standard-processors/processors/PutFile.cpp +++ b/extensions/standard-processors/processors/PutFile.cpp @@ -42,7 +42,7 @@ void PutFile::initialize() { } void PutFile::onSchedule(core::ProcessContext *context, core::ProcessSessionFactory* /*sessionFactory*/) { - conflict_resolution_strategy_ = utils::parseEnumPropertyOrWarnAndFallbackToDefault(*context, ConflictResolution, *logger_, FileExistsResolutionStrategy::fail); + conflict_resolution_strategy_ = utils::parseEnumProperty(*context, ConflictResolution); try_mkdirs_ = context->getProperty(CreateDirs).value_or(true); if (auto max_dest_files = context->getProperty(MaxDestFiles); max_dest_files && *max_dest_files > 0) { max_dest_files_ = gsl::narrow_cast(*max_dest_files); diff --git a/libminifi/include/utils/ProcessorConfigUtils.h b/libminifi/include/utils/ProcessorConfigUtils.h index 8708a36b68..1b9f81c90c 100644 --- a/libminifi/include/utils/ProcessorConfigUtils.h +++ b/libminifi/include/utils/ProcessorConfigUtils.h @@ -42,33 +42,18 @@ bool parseBooleanPropertyOrThrow(const core::ProcessContext& context, std::strin std::chrono::milliseconds parseTimePropertyMSOrThrow(const core::ProcessContext& context, std::string_view property_name); template -nonstd::expected parseEnumPropertyNoThrow(const core::ProcessContext& context, const core::PropertyReference& prop) { +T parseEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { std::string value; if (!context.getProperty(prop.name, value)) { - return nonstd::make_unexpected(fmt::format("Property '{}' is missing'", prop.name)); + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' is missing"); } auto result = magic_enum::enum_cast(value); if (!result) { - return nonstd::make_unexpected(fmt::format("Property '{}' has invalid value: '{}'", prop.name, value)); + throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Property '" + std::string(prop.name) + "' has invalid value: '" + value + "'"); } return result.value(); } -template -T parseEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { - auto parsed_enum = parseEnumPropertyNoThrow(context, prop); - if (!parsed_enum) - throw Exception(PROCESS_SCHEDULE_EXCEPTION, parsed_enum.error()); - return parsed_enum.value(); -} - -template -T parseEnumPropertyOrWarnAndFallbackToDefault(const core::ProcessContext& context, const core::PropertyReference& prop, core::logging::Logger& logger, T&& default_value) { - auto parsed_enum = parseEnumPropertyNoThrow(context, prop); - if (!parsed_enum) { logger.log_warn("%s", parsed_enum.error()); } - return parsed_enum.value_or(default_value); -} - template std::optional parseOptionalEnumProperty(const core::ProcessContext& context, const core::PropertyReference& prop) { std::string value; From 8c19ad11743a212f6bfbbf768fa710bf3da37ba7 Mon Sep 17 00:00:00 2001 From: Martin Zink Date: Tue, 19 Sep 2023 14:11:33 +0200 Subject: [PATCH 25/25] increase time delta for MockSmbConnectionControllerService --- .../smb/tests/utils/MockSmbConnectionControllerService.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions/smb/tests/utils/MockSmbConnectionControllerService.h b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h index 32c9eeb7be..6e85b006cf 100644 --- a/extensions/smb/tests/utils/MockSmbConnectionControllerService.h +++ b/extensions/smb/tests/utils/MockSmbConnectionControllerService.h @@ -46,9 +46,9 @@ struct ListSmbExpectedAttributes { auto creation_time_from_attribute = utils::timeutils::parseDateTimeStr(*flow_file.getAttribute(ListSmb::CreationTime.name)); auto last_access_time_from_attribute = utils::timeutils::parseDateTimeStr(*flow_file.getAttribute(ListSmb::LastAccessTime.name)); - CHECK(std::chrono::abs(expected_last_modified_time - *last_modified_time_from_attribute) < 1s); - CHECK(std::chrono::abs(expected_creation_time - *creation_time_from_attribute) < 1s); - CHECK(std::chrono::abs(expected_last_access_time - *last_access_time_from_attribute) < 1s); + CHECK(std::chrono::abs(expected_last_modified_time - *last_modified_time_from_attribute) < 5s); + CHECK(std::chrono::abs(expected_creation_time - *creation_time_from_attribute) < 5s); + CHECK(std::chrono::abs(expected_last_access_time - *last_access_time_from_attribute) < 5s); CHECK(flow_file.getAttribute(ListSmb::Size.name) == expected_size); } };