From 2bd82558a028c898ae19821716820ecbd8ec33bd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 3 Apr 2025 16:39:06 -0400 Subject: [PATCH 1/7] Update dev util dependency --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index ad576e43c..d22183ffa 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit ad576e43c1a43d7a6afde79fc9c3c952b7bf28bd +Subproject commit d22183ffa3fb95745b012eb38a6e13b7e5f06fca From 224930f132064841210b1b93b4cdf983198a7b70 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 3 Apr 2025 17:09:28 -0400 Subject: [PATCH 2/7] Checkout files --- lint-tasks.yaml | 22 +++++++++++----------- taskfile.yaml | 2 +- test-tasks.yaml | 16 ++++++++-------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 6897b7f1e..5daf5aae8 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -32,7 +32,7 @@ tasks: vars: FLAGS: "--in-place" - cpp-configs: "tools/yscope-dev-utils/lint-configs/symlink-cpp-lint-configs.sh" + cpp-configs: "tools/yscope-dev-utils/exports/lint-configs/symlink-cpp-lint-configs.sh" cpp-check: cmds: @@ -58,7 +58,7 @@ tasks: - "{{.G_TEST_DIR}}/**/*.h" - "{{.G_TEST_DIR}}/**/*.hpp" - "{{.TASKFILE}}" - - "tools/yscope-dev-utils/lint-configs/.clang-format" + - "tools/yscope-dev-utils/exports/lint-configs/.clang-format" deps: ["cpp-configs", "venv"] cmds: - task: "clang-format" @@ -112,7 +112,7 @@ tasks: - "{{.G_SPIDER_COMPILE_COMMANDS_DB}}" - "{{.TASKFILE}}" - "taskfile.yaml" - - "tools/yscope-dev-utils/lint-configs/.clang-tidy" + - "tools/yscope-dev-utils/exports/lint-configs/.clang-tidy" deps: [":config-cmake-project", "cpp-configs", "venv"] cmds: - task: "clang-tidy" @@ -165,7 +165,7 @@ tasks: - |- . "{{.G_LINT_VENV_DIR}}/bin/activate" yamllint \ - --config-file "tools/yscope-dev-utils/lint-configs/.yamllint.yml" \ + --config-file "tools/yscope-dev-utils/exports/lint-configs/.yamllint.yml" \ --strict \ .gersemirc \ .github/ \ @@ -229,18 +229,18 @@ tasks: run: "once" deps: - ":init" - - task: ":utils:validate-checksum" + - task: ":utils:checksum:validate" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - DATA_DIR: "{{.OUTPUT_DIR}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] cmds: - - task: ":utils:create-venv" + - task: ":utils:misc:create-venv" vars: LABEL: "lint" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "lint-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-requirements.txt" # This command must be last - - task: ":utils:compute-checksum" + - task: ":utils:checksum:compute" vars: - DATA_DIR: "{{.OUTPUT_DIR}}" - OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] diff --git a/taskfile.yaml b/taskfile.yaml index 8b8778fde..cd5884ad3 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -6,7 +6,7 @@ includes: docs: "docs/tasks.yaml" lint: "lint-tasks.yaml" test: "test-tasks.yaml" - utils: "tools/yscope-dev-utils/taskfiles/utils.yml" + utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" vars: G_BUILD_DIR: "{{.ROOT_DIR}}/build" diff --git a/test-tasks.yaml b/test-tasks.yaml index 55bd14215..8b082dd4e 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -61,18 +61,18 @@ tasks: run: "once" deps: - ":init" - - task: ":utils:validate-checksum" + - task: ":utils:checksum:validate" vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - DATA_DIR: "{{.OUTPUT_DIR}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] cmds: - - task: ":utils:create-venv" + - task: ":utils:misc:create-venv" vars: - LABEL: "test" + LABEL: "lint" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "test-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-requirements.txt" # This command must be last - - task: ":utils:compute-checksum" + - task: ":utils:checksum:compute" vars: - DATA_DIR: "{{.OUTPUT_DIR}}" - OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] From 2e04942d78d5198d07d33e202cbeeff4457e7f79 Mon Sep 17 00:00:00 2001 From: sitao Date: Thu, 3 Apr 2025 18:02:59 -0400 Subject: [PATCH 3/7] Update lint requirement version --- lint-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lint-requirements.txt b/lint-requirements.txt index 9430c0990..9eff18489 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,8 +1,6 @@ black>=24.4.2 -# Lock to v18.x until we can upgrade our code to meet v19's formatting standards. -clang-format~=18.1 +clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 ruff>=0.4.4 -gersemi>=0.16.2 yamllint>=1.35.1 From 854f6f2b42b5427329ac38bf8825d4f554ad081b Mon Sep 17 00:00:00 2001 From: sitao Date: Thu, 3 Apr 2025 19:48:17 -0400 Subject: [PATCH 4/7] Add gersemi in lint requirements --- lint-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/lint-requirements.txt b/lint-requirements.txt index 9eff18489..717adb508 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -3,4 +3,5 @@ clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 ruff>=0.4.4 +gersemi>=0.16.2 yamllint>=1.35.1 From d5cea202e840c2a8bcda807c8ba67212e33a4f1b Mon Sep 17 00:00:00 2001 From: sitao Date: Fri, 4 Apr 2025 12:42:47 -0400 Subject: [PATCH 5/7] Use clang-format task from dev-utils --- lint-tasks.yaml | 66 ++++++++++++++----------------------------------- 1 file changed, 18 insertions(+), 48 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 5daf5aae8..bff7c0025 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -45,51 +45,33 @@ tasks: - task: "cpp-static-fix" cpp-format-check: - sources: &cpp_format_src_files - - "{{.G_LINT_VENV_CHECKSUM_FILE}}" - - "{{.G_SRC_SPIDER_DIR}}/.clang-format" - - "{{.G_EXAMPLES_DIR}}/**/*.cpp" - - "{{.G_EXAMPLES_DIR}}/**/*.h" - - "{{.G_EXAMPLES_DIR}}/**/*.hpp" - - "{{.G_SRC_SPIDER_DIR}}/**/*.cpp" - - "{{.G_SRC_SPIDER_DIR}}/**/*.h" - - "{{.G_SRC_SPIDER_DIR}}/**/*.hpp" - - "{{.G_TEST_DIR}}/**/*.cpp" - - "{{.G_TEST_DIR}}/**/*.h" - - "{{.G_TEST_DIR}}/**/*.hpp" + sources: &cpp_source_files + - "{{.ROOT_DIR}}/.clang-format" + - "{{.ROOT_DIR}}/.clang-tidy" - "{{.TASKFILE}}" - - "tools/yscope-dev-utils/exports/lint-configs/.clang-format" + - "{{.G_SRC_SPIDER_DIR}}/.clang-format" + - "{{.G_EXAMPLES_DIR}}/**" + - "{{.G_SRC_SPIDER_DIR}}/**" + - "{{.G_TEST_DIR}}/**" deps: ["cpp-configs", "venv"] cmds: - - task: "clang-format" - vars: - FLAGS: "--dry-run" - SRC_DIR: "{{.G_SRC_SPIDER_DIR}}" - - task: "clang-format" - vars: - FLAGS: "--dry-run" - SRC_DIR: "{{.G_TEST_DIR}}" - - task: "clang-format" + - task: ":utils:cpp-lint:clang-format" vars: - FLAGS: "--dry-run" - SRC_DIR: "{{.G_EXAMPLES_DIR}}" + FLAGS: ["--dry-run"] + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] + ROOT_PATHS: *cpp_source_files + VENV_DIR: "{{.G_LINT_VENV_DIR}}" cpp-format-fix: - sources: *cpp_format_src_files + sources: *cpp_source_files deps: ["cpp-configs", "venv"] cmds: - - task: "clang-format" - vars: - FLAGS: "-i" - SRC_DIR: "{{.G_SRC_SPIDER_DIR}}" - - task: "clang-format" - vars: - FLAGS: "-i" - SRC_DIR: "{{.G_TEST_DIR}}" - - task: "clang-format" + - task: ":utils:cpp-lint:clang-format" vars: - FLAGS: "-i" - SRC_DIR: "{{.G_EXAMPLES_DIR}}" + FLAGS: ["-i"] + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] + ROOT_PATHS: *cpp_source_files + VENV_DIR: "{{.G_LINT_VENV_DIR}}" cpp-static-check: # Alias task to `cpp-static-fix` since we don't currently support automatic fixes. @@ -176,18 +158,6 @@ tasks: taskfile.yaml \ test-tasks.yaml - clang-format: - internal: true - requires: - vars: ["FLAGS", "SRC_DIR"] - cmd: |- - . "{{.G_LINT_VENV_DIR}}/bin/activate" - find "{{.SRC_DIR}}" \ - -type f \ - \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) \ - -print0 | \ - xargs -0 --no-run-if-empty clang-format {{.FLAGS}} -Werror - clang-tidy: internal: true requires: From f805e59bc8654e4aa1a29b89e1245315797a9c6b Mon Sep 17 00:00:00 2001 From: sitao Date: Fri, 4 Apr 2025 12:44:29 -0400 Subject: [PATCH 6/7] Reformat source files --- src/spider/client/Data.hpp | 1 - src/spider/client/Driver.cpp | 2 - src/spider/client/Driver.hpp | 8 +- src/spider/client/Job.hpp | 9 +- src/spider/client/TaskContext.cpp | 2 - src/spider/client/TaskContext.hpp | 8 +- src/spider/client/task.hpp | 2 - src/spider/client/type_utils.hpp | 1 - src/spider/core/Data.hpp | 1 - src/spider/core/DataImpl.hpp | 2 - src/spider/core/Driver.hpp | 2 - src/spider/core/Error.hpp | 1 - src/spider/core/JobMetadata.hpp | 2 - src/spider/core/Task.hpp | 19 +-- src/spider/core/TaskContextImpl.hpp | 9 +- src/spider/core/TaskGraph.hpp | 16 +- src/spider/core/TaskGraphImpl.hpp | 17 ++- src/spider/io/msgpack_message.cpp | 14 +- src/spider/io/msgpack_message.hpp | 6 +- src/spider/scheduler/FifoPolicy.cpp | 13 +- src/spider/scheduler/FifoPolicy.hpp | 2 - src/spider/scheduler/SchedulerMessage.hpp | 2 - src/spider/scheduler/SchedulerPolicy.hpp | 4 +- src/spider/scheduler/SchedulerServer.cpp | 6 +- src/spider/scheduler/SchedulerServer.hpp | 2 - src/spider/scheduler/scheduler.cpp | 11 +- src/spider/storage/DataStorage.hpp | 48 +++--- src/spider/storage/MetadataStorage.hpp | 141 +++++++++--------- src/spider/storage/StorageConnection.hpp | 2 - src/spider/storage/StorageFactory.hpp | 11 +- src/spider/storage/mysql/MySqlConnection.cpp | 6 +- src/spider/storage/mysql/MySqlConnection.hpp | 9 +- .../storage/mysql/MySqlJobSubmissionBatch.cpp | 2 - .../storage/mysql/MySqlJobSubmissionBatch.hpp | 1 - src/spider/storage/mysql/MySqlStorage.cpp | 38 +++-- src/spider/storage/mysql/MySqlStorage.hpp | 62 ++++---- .../storage/mysql/MySqlStorageFactory.cpp | 9 +- .../storage/mysql/MySqlStorageFactory.hpp | 4 +- src/spider/utils/LruCache.hpp | 2 - src/spider/worker/DllLoader.cpp | 2 - src/spider/worker/DllLoader.hpp | 2 - src/spider/worker/FunctionManager.cpp | 10 +- src/spider/worker/FunctionManager.hpp | 17 +-- src/spider/worker/FunctionNameManager.cpp | 2 - src/spider/worker/FunctionNameManager.hpp | 2 - src/spider/worker/Process.cpp | 4 - src/spider/worker/Process.hpp | 2 - src/spider/worker/TaskExecutor.cpp | 11 +- src/spider/worker/TaskExecutor.hpp | 2 - src/spider/worker/TaskExecutorMessage.hpp | 1 - src/spider/worker/WorkerClient.cpp | 6 +- src/spider/worker/WorkerClient.hpp | 4 +- src/spider/worker/message_pipe.cpp | 6 +- src/spider/worker/message_pipe.hpp | 6 +- src/spider/worker/task_executor.cpp | 3 - src/spider/worker/worker.cpp | 39 +++-- tests/client/client-test.cpp | 1 - tests/client/test-Driver.cpp | 1 - tests/io/test-MsgpackMessage.cpp | 2 - tests/scheduler/test-SchedulerServer.cpp | 1 - tests/storage/StorageTestHelper.hpp | 1 - tests/storage/test-DataStorage.cpp | 1 - tests/storage/test-MetadataStorage.cpp | 8 +- tests/utils/CoreDataUtils.hpp | 2 - tests/utils/CoreTaskUtils.cpp | 3 - tests/utils/CoreTaskUtils.hpp | 2 - tests/worker/test-Process.cpp | 2 - tests/worker/test-TaskExecutor.cpp | 5 +- 68 files changed, 275 insertions(+), 370 deletions(-) diff --git a/src/spider/client/Data.hpp b/src/spider/client/Data.hpp index 549674f52..9ef8116ef 100644 --- a/src/spider/client/Data.hpp +++ b/src/spider/client/Data.hpp @@ -20,7 +20,6 @@ #include "Exception.hpp" namespace spider { - namespace core { class Data; class DataStorage; diff --git a/src/spider/client/Driver.cpp b/src/spider/client/Driver.cpp index 1ee399eff..f4acb8fad 100644 --- a/src/spider/client/Driver.cpp +++ b/src/spider/client/Driver.cpp @@ -21,7 +21,6 @@ #include "Exception.hpp" namespace spider { - Driver::Driver(std::string const& storage_url) : m_storage_factory{std::make_shared(storage_url)} { boost::uuids::random_generator gen; @@ -126,5 +125,4 @@ auto Driver::kv_store_get(std::string const& key) -> std::optional } return value; } - } // namespace spider diff --git a/src/spider/client/Driver.hpp b/src/spider/client/Driver.hpp index 83f184782..558cfae83 100644 --- a/src/spider/client/Driver.hpp +++ b/src/spider/client/Driver.hpp @@ -183,8 +183,8 @@ class Driver { * @throw spider::ConnectionException */ template - auto - start(TaskFunction const& task, Inputs&&... inputs) -> Job { + auto start(TaskFunction const& task, Inputs&&... inputs) + -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), @@ -247,8 +247,8 @@ class Driver { * @throw spider::ConnectionException */ template - auto - start(TaskGraph const& graph, Inputs&&... inputs) -> Job { + auto start(TaskGraph const& graph, Inputs&&... inputs) + -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), diff --git a/src/spider/client/Job.hpp b/src/spider/client/Job.hpp index a5c7136a6..c79b6d668 100644 --- a/src/spider/client/Job.hpp +++ b/src/spider/client/Job.hpp @@ -223,7 +223,8 @@ class Job { for_n>([&](auto i) { using T = std::tuple_element_t; if (task_index >= output_task_ids.size()) { - throw ConnectionException{fmt::format("Not enough output tasks for job result") + throw ConnectionException{ + fmt::format("Not enough output tasks for job result") }; } core::Task const& task = tasks[task_index]; @@ -266,7 +267,8 @@ class Job { msgpack::object const& obj = handle.get(); std::get(result) = obj.as(); } catch (msgpack::type_error const& e) { - throw ConnectionException{fmt::format("Failed to unpack data: {}", e.what()) + throw ConnectionException{ + fmt::format("Failed to unpack data: {}", e.what()) }; } } @@ -302,7 +304,8 @@ class Job { } err = m_data_storage->get_data(conn, optional_data_id.value(), &data); if (!err.success()) { - throw ConnectionException{fmt::format("Failed to get data: {}", err.description) + throw ConnectionException{ + fmt::format("Failed to get data: {}", err.description) }; } return core::DataImpl::create_data( diff --git a/src/spider/client/TaskContext.cpp b/src/spider/client/TaskContext.cpp index 571d4a697..75dc6474b 100644 --- a/src/spider/client/TaskContext.cpp +++ b/src/spider/client/TaskContext.cpp @@ -15,7 +15,6 @@ #include "Exception.hpp" namespace spider { - auto TaskContext::get_id() const -> boost::uuids::uuid { return m_task_id; } @@ -70,5 +69,4 @@ auto TaskContext::get_jobs() -> std::vector { } return job_ids; } - } // namespace spider diff --git a/src/spider/client/TaskContext.hpp b/src/spider/client/TaskContext.hpp index 0cb0ea40d..db3b9292d 100644 --- a/src/spider/client/TaskContext.hpp +++ b/src/spider/client/TaskContext.hpp @@ -129,8 +129,8 @@ class TaskContext { * @throw spider::ConnectionException */ template - auto - start(TaskFunction const& task, Inputs&&... inputs) -> Job { + auto start(TaskFunction const& task, Inputs&&... inputs) + -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), @@ -187,8 +187,8 @@ class TaskContext { * @throw spider::ConnectionException */ template - auto - start(TaskGraph const& graph, Inputs&&... inputs) -> Job { + auto start(TaskGraph const& graph, Inputs&&... inputs) + -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), diff --git a/src/spider/client/task.hpp b/src/spider/client/task.hpp index 5a48adc5b..fffae7175 100644 --- a/src/spider/client/task.hpp +++ b/src/spider/client/task.hpp @@ -8,7 +8,6 @@ #include "type_utils.hpp" namespace spider { - /** * Concept that represents the input to or output from a Task. * @@ -92,7 +91,6 @@ struct MergeTaskGraphTypes, InputType, Inp template using TaskGraphType = typename MergeTaskGraphTypes, Inputs...>::Type; - } // namespace spider #endif // SPIDER_CLIENT_TASK_HPP diff --git a/src/spider/client/type_utils.hpp b/src/spider/client/type_utils.hpp index 6c87cb12e..c96b7c515 100644 --- a/src/spider/client/type_utils.hpp +++ b/src/spider/client/type_utils.hpp @@ -58,6 +58,5 @@ struct ExtractTemplateParam> { template using ExtractTemplateParamT = typename ExtractTemplateParam::Type; - } // namespace spider #endif // SPIDER_CLIENT_TYPE_UTILS_HPP diff --git a/src/spider/core/Data.hpp b/src/spider/core/Data.hpp index d278794b0..b5a1d556e 100644 --- a/src/spider/core/Data.hpp +++ b/src/spider/core/Data.hpp @@ -42,7 +42,6 @@ class Data { m_id = gen(); } }; - } // namespace spider::core #endif // SPIDER_CORE_DATA_HPP diff --git a/src/spider/core/DataImpl.hpp b/src/spider/core/DataImpl.hpp index faa26c558..fa60e7be0 100644 --- a/src/spider/core/DataImpl.hpp +++ b/src/spider/core/DataImpl.hpp @@ -9,7 +9,6 @@ #include "Data.hpp" namespace spider::core { - class DataImpl { public: template @@ -26,7 +25,6 @@ class DataImpl { return data.get_impl(); } }; - } // namespace spider::core #endif diff --git a/src/spider/core/Driver.hpp b/src/spider/core/Driver.hpp index deb8890c1..178d4b3e5 100644 --- a/src/spider/core/Driver.hpp +++ b/src/spider/core/Driver.hpp @@ -7,7 +7,6 @@ #include namespace spider::core { - class Driver { public: explicit Driver(boost::uuids::uuid const id) : m_id{id} {} @@ -36,7 +35,6 @@ class Scheduler { std::string m_addr; int m_port; }; - } // namespace spider::core #endif // SPIDER_CORE_DRIVER_HPP diff --git a/src/spider/core/Error.hpp b/src/spider/core/Error.hpp index 4d72d2903..7f1e149f2 100644 --- a/src/spider/core/Error.hpp +++ b/src/spider/core/Error.hpp @@ -29,7 +29,6 @@ struct StorageErr { [[nodiscard]] auto success() const -> bool { return StorageErrType::Success == type; } }; - } // namespace spider::core #endif // SPIDER_CORE_ERROR_HPP diff --git a/src/spider/core/JobMetadata.hpp b/src/spider/core/JobMetadata.hpp index 9b27d8879..76bd115c8 100644 --- a/src/spider/core/JobMetadata.hpp +++ b/src/spider/core/JobMetadata.hpp @@ -7,7 +7,6 @@ #include namespace spider::core { - class JobMetadata { public: JobMetadata() = default; @@ -41,7 +40,6 @@ enum class JobStatus : std::uint8_t { Failed, Cancelled }; - } // namespace spider::core #endif // SPIDER_CORE_JOBMETADATA_HPP diff --git a/src/spider/core/Task.hpp b/src/spider/core/Task.hpp index 368432a89..ad67265d5 100644 --- a/src/spider/core/Task.hpp +++ b/src/spider/core/Task.hpp @@ -18,20 +18,22 @@ namespace spider::core { class TaskInput { public: - explicit TaskInput(std::string type) : m_type(std::move(type)) {}; + explicit TaskInput(std::string type) : m_type(std::move(type)) {} TaskInput(boost::uuids::uuid output_task_id, std::uint8_t position, std::string type) : m_task_output({output_task_id, position}), - m_type(std::move(type)) {}; + m_type(std::move(type)) {} + TaskInput(std::string value, std::string type) : m_value(std::move(value)), - m_type(std::move(type)) {}; + m_type(std::move(type)) {} + explicit TaskInput(boost::uuids::uuid data_id) : m_data_id(data_id), - m_type(typeid(spider::core::Data).name()) {}; + m_type(typeid(spider::core::Data).name()) {} - [[nodiscard]] auto get_task_output( - ) const -> std::optional> { + [[nodiscard]] auto get_task_output() const + -> std::optional> { return m_task_output; } @@ -145,8 +147,8 @@ class ScheduleTaskMetadata { auto set_client_id(boost::uuids::uuid const client_id) -> void { m_client_id = client_id; } - auto set_job_creation_time(std::chrono::system_clock::time_point const job_creation_time - ) -> void { + auto set_job_creation_time(std::chrono::system_clock::time_point const job_creation_time) + -> void { m_job_creation_time = job_creation_time; } @@ -222,7 +224,6 @@ class Task { std::vector m_inputs; std::vector m_outputs; }; - } // namespace spider::core #endif // SPIDER_CORE_TASK_HPP diff --git a/src/spider/core/TaskContextImpl.hpp b/src/spider/core/TaskContextImpl.hpp index e8b569c85..d5b672081 100644 --- a/src/spider/core/TaskContextImpl.hpp +++ b/src/spider/core/TaskContextImpl.hpp @@ -26,17 +26,16 @@ class TaskContextImpl { return task_context.m_data_store; } - static auto get_metadata_store(TaskContext const& task_context - ) -> std::shared_ptr { + static auto get_metadata_store(TaskContext const& task_context) + -> std::shared_ptr { return task_context.m_metadata_store; } - static auto get_storage_factory(TaskContext const& task_context - ) -> std::shared_ptr { + static auto get_storage_factory(TaskContext const& task_context) + -> std::shared_ptr { return task_context.m_storage_factory; } }; - } // namespace spider::core #endif diff --git a/src/spider/core/TaskGraph.hpp b/src/spider/core/TaskGraph.hpp index d4f2ba23f..cb49a22fc 100644 --- a/src/spider/core/TaskGraph.hpp +++ b/src/spider/core/TaskGraph.hpp @@ -65,8 +65,8 @@ class TaskGraph { return std::nullopt; } - [[nodiscard]] auto get_child_tasks(boost::uuids::uuid id - ) const -> std::vector { + [[nodiscard]] auto get_child_tasks(boost::uuids::uuid id) const + -> std::vector { std::vector children; for (std::pair const dep : m_dependencies) { if (dep.first == id) { @@ -76,8 +76,8 @@ class TaskGraph { return children; } - [[nodiscard]] auto get_parent_tasks(boost::uuids::uuid id - ) const -> std::vector { + [[nodiscard]] auto get_parent_tasks(boost::uuids::uuid id) const + -> std::vector { std::vector parents; for (std::pair const dep : m_dependencies) { if (dep.second == id) { @@ -88,8 +88,8 @@ class TaskGraph { } // NOLINTBEGIN(misc-include-cleaner) - [[nodiscard]] auto get_tasks( - ) const -> absl::flat_hash_map> const& { + [[nodiscard]] auto get_tasks() const + -> absl::flat_hash_map> const& { return m_tasks; } @@ -107,8 +107,8 @@ class TaskGraph { auto add_output_task(boost::uuids::uuid id) -> void { m_output_tasks.emplace_back(id); } - [[nodiscard]] auto get_dependencies( - ) const -> std::vector> const& { + [[nodiscard]] auto get_dependencies() const + -> std::vector> const& { return m_dependencies; } diff --git a/src/spider/core/TaskGraphImpl.hpp b/src/spider/core/TaskGraphImpl.hpp index cdd29739f..d7d2fbd66 100644 --- a/src/spider/core/TaskGraphImpl.hpp +++ b/src/spider/core/TaskGraphImpl.hpp @@ -28,8 +28,8 @@ class TaskGraphImpl { // NOLINTBEGIN(readability-function-cognitive-complexity, cppcoreguidelines-missing-std-forward) template static auto - bind(TaskFunction const& task_function, - Inputs&&... inputs) -> std::optional { + bind(TaskFunction const& task_function, Inputs&&... inputs) + -> std::optional { std::optional optional_task = create_task(task_function); if (!optional_task.has_value()) { return std::nullopt; @@ -99,9 +99,11 @@ class TaskGraphImpl { fail = true; return; } - input.set_data_id(std::get(std::forward_as_tuple(inputs...)) - .get_impl() - ->get_id()); + input.set_data_id( + std::get(std::forward_as_tuple(inputs...)) + .get_impl() + ->get_id() + ); } else if constexpr (Serializable) { if (input.get_type() != typeid(InputType).name()) { fail = true; @@ -131,8 +133,8 @@ class TaskGraphImpl { // NOLINTEND(readability-function-cognitive-complexity, cppcoreguidelines-missing-std-forward) template - static auto create_task(TaskFunction const& task_function - ) -> std::optional { + static auto create_task(TaskFunction const& task_function) + -> std::optional { // NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast) std::optional const function_name = FunctionNameManager::get_instance().get_function_name( @@ -343,7 +345,6 @@ class TaskGraphImpl { TaskGraph m_graph; }; - } // namespace spider::core #endif // SPIDER_CORE_TASKGRAPHIMPL_HPP diff --git a/src/spider/io/msgpack_message.cpp b/src/spider/io/msgpack_message.cpp index df5417143..ccb449018 100644 --- a/src/spider/io/msgpack_message.cpp +++ b/src/spider/io/msgpack_message.cpp @@ -18,7 +18,6 @@ #include "MsgPack.hpp" // IWYU pragma: keep namespace { - /** * Read the type of ext msgpack message. * @@ -70,11 +69,9 @@ auto read_ext_body_size(std::u8string_view const body_size) -> std::optional bool { msgpack::sbuffer message_buffer; msgpack::packer packer{message_buffer}; @@ -140,9 +137,9 @@ auto receive_message(boost::asio::ip::tcp::socket& socket) -> std::optional(&body_size_vec[1]), body_size_vec.size() - 1); return buffer; } - std::optional const optional_body_size - = read_ext_body_size(std::u8string_view{body_size_vec.data(), body_size_vec.size()} - ); + std::optional const optional_body_size = read_ext_body_size( + std::u8string_view{body_size_vec.data(), body_size_vec.size()} + ); if (false == optional_body_size.has_value()) { return std::nullopt; } @@ -167,8 +164,8 @@ auto receive_message(boost::asio::ip::tcp::socket& socket) -> std::optional socket -) -> boost::asio::awaitable> { +auto receive_message_async(std::reference_wrapper socket) + -> boost::asio::awaitable> { // Read header char8_t header = 0; // Suppress clang-tidy warning inside boost asio @@ -264,5 +261,4 @@ auto receive_message_async(std::reference_wrapper buffer.write(std::bit_cast(&body_vec[1]), body_vec.size() - 1); co_return buffer; } - } // namespace spider::core diff --git a/src/spider/io/msgpack_message.hpp b/src/spider/io/msgpack_message.hpp index d8d992947..9293ca444 100644 --- a/src/spider/io/msgpack_message.hpp +++ b/src/spider/io/msgpack_message.hpp @@ -8,7 +8,6 @@ #include "MsgPack.hpp" // IWYU pragma :keep namespace spider::core { - auto send_message(boost::asio::ip::tcp::socket& socket, msgpack::sbuffer const& buffer) -> bool; auto send_message_async( @@ -18,9 +17,8 @@ auto send_message_async( auto receive_message(boost::asio::ip::tcp::socket& socket) -> std::optional; -auto receive_message_async(std::reference_wrapper socket -) -> boost::asio::awaitable>; - +auto receive_message_async(std::reference_wrapper socket) + -> boost::asio::awaitable>; } // namespace spider::core #endif // SPIDER_CORE_MSGPACKMESSAGE_HPP diff --git a/src/spider/scheduler/FifoPolicy.cpp b/src/spider/scheduler/FifoPolicy.cpp index f1fc33123..f58a5d467 100644 --- a/src/spider/scheduler/FifoPolicy.cpp +++ b/src/spider/scheduler/FifoPolicy.cpp @@ -16,7 +16,6 @@ #include "../storage/StorageConnection.hpp" namespace spider::scheduler { - FifoPolicy::FifoPolicy( std::shared_ptr const& metadata_store, std::shared_ptr const& data_store, @@ -26,10 +25,9 @@ FifoPolicy::FifoPolicy( m_data_store{data_store}, m_conn{conn} {} -auto FifoPolicy::schedule_next( - boost::uuids::uuid const /*worker_id*/, - std::string const& worker_addr -) -> std::optional { +auto +FifoPolicy::schedule_next(boost::uuids::uuid const /*worker_id*/, std::string const& worker_addr) + -> std::optional { std::optional const next_task = pop_next_task(worker_addr); if (next_task.has_value()) { return next_task; @@ -42,8 +40,8 @@ auto FifoPolicy::schedule_next( return pop_next_task(worker_addr); } -auto FifoPolicy::pop_next_task(std::string const& worker_addr -) -> std::optional { +auto FifoPolicy::pop_next_task(std::string const& worker_addr) + -> std::optional { auto const reverse_begin = std::reverse_iterator(m_tasks.end()); auto const reverse_end = std::reverse_iterator(m_tasks.begin()); auto const it @@ -76,5 +74,4 @@ auto FifoPolicy::fetch_tasks() -> void { } ); } - } // namespace spider::scheduler diff --git a/src/spider/scheduler/FifoPolicy.hpp b/src/spider/scheduler/FifoPolicy.hpp index b62592460..924c863d6 100644 --- a/src/spider/scheduler/FifoPolicy.hpp +++ b/src/spider/scheduler/FifoPolicy.hpp @@ -15,7 +15,6 @@ #include "SchedulerPolicy.hpp" namespace spider::scheduler { - class FifoPolicy final : public SchedulerPolicy { public: FifoPolicy( @@ -38,7 +37,6 @@ class FifoPolicy final : public SchedulerPolicy { std::vector m_tasks; }; - } // namespace spider::scheduler #endif // SPIDER_SCHEDULER_FIFOPOLICY_HPP diff --git a/src/spider/scheduler/SchedulerMessage.hpp b/src/spider/scheduler/SchedulerMessage.hpp index b3d1e07a0..1367271ea 100644 --- a/src/spider/scheduler/SchedulerMessage.hpp +++ b/src/spider/scheduler/SchedulerMessage.hpp @@ -11,7 +11,6 @@ #include "../io/Serializer.hpp" // IWYU pragma: keep namespace spider::scheduler { - class ScheduleTaskRequest { public: /** @@ -68,7 +67,6 @@ class ScheduleTaskResponse { private: std::optional m_task_id = std::nullopt; }; - } // namespace spider::scheduler #endif // SPIDER_SCHEDULER_SCHEDULERMESSAGE_HPP diff --git a/src/spider/scheduler/SchedulerPolicy.hpp b/src/spider/scheduler/SchedulerPolicy.hpp index feb6a7936..17510240b 100644 --- a/src/spider/scheduler/SchedulerPolicy.hpp +++ b/src/spider/scheduler/SchedulerPolicy.hpp @@ -17,9 +17,9 @@ class SchedulerPolicy { virtual ~SchedulerPolicy() = default; virtual auto schedule_next(boost::uuids::uuid worker_id, std::string const& worker_addr) - -> std::optional = 0; + -> std::optional + = 0; }; - } // namespace spider::scheduler #endif // SPIDER_SCHEDULER_SCHEDULERPOLICY_HPP diff --git a/src/spider/scheduler/SchedulerServer.cpp b/src/spider/scheduler/SchedulerServer.cpp index dcc0111ee..98913ab00 100644 --- a/src/spider/scheduler/SchedulerServer.cpp +++ b/src/spider/scheduler/SchedulerServer.cpp @@ -24,7 +24,6 @@ #include "SchedulerPolicy.hpp" namespace spider::scheduler { - SchedulerServer::SchedulerServer( unsigned short const port, std::shared_ptr policy, @@ -115,8 +114,8 @@ auto deserialize_message(msgpack::sbuffer const& buffer) -> std::optional boost::asio::awaitable { +auto SchedulerServer::process_message(boost::asio::ip::tcp::socket socket) + -> boost::asio::awaitable { // NOLINTBEGIN(clang-analyzer-core.CallAndMessage) std::optional const& optional_message_buffer = co_await core::receive_message_async(socket); @@ -174,5 +173,4 @@ auto SchedulerServer::process_message(boost::asio::ip::tcp::socket socket } co_return; } - } // namespace spider::scheduler diff --git a/src/spider/scheduler/SchedulerServer.hpp b/src/spider/scheduler/SchedulerServer.hpp index 73dae91da..3aa8cd318 100644 --- a/src/spider/scheduler/SchedulerServer.hpp +++ b/src/spider/scheduler/SchedulerServer.hpp @@ -13,7 +13,6 @@ #include "SchedulerPolicy.hpp" namespace spider::scheduler { - class SchedulerServer { public: // Delete copy & move constructor and assignment operator @@ -55,7 +54,6 @@ class SchedulerServer { core::StopToken& m_stop_token; }; - } // namespace spider::scheduler #endif // SPIDER_SCHEDULER_SCHEDULERSERVER_HPP diff --git a/src/spider/scheduler/scheduler.cpp b/src/spider/scheduler/scheduler.cpp index af00be1f2..d47e1c17d 100644 --- a/src/spider/scheduler/scheduler.cpp +++ b/src/spider/scheduler/scheduler.cpp @@ -1,4 +1,3 @@ - #include #include #include @@ -91,8 +90,9 @@ auto heartbeat_loop( fail_count++; continue; } - auto conn = std::move(std::get>(conn_result - )); + auto conn = std::move( + std::get>(conn_result) + ); spider::core::StorageErr const err = metadata_store->update_heartbeat(*conn, scheduler.get_id()); @@ -128,8 +128,9 @@ auto cleanup_loop( ); continue; } - auto conn = std::move(std::get>(conn_result - )); + auto conn = std::move( + std::get>(conn_result) + ); spider::core::StorageErr err = metadata_store->set_scheduler_state(*conn, scheduler.get_id(), "gc"); diff --git a/src/spider/storage/DataStorage.hpp b/src/spider/storage/DataStorage.hpp index 68d47b821..041f04871 100644 --- a/src/spider/storage/DataStorage.hpp +++ b/src/spider/storage/DataStorage.hpp @@ -22,58 +22,60 @@ class DataStorage { virtual auto initialize(StorageConnection& conn) -> StorageErr = 0; - virtual auto add_driver_data( - StorageConnection& conn, - boost::uuids::uuid driver_id, - Data const& data - ) -> StorageErr = 0; - virtual auto add_task_data( - StorageConnection& conn, - boost::uuids::uuid task_id, - Data const& data - ) -> StorageErr = 0; + virtual auto + add_driver_data(StorageConnection& conn, boost::uuids::uuid driver_id, Data const& data) + -> StorageErr + = 0; + virtual auto + add_task_data(StorageConnection& conn, boost::uuids::uuid task_id, Data const& data) + -> StorageErr + = 0; virtual auto get_data(StorageConnection& conn, boost::uuids::uuid id, Data* data) -> StorageErr - = 0; + = 0; virtual auto set_data_locality(StorageConnection& conn, Data const& data) -> StorageErr = 0; virtual auto remove_data(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; - virtual auto add_task_reference( - StorageConnection& conn, - boost::uuids::uuid id, - boost::uuids::uuid task_id - ) -> StorageErr = 0; + virtual auto + add_task_reference(StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid task_id) + -> StorageErr + = 0; virtual auto remove_task_reference( StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid task_id - ) -> StorageErr = 0; + ) -> StorageErr + = 0; virtual auto add_driver_reference( StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid driver_id - ) -> StorageErr = 0; + ) -> StorageErr + = 0; virtual auto remove_driver_reference( StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid driver_id - ) -> StorageErr = 0; + ) -> StorageErr + = 0; virtual auto remove_dangling_data(StorageConnection& conn) -> StorageErr = 0; virtual auto add_client_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr - = 0; + = 0; virtual auto add_task_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr - = 0; + = 0; virtual auto get_client_kv_data( StorageConnection& conn, boost::uuids::uuid const& client_id, std::string const& key, std::string* value - ) -> StorageErr = 0; + ) -> StorageErr + = 0; virtual auto get_task_kv_data( StorageConnection& conn, boost::uuids::uuid const& task_id, std::string const& key, std::string* value - ) -> StorageErr = 0; + ) -> StorageErr + = 0; }; } // namespace spider::core diff --git a/src/spider/storage/MetadataStorage.hpp b/src/spider/storage/MetadataStorage.hpp index 24f3ec7b8..b5dbe5ea5 100644 --- a/src/spider/storage/MetadataStorage.hpp +++ b/src/spider/storage/MetadataStorage.hpp @@ -28,111 +28,114 @@ class MetadataStorage { virtual auto add_driver(StorageConnection& conn, Driver const& driver) -> StorageErr = 0; virtual auto add_scheduler(StorageConnection& conn, Scheduler const& scheduler) -> StorageErr - = 0; + = 0; virtual auto get_active_scheduler(StorageConnection& conn, std::vector* schedulers) - -> StorageErr = 0; + -> StorageErr + = 0; - virtual auto add_job( - StorageConnection& conn, + virtual auto + add_job(StorageConnection& conn, boost::uuids::uuid job_id, boost::uuids::uuid client_id, - TaskGraph const& task_graph - ) -> StorageErr = 0; + TaskGraph const& task_graph) -> StorageErr + = 0; virtual auto add_job_batch( StorageConnection& conn, JobSubmissionBatch& batch, boost::uuids::uuid job_id, boost::uuids::uuid client_id, TaskGraph const& task_graph - ) -> StorageErr = 0; + ) -> StorageErr + = 0; virtual auto get_job_metadata(StorageConnection& conn, boost::uuids::uuid id, JobMetadata* job) - -> StorageErr = 0; + -> StorageErr + = 0; virtual auto get_job_complete(StorageConnection& conn, boost::uuids::uuid id, bool* complete) - -> StorageErr = 0; + -> StorageErr + = 0; virtual auto get_job_status(StorageConnection& conn, boost::uuids::uuid id, JobStatus* status) - -> StorageErr = 0; + -> StorageErr + = 0; virtual auto get_job_output_tasks( StorageConnection& conn, boost::uuids::uuid id, std::vector* task_ids - ) -> StorageErr = 0; - virtual auto get_task_graph( - StorageConnection& conn, - boost::uuids::uuid id, - TaskGraph* task_graph - ) -> StorageErr = 0; + ) -> StorageErr + = 0; + virtual auto + get_task_graph(StorageConnection& conn, boost::uuids::uuid id, TaskGraph* task_graph) + -> StorageErr + = 0; virtual auto get_jobs_by_client_id( StorageConnection& conn, boost::uuids::uuid client_id, std::vector* job_ids - ) -> StorageErr = 0; + ) -> StorageErr + = 0; virtual auto remove_job(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; virtual auto reset_job(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; virtual auto add_child(StorageConnection& conn, boost::uuids::uuid parent_id, Task const& child) - -> StorageErr = 0; + -> StorageErr + = 0; virtual auto get_task(StorageConnection& conn, boost::uuids::uuid id, Task* task) -> StorageErr - = 0; - virtual auto get_task_job_id( - StorageConnection& conn, - boost::uuids::uuid id, - boost::uuids::uuid* job_id - ) -> StorageErr = 0; + = 0; + virtual auto + get_task_job_id(StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid* job_id) + -> StorageErr + = 0; virtual auto get_ready_tasks(StorageConnection& conn, std::vector* tasks) - -> StorageErr = 0; + -> StorageErr + = 0; virtual auto set_task_state(StorageConnection& conn, boost::uuids::uuid id, TaskState state) - -> StorageErr = 0; + -> StorageErr + = 0; virtual auto set_task_running(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; - virtual auto - add_task_instance(StorageConnection& conn, TaskInstance const& instance) -> StorageErr = 0; + virtual auto add_task_instance(StorageConnection& conn, TaskInstance const& instance) + -> StorageErr + = 0; // Set task state and add new task instance if task is ready or all instances timed out - virtual auto - create_task_instance(StorageConnection& conn, TaskInstance const& instance) -> StorageErr = 0; + virtual auto create_task_instance(StorageConnection& conn, TaskInstance const& instance) + -> StorageErr + = 0; virtual auto task_finish( StorageConnection& conn, TaskInstance const& instance, std::vector const& outputs - ) -> StorageErr = 0; - virtual auto task_fail( - StorageConnection& conn, - TaskInstance const& instance, - std::string const& error - ) -> StorageErr = 0; + ) -> StorageErr + = 0; + virtual auto + task_fail(StorageConnection& conn, TaskInstance const& instance, std::string const& error) + -> StorageErr + = 0; virtual auto get_task_timeout(StorageConnection& conn, std::vector* tasks) - -> StorageErr = 0; - virtual auto get_child_tasks( - StorageConnection& conn, - boost::uuids::uuid id, - std::vector* children - ) -> StorageErr = 0; - virtual auto get_parent_tasks( - StorageConnection& conn, - boost::uuids::uuid id, - std::vector* tasks - ) -> StorageErr = 0; + -> StorageErr + = 0; + virtual auto + get_child_tasks(StorageConnection& conn, boost::uuids::uuid id, std::vector* children) + -> StorageErr + = 0; + virtual auto + get_parent_tasks(StorageConnection& conn, boost::uuids::uuid id, std::vector* tasks) + -> StorageErr + = 0; virtual auto update_heartbeat(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; - virtual auto heartbeat_timeout( - StorageConnection& conn, - double timeout, - std::vector* ids - ) -> StorageErr = 0; - virtual auto get_scheduler_state( - StorageConnection& conn, - boost::uuids::uuid id, - std::string* state - ) -> StorageErr = 0; - virtual auto get_scheduler_addr( - StorageConnection& conn, - boost::uuids::uuid id, - std::string* addr, - int* port - ) -> StorageErr = 0; - virtual auto set_scheduler_state( - StorageConnection& conn, - boost::uuids::uuid id, - std::string const& state - ) -> StorageErr = 0; + virtual auto + heartbeat_timeout(StorageConnection& conn, double timeout, std::vector* ids) + -> StorageErr + = 0; + virtual auto + get_scheduler_state(StorageConnection& conn, boost::uuids::uuid id, std::string* state) + -> StorageErr + = 0; + virtual auto + get_scheduler_addr(StorageConnection& conn, boost::uuids::uuid id, std::string* addr, int* port) + -> StorageErr + = 0; + virtual auto + set_scheduler_state(StorageConnection& conn, boost::uuids::uuid id, std::string const& state) + -> StorageErr + = 0; }; - } // namespace spider::core #endif // SPIDER_STORAGE_METADATASTORAGE_HPP diff --git a/src/spider/storage/StorageConnection.hpp b/src/spider/storage/StorageConnection.hpp index c1ae7f9d9..de39dc912 100644 --- a/src/spider/storage/StorageConnection.hpp +++ b/src/spider/storage/StorageConnection.hpp @@ -2,7 +2,6 @@ #define SPIDER_STORAGE_STORAGECONNECTION_HPP namespace spider::core { - class StorageConnection { public: StorageConnection() = default; @@ -12,7 +11,6 @@ class StorageConnection { auto operator=(StorageConnection&&) -> StorageConnection& = default; virtual ~StorageConnection() = default; }; - } // namespace spider::core #endif diff --git a/src/spider/storage/StorageFactory.hpp b/src/spider/storage/StorageFactory.hpp index 973c8e19e..4cfca3c28 100644 --- a/src/spider/storage/StorageFactory.hpp +++ b/src/spider/storage/StorageFactory.hpp @@ -15,10 +15,12 @@ class StorageFactory { public: virtual auto provide_data_storage() -> std::unique_ptr = 0; virtual auto provide_metadata_storage() -> std::unique_ptr = 0; - virtual auto provide_storage_connection( - ) -> std::variant, StorageErr> = 0; - virtual auto - provide_job_submission_batch(StorageConnection&) -> std::unique_ptr = 0; + virtual auto provide_storage_connection() + -> std::variant, StorageErr> + = 0; + virtual auto provide_job_submission_batch(StorageConnection&) + -> std::unique_ptr + = 0; StorageFactory() = default; StorageFactory(StorageFactory const&) = default; @@ -27,7 +29,6 @@ class StorageFactory { auto operator=(StorageFactory&&) -> StorageFactory& = default; virtual ~StorageFactory() = default; }; - } // namespace spider::core #endif diff --git a/src/spider/storage/mysql/MySqlConnection.cpp b/src/spider/storage/mysql/MySqlConnection.cpp index 7f6def7ff..0ed5fb04f 100644 --- a/src/spider/storage/mysql/MySqlConnection.cpp +++ b/src/spider/storage/mysql/MySqlConnection.cpp @@ -16,9 +16,8 @@ #include "../StorageConnection.hpp" namespace spider::core { - -auto MySqlConnection::create(std::string const& url -) -> std::variant, StorageErr> { +auto MySqlConnection::create(std::string const& url) + -> std::variant, StorageErr> { // Validate jdbc url std::regex const url_regex(R"(jdbc:mariadb://[^?]+(\?user=([^&]*)(&password=([^&]*))?)?)"); std::smatch match; @@ -53,5 +52,4 @@ auto MySqlConnection::operator*() const -> sql::Connection& { auto MySqlConnection::operator->() const -> sql::Connection* { return &*m_connection; } - } // namespace spider::core diff --git a/src/spider/storage/mysql/MySqlConnection.hpp b/src/spider/storage/mysql/MySqlConnection.hpp index 4c707b9a0..6c14f3670 100644 --- a/src/spider/storage/mysql/MySqlConnection.hpp +++ b/src/spider/storage/mysql/MySqlConnection.hpp @@ -12,7 +12,6 @@ #include "../StorageConnection.hpp" namespace spider::core { - // Forward declaration for friend class class MySqlStorageFactory; @@ -32,16 +31,16 @@ class MySqlConnection : public StorageConnection { auto operator->() const -> sql::Connection*; private: - static auto create(std::string const& url - ) -> std::variant, StorageErr>; + static auto create(std::string const& url) + -> std::variant, StorageErr>; explicit MySqlConnection(std::unique_ptr conn) - : m_connection{std::move(conn)} {}; + : m_connection{std::move(conn)} {} + std::unique_ptr m_connection; friend class MySqlStorageFactory; }; - } // namespace spider::core #endif diff --git a/src/spider/storage/mysql/MySqlJobSubmissionBatch.cpp b/src/spider/storage/mysql/MySqlJobSubmissionBatch.cpp index 8c54c713f..84014b08c 100644 --- a/src/spider/storage/mysql/MySqlJobSubmissionBatch.cpp +++ b/src/spider/storage/mysql/MySqlJobSubmissionBatch.cpp @@ -10,7 +10,6 @@ #include "MySqlConnection.hpp" namespace spider::core { - // NOLINTBEGIN(cppcoreguidelines-pro-type-static-cast-downcast) MySqlJobSubmissionBatch::MySqlJobSubmissionBatch(StorageConnection& conn) : m_job_stmt{static_cast(conn)->prepareStatement(mysql::cInsertJob)}, @@ -59,5 +58,4 @@ auto MySqlJobSubmissionBatch::submit_batch(StorageConnection& conn) -> StorageEr static_cast(conn)->commit(); return StorageErr{}; } - } // namespace spider::core diff --git a/src/spider/storage/mysql/MySqlJobSubmissionBatch.hpp b/src/spider/storage/mysql/MySqlJobSubmissionBatch.hpp index 02a7929fb..532457da7 100644 --- a/src/spider/storage/mysql/MySqlJobSubmissionBatch.hpp +++ b/src/spider/storage/mysql/MySqlJobSubmissionBatch.hpp @@ -10,7 +10,6 @@ #include "../StorageConnection.hpp" namespace spider::core { - // Forward declaration for friend class class MySqlStorageFactory; diff --git a/src/spider/storage/mysql/MySqlStorage.cpp b/src/spider/storage/mysql/MySqlStorage.cpp index 776616628..c1ef69d13 100644 --- a/src/spider/storage/mysql/MySqlStorage.cpp +++ b/src/spider/storage/mysql/MySqlStorage.cpp @@ -56,7 +56,6 @@ enum MariadbErr : uint16_t { namespace spider::core { namespace { - auto uuid_get_bytes(boost::uuids::uuid const& id) -> sql::bytes { // NOLINTBEGIN(cppcoreguidelines-pro-type-cstyle-cast) return {(char const*)id.data(), id.size()}; @@ -227,8 +226,9 @@ void MySqlMetadataStorage::add_task( std::optional const& state ) { // Add task - std::unique_ptr task_statement(conn->prepareStatement(mysql::cInsertTask - )); + std::unique_ptr task_statement( + conn->prepareStatement(mysql::cInsertTask) + ); sql::bytes task_id_bytes = uuid_get_bytes(task.get_id()); // NOLINTBEGIN(cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) task_statement->setBytes(1, &task_id_bytes); @@ -626,7 +626,6 @@ auto MySqlMetadataStorage::add_job_batch( // NOLINTEND(readability-function-cognitive-complexity) namespace { - auto fetch_task(std::unique_ptr const& res) -> Task { boost::uuids::uuid const id = read_id(res->getBinaryStream("id")); std::string const function_name = get_sql_string(res->getString("func_name")); @@ -738,10 +737,10 @@ auto MySqlMetadataStorage::fetch_full_task( } // Get task outputs - std::unique_ptr output_statement{ - conn->prepareStatement("SELECT `task_id`, `position`, `type`, `value`, `data_id` FROM " - "`task_outputs` WHERE `task_id` = ? ORDER BY `position`") - }; + std::unique_ptr output_statement{conn->prepareStatement( + "SELECT `task_id`, `position`, `type`, `value`, `data_id` FROM " + "`task_outputs` WHERE `task_id` = ? ORDER BY `position`" + )}; output_statement->setBytes(1, &id_bytes); std::unique_ptr const output_res{output_statement->executeQuery()}; while (output_res->next()) { @@ -870,9 +869,8 @@ auto MySqlMetadataStorage::get_task_graph( } // namespace spider::core namespace { - -auto parse_timestamp(std::string const& timestamp -) -> std::optional { +auto parse_timestamp(std::string const& timestamp) + -> std::optional { std::tm time_date{}; std::stringstream ss{timestamp}; ss >> std::get_time(&time_date, "%Y-%m-%d %H:%M:%S"); @@ -881,11 +879,9 @@ auto parse_timestamp(std::string const& timestamp } return std::chrono::system_clock::from_time_t(std::mktime(&time_date)); } - } // namespace namespace spider::core { - auto MySqlMetadataStorage::get_job_metadata( StorageConnection& conn, boost::uuids::uuid id, @@ -1413,10 +1409,9 @@ auto MySqlMetadataStorage::add_task_instance(StorageConnection& conn, TaskInstan return StorageErr{}; } -auto MySqlMetadataStorage::create_task_instance( - StorageConnection& conn, - TaskInstance const& instance -) -> StorageErr { +auto +MySqlMetadataStorage::create_task_instance(StorageConnection& conn, TaskInstance const& instance) + -> StorageErr { try { // Check the state of the task std::unique_ptr ready_statement( @@ -2250,9 +2245,11 @@ auto MySqlDataStorage::remove_dangling_data(StorageConnection& conn) -> StorageE std::unique_ptr statement{ static_cast(conn)->createStatement() }; - statement->execute("DELETE FROM `data` WHERE `id` NOT IN (SELECT driver_ref.`id` FROM " - "`data_ref_driver` driver_ref) AND `id` NOT IN (SELECT task_ref.`id` " - "FROM `data_ref_task` task_ref)"); + statement->execute( + "DELETE FROM `data` WHERE `id` NOT IN (SELECT driver_ref.`id` FROM " + "`data_ref_driver` driver_ref) AND `id` NOT IN (SELECT task_ref.`id` " + "FROM `data_ref_task` task_ref)" + ); } catch (sql::SQLException& e) { static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; @@ -2386,5 +2383,4 @@ auto MySqlDataStorage::get_task_kv_data( } // NOLINTEND(cppcoreguidelines-pro-type-static-cast-downcast) - } // namespace spider::core diff --git a/src/spider/storage/mysql/MySqlStorage.hpp b/src/spider/storage/mysql/MySqlStorage.hpp index a563b54c5..3a2a58caf 100644 --- a/src/spider/storage/mysql/MySqlStorage.hpp +++ b/src/spider/storage/mysql/MySqlStorage.hpp @@ -25,7 +25,6 @@ #include "MySqlJobSubmissionBatch.hpp" namespace spider::core { - // Forward declaration for friend class class MySqlStorageFactory; @@ -76,8 +75,8 @@ class MySqlMetadataStorage : public MetadataStorage { auto reset_job(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; auto add_child(StorageConnection& conn, boost::uuids::uuid parent_id, Task const& child) -> StorageErr override; - auto - get_task(StorageConnection& conn, boost::uuids::uuid id, Task* task) -> StorageErr override; + auto get_task(StorageConnection& conn, boost::uuids::uuid id, Task* task) + -> StorageErr override; auto get_task_job_id(StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid* job_id) -> StorageErr override; auto get_ready_tasks(StorageConnection& conn, std::vector* tasks) @@ -85,8 +84,8 @@ class MySqlMetadataStorage : public MetadataStorage { auto set_task_state(StorageConnection& conn, boost::uuids::uuid id, TaskState state) -> StorageErr override; auto set_task_running(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; - auto - add_task_instance(StorageConnection& conn, TaskInstance const& instance) -> StorageErr override; + auto add_task_instance(StorageConnection& conn, TaskInstance const& instance) + -> StorageErr override; auto create_task_instance(StorageConnection& conn, TaskInstance const& instance) -> StorageErr override; auto task_finish( @@ -98,32 +97,23 @@ class MySqlMetadataStorage : public MetadataStorage { -> StorageErr override; auto get_task_timeout(StorageConnection& conn, std::vector* tasks) -> StorageErr override; - auto get_child_tasks( - StorageConnection& conn, - boost::uuids::uuid id, - std::vector* children - ) -> StorageErr override; + auto + get_child_tasks(StorageConnection& conn, boost::uuids::uuid id, std::vector* children) + -> StorageErr override; auto get_parent_tasks(StorageConnection& conn, boost::uuids::uuid id, std::vector* tasks) -> StorageErr override; auto update_heartbeat(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; - auto heartbeat_timeout( - StorageConnection& conn, - double timeout, - std::vector* ids - ) -> StorageErr override; + auto + heartbeat_timeout(StorageConnection& conn, double timeout, std::vector* ids) + -> StorageErr override; auto get_scheduler_state(StorageConnection& conn, boost::uuids::uuid id, std::string* state) -> StorageErr override; - auto get_scheduler_addr( - StorageConnection& conn, - boost::uuids::uuid id, - std::string* addr, - int* port - ) -> StorageErr override; - auto set_scheduler_state( - StorageConnection& conn, - boost::uuids::uuid id, - std::string const& state - ) -> StorageErr override; + auto + get_scheduler_addr(StorageConnection& conn, boost::uuids::uuid id, std::string* addr, int* port) + -> StorageErr override; + auto + set_scheduler_state(StorageConnection& conn, boost::uuids::uuid id, std::string const& state) + -> StorageErr override; private: MySqlMetadataStorage() = default; @@ -140,8 +130,8 @@ class MySqlMetadataStorage : public MetadataStorage { Task const& task, std::optional const& state ); - static auto - fetch_full_task(MySqlConnection& conn, std::unique_ptr const& res) -> Task; + static auto fetch_full_task(MySqlConnection& conn, std::unique_ptr const& res) + -> Task; friend class MySqlStorageFactory; }; @@ -158,15 +148,13 @@ class MySqlDataStorage : public DataStorage { -> StorageErr override; auto add_task_data(StorageConnection& conn, boost::uuids::uuid task_id, Data const& data) -> StorageErr override; - auto - get_data(StorageConnection& conn, boost::uuids::uuid id, Data* data) -> StorageErr override; + auto get_data(StorageConnection& conn, boost::uuids::uuid id, Data* data) + -> StorageErr override; auto set_data_locality(StorageConnection& conn, Data const& data) -> StorageErr override; auto remove_data(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; - auto add_task_reference( - StorageConnection& conn, - boost::uuids::uuid id, - boost::uuids::uuid task_id - ) -> StorageErr override; + auto + add_task_reference(StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid task_id) + -> StorageErr override; auto remove_task_reference( StorageConnection& conn, boost::uuids::uuid id, @@ -184,8 +172,8 @@ class MySqlDataStorage : public DataStorage { ) -> StorageErr override; auto remove_dangling_data(StorageConnection& conn) -> StorageErr override; - auto - add_client_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr override; + auto add_client_kv_data(StorageConnection& conn, KeyValueData const& data) + -> StorageErr override; auto add_task_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr override; auto get_client_kv_data( StorageConnection& conn, diff --git a/src/spider/storage/mysql/MySqlStorageFactory.cpp b/src/spider/storage/mysql/MySqlStorageFactory.cpp index 598eaf5f3..30c9061b6 100644 --- a/src/spider/storage/mysql/MySqlStorageFactory.cpp +++ b/src/spider/storage/mysql/MySqlStorageFactory.cpp @@ -15,7 +15,6 @@ #include "MySqlStorage.hpp" namespace spider::core { - MySqlStorageFactory::MySqlStorageFactory(std::string url) : m_url{std::move(url)} {} auto MySqlStorageFactory::provide_data_storage() -> std::unique_ptr { @@ -26,8 +25,8 @@ auto MySqlStorageFactory::provide_metadata_storage() -> std::unique_ptr(new MySqlMetadataStorage()); } -auto MySqlStorageFactory::provide_storage_connection( -) -> std::variant, StorageErr> { +auto MySqlStorageFactory::provide_storage_connection() + -> std::variant, StorageErr> { std::variant, StorageErr> connection = MySqlConnection::create(m_url); if (std::holds_alternative(connection)) { @@ -36,8 +35,8 @@ auto MySqlStorageFactory::provide_storage_connection( return std::move(std::get>(connection)); } -auto MySqlStorageFactory::provide_job_submission_batch(StorageConnection& connection -) -> std::unique_ptr { +auto MySqlStorageFactory::provide_job_submission_batch(StorageConnection& connection) + -> std::unique_ptr { return std::unique_ptr(new MySqlJobSubmissionBatch(connection)); } } // namespace spider::core diff --git a/src/spider/storage/mysql/MySqlStorageFactory.hpp b/src/spider/storage/mysql/MySqlStorageFactory.hpp index daf02a8ed..fe00d3fa5 100644 --- a/src/spider/storage/mysql/MySqlStorageFactory.hpp +++ b/src/spider/storage/mysql/MySqlStorageFactory.hpp @@ -19,8 +19,8 @@ class MySqlStorageFactory : public StorageFactory { auto provide_data_storage() -> std::unique_ptr override; auto provide_metadata_storage() -> std::unique_ptr override; - auto provide_storage_connection( - ) -> std::variant, StorageErr> override; + auto provide_storage_connection() + -> std::variant, StorageErr> override; auto provide_job_submission_batch(StorageConnection&) -> std::unique_ptr override; diff --git a/src/spider/utils/LruCache.hpp b/src/spider/utils/LruCache.hpp index 3eff49162..033b21948 100644 --- a/src/spider/utils/LruCache.hpp +++ b/src/spider/utils/LruCache.hpp @@ -8,7 +8,6 @@ #include namespace spider::core { - namespace utils { constexpr size_t cDefaultCacheSize = 100; } // namespace utils @@ -63,7 +62,6 @@ class LruCache { std::list> m_list; absl::flat_hash_map>::iterator> m_map; }; - } // namespace spider::core #endif // SPIDER_UTILS_TIMEDCACHE_HPP diff --git a/src/spider/worker/DllLoader.cpp b/src/spider/worker/DllLoader.cpp index d6c789eb1..59e26b1bb 100644 --- a/src/spider/worker/DllLoader.cpp +++ b/src/spider/worker/DllLoader.cpp @@ -12,7 +12,6 @@ #include "../worker/FunctionNameManager.hpp" namespace spider::worker { - auto DllLoader::load_dll(std::string const& path_str) -> bool { std::filesystem::path const dll_path(path_str); @@ -63,5 +62,4 @@ auto DllLoader::load_dll(std::string const& path_str) -> bool { return true; } - } // namespace spider::worker diff --git a/src/spider/worker/DllLoader.hpp b/src/spider/worker/DllLoader.hpp index 63c5b6da8..eea72e866 100644 --- a/src/spider/worker/DllLoader.hpp +++ b/src/spider/worker/DllLoader.hpp @@ -7,7 +7,6 @@ #include namespace spider::worker { - class DllLoader { public: static auto get_instance() -> DllLoader& { @@ -22,7 +21,6 @@ class DllLoader { private: absl::flat_hash_map m_libraries; }; - } // namespace spider::worker #endif // SPIDER_WORKER_DLLLOADER_HPP diff --git a/src/spider/worker/FunctionManager.cpp b/src/spider/worker/FunctionManager.cpp index 5cfdadeed..b7d31da39 100644 --- a/src/spider/worker/FunctionManager.cpp +++ b/src/spider/worker/FunctionManager.cpp @@ -14,9 +14,8 @@ #include "TaskExecutorMessage.hpp" namespace spider::core { - -auto response_get_error(msgpack::sbuffer const& buffer -) -> std::optional> { +auto response_get_error(msgpack::sbuffer const& buffer) + -> std::optional> { // NOLINTBEGIN(cppcoreguidelines-pro-type-union-access,cppcoreguidelines-pro-bounds-pointer-arithmetic) try { msgpack::object_handle const handle = msgpack::unpack(buffer.data(), buffer.size()); @@ -64,8 +63,8 @@ void create_error_buffer( packer.pack(message); } -auto response_get_result_buffers(msgpack::sbuffer const& buffer -) -> std::optional> { +auto response_get_result_buffers(msgpack::sbuffer const& buffer) + -> std::optional> { // NOLINTBEGIN(cppcoreguidelines-pro-type-union-access,cppcoreguidelines-pro-bounds-pointer-arithmetic) try { std::vector result_buffers; @@ -113,7 +112,6 @@ auto FunctionManager::get_function(std::string const& name) const -> Function co } return nullptr; } - } // namespace spider::core // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/src/spider/worker/FunctionManager.hpp b/src/spider/worker/FunctionManager.hpp index 92088cb33..6a8163f92 100644 --- a/src/spider/worker/FunctionManager.hpp +++ b/src/spider/worker/FunctionManager.hpp @@ -87,9 +87,8 @@ enum class FunctionInvokeError : std::uint8_t { MSGPACK_ADD_ENUM(spider::core::FunctionInvokeError); namespace spider::core { - -auto response_get_error(msgpack::sbuffer const& buffer -) -> std::optional>; +auto response_get_error(msgpack::sbuffer const& buffer) + -> std::optional>; auto create_error_response(FunctionInvokeError error, std::string const& message) -> msgpack::sbuffer; @@ -162,8 +161,8 @@ auto response_get_result(msgpack::sbuffer const& buffer) -> std::optional std::optional>; +auto response_get_result_buffers(msgpack::sbuffer const& buffer) + -> std::optional>; template auto create_result_response(T const& t) -> msgpack::sbuffer { @@ -218,8 +217,8 @@ auto create_args_request(Args&&... args) -> msgpack::sbuffer { return buffer; } -inline auto create_args_request(std::vector const& args_buffers -) -> msgpack::sbuffer { +inline auto create_args_request(std::vector const& args_buffers) + -> msgpack::sbuffer { msgpack::sbuffer buffer; msgpack::packer packer{buffer}; packer.pack_array(2); @@ -236,8 +235,8 @@ inline auto create_args_request(std::vector const& args_buffer template class FunctionInvoker { public: - static auto - apply(F const& function, TaskContext& context, ArgsBuffer const& args_buffer) -> ResultBuffer { + static auto apply(F const& function, TaskContext& context, ArgsBuffer const& args_buffer) + -> ResultBuffer { // NOLINTBEGIN(cppcoreguidelines-pro-type-union-access,cppcoreguidelines-pro-bounds-pointer-arithmetic) using ArgsTuple = signature::args_t; using ReturnType = signature::ret_t; diff --git a/src/spider/worker/FunctionNameManager.cpp b/src/spider/worker/FunctionNameManager.cpp index 007363cf1..9d356ef32 100644 --- a/src/spider/worker/FunctionNameManager.cpp +++ b/src/spider/worker/FunctionNameManager.cpp @@ -6,7 +6,6 @@ #include namespace spider::core { - auto FunctionNameManager::get_instance() -> FunctionNameManager& { static FunctionNameManager instance; return instance; @@ -18,7 +17,6 @@ auto FunctionNameManager::get_function_name(void const* ptr) const -> std::optio } return std::nullopt; } - } // namespace spider::core // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/src/spider/worker/FunctionNameManager.hpp b/src/spider/worker/FunctionNameManager.hpp index 6ae4405b5..867919cbe 100644 --- a/src/spider/worker/FunctionNameManager.hpp +++ b/src/spider/worker/FunctionNameManager.hpp @@ -17,7 +17,6 @@ = spider::core::FunctionNameManager::get_instance().register_function(#func, func); namespace spider::core { - using FunctionNameMap = absl::flat_hash_map; class FunctionNameManager { @@ -51,7 +50,6 @@ class FunctionNameManager { FunctionNameMap m_name_map; }; - } // namespace spider::core #endif // SPIDER_CORE_FUNCTIONNAMEMANAGER_HPP diff --git a/src/spider/worker/Process.cpp b/src/spider/worker/Process.cpp index 2d631c476..b9b856c05 100644 --- a/src/spider/worker/Process.cpp +++ b/src/spider/worker/Process.cpp @@ -17,9 +17,7 @@ #include namespace spider::worker { - namespace { - auto close_all_fds(std::vector const& whitelist) -> bool { std::unique_ptr const dir{opendir("/dev/fd"), [](DIR* p) { closedir(p); }}; if (nullptr == dir) { @@ -50,7 +48,6 @@ auto close_all_fds(std::vector const& whitelist) -> bool { return true; } - } // namespace auto Process::spawn( @@ -121,5 +118,4 @@ auto Process::terminate() const -> void { throw std::runtime_error("Failed to terminate process"); } } - } // namespace spider::worker diff --git a/src/spider/worker/Process.hpp b/src/spider/worker/Process.hpp index b8e0a1086..2e5f1a475 100644 --- a/src/spider/worker/Process.hpp +++ b/src/spider/worker/Process.hpp @@ -8,7 +8,6 @@ #include namespace spider::worker { - class Process { public: static auto spawn( @@ -43,7 +42,6 @@ class Process { explicit Process(pid_t const pid) : m_pid(pid) {} }; - } // namespace spider::worker #endif diff --git a/src/spider/worker/TaskExecutor.cpp b/src/spider/worker/TaskExecutor.cpp index 7b2a16bfb..dad1d3705 100644 --- a/src/spider/worker/TaskExecutor.cpp +++ b/src/spider/worker/TaskExecutor.cpp @@ -15,7 +15,6 @@ #include "TaskExecutorMessage.hpp" namespace spider::worker { - auto TaskExecutor::completed() -> bool { std::lock_guard const lock(m_state_mutex); return TaskExecutorState::Succeed == m_state || TaskExecutorState::Error == m_state @@ -129,9 +128,11 @@ auto TaskExecutor::get_result_buffers() const -> std::optional std::tuple { return core::response_get_error(m_result_buffer) - .value_or(std::make_tuple( - core::FunctionInvokeError::ResultParsingError, - "Fail to parse error message" - )); + .value_or( + std::make_tuple( + core::FunctionInvokeError::ResultParsingError, + "Fail to parse error message" + ) + ); } } // namespace spider::worker diff --git a/src/spider/worker/TaskExecutor.hpp b/src/spider/worker/TaskExecutor.hpp index 70a3e5883..f7bb960d4 100644 --- a/src/spider/worker/TaskExecutor.hpp +++ b/src/spider/worker/TaskExecutor.hpp @@ -28,7 +28,6 @@ #include "Process.hpp" namespace spider::worker { - enum class TaskExecutorState : std::uint8_t { Running, Waiting, @@ -183,7 +182,6 @@ class TaskExecutor { msgpack::sbuffer m_result_buffer; }; - } // namespace spider::worker #endif // SPIDER_WORKER_TASKEXECUTOR_HPP diff --git a/src/spider/worker/TaskExecutorMessage.hpp b/src/spider/worker/TaskExecutorMessage.hpp index 4fa8de1b8..a26279a14 100644 --- a/src/spider/worker/TaskExecutorMessage.hpp +++ b/src/spider/worker/TaskExecutorMessage.hpp @@ -77,7 +77,6 @@ class TaskExecutorRequestParser { private: msgpack::object_handle m_obj; }; - } // namespace spider::worker // MSGPACK_ADD_ENUM must be called in global namespace diff --git a/src/spider/worker/WorkerClient.cpp b/src/spider/worker/WorkerClient.cpp index 705399ff8..489c116cf 100644 --- a/src/spider/worker/WorkerClient.cpp +++ b/src/spider/worker/WorkerClient.cpp @@ -28,7 +28,6 @@ #include "../storage/StorageFactory.hpp" namespace spider::worker { - WorkerClient::WorkerClient( boost::uuids::uuid const worker_id, std::string worker_addr, @@ -42,8 +41,8 @@ WorkerClient::WorkerClient( m_metadata_store(std::move(metadata_store)), m_storage_factory(std::move(storage_factory)) {} -auto WorkerClient::get_next_task(std::optional const& fail_task_id -) -> std::optional> { +auto WorkerClient::get_next_task(std::optional const& fail_task_id) + -> std::optional> { // Get schedulers std::vector schedulers; @@ -141,5 +140,4 @@ auto WorkerClient::get_next_task(std::optional const& fail_t return std::nullopt; } } - } // namespace spider::worker diff --git a/src/spider/worker/WorkerClient.hpp b/src/spider/worker/WorkerClient.hpp index 7f408c2c4..11f189176 100644 --- a/src/spider/worker/WorkerClient.hpp +++ b/src/spider/worker/WorkerClient.hpp @@ -31,8 +31,8 @@ class WorkerClient { std::shared_ptr storage_factory ); - auto get_next_task(std::optional const& fail_task_id - ) -> std::optional>; + auto get_next_task(std::optional const& fail_task_id) + -> std::optional>; private: boost::uuids::uuid m_worker_id; diff --git a/src/spider/worker/message_pipe.cpp b/src/spider/worker/message_pipe.cpp index 69c0c4261..0d8323ea5 100644 --- a/src/spider/worker/message_pipe.cpp +++ b/src/spider/worker/message_pipe.cpp @@ -15,7 +15,6 @@ #include "../io/MsgPack.hpp" // IWYU pragma: keep namespace spider::worker { - constexpr size_t cHeaderSize = 16; namespace { @@ -85,8 +84,8 @@ auto receive_message(boost::asio::posix::stream_descriptor& fd) -> std::optional return buffer; } -auto receive_message_async(std::reference_wrapper pipe -) -> boost::asio::awaitable> { +auto receive_message_async(std::reference_wrapper pipe) + -> boost::asio::awaitable> { std::array header_buffer{0}; // NOLINTNEXTLINE(clang-analyzer-core.NullDereference) auto [header_ec, header_n] = co_await boost::asio::async_read( @@ -142,5 +141,4 @@ auto receive_message_async(std::reference_wrapper pi buffer.write(body_buffer.data(), body_buffer.size()); co_return buffer; } - } // namespace spider::worker diff --git a/src/spider/worker/message_pipe.hpp b/src/spider/worker/message_pipe.hpp index 549f5639b..6d70f700d 100644 --- a/src/spider/worker/message_pipe.hpp +++ b/src/spider/worker/message_pipe.hpp @@ -8,17 +8,15 @@ #include "../io/MsgPack.hpp" // IWYU pragma: keep namespace spider::worker { - auto send_message(boost::asio::writable_pipe& pipe, msgpack::sbuffer const& request) -> bool; auto send_message(boost::asio::posix::stream_descriptor& fd, msgpack::sbuffer const& request) -> bool; -auto receive_message_async(std::reference_wrapper pipe -) -> boost::asio::awaitable>; +auto receive_message_async(std::reference_wrapper pipe) + -> boost::asio::awaitable>; auto receive_message(boost::asio::posix::stream_descriptor& fd) -> std::optional; - } // namespace spider::worker #endif // SPIDER_WORKER_MESSAGE_PIPE_HPP diff --git a/src/spider/worker/task_executor.cpp b/src/spider/worker/task_executor.cpp index 1b15c5d6b..064b48b1a 100644 --- a/src/spider/worker/task_executor.cpp +++ b/src/spider/worker/task_executor.cpp @@ -1,4 +1,3 @@ - #include #include @@ -32,7 +31,6 @@ #include "TaskExecutorMessage.hpp" namespace { - auto parse_arg(int const argc, char** const& argv) -> boost::program_options::variables_map { boost::program_options::options_description desc; desc.add_options()("help", "spider task executor"); @@ -62,7 +60,6 @@ auto parse_arg(int const argc, char** const& argv) -> boost::program_options::va boost::program_options::notify(variables); return variables; } - } // namespace constexpr int cCmdArgParseErr = 1; diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index 983c28241..960a38198 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -1,4 +1,3 @@ - #include #include #include @@ -81,8 +80,8 @@ auto parse_args(int const argc, char** argv) -> boost::program_options::variable } auto get_environment_variable() -> absl::flat_hash_map< - boost::process::v2::environment::key, - boost::process::v2::environment::value> { + boost::process::v2::environment::key, + boost::process::v2::environment::value> { boost::filesystem::path const executable_dir = boost::dll::program_location().parent_path(); // NOLINTNEXTLINE(concurrency-mt-unsafe) @@ -121,8 +120,9 @@ auto heartbeat_loop( fail_count++; continue; } - auto conn = std::move(std::get>(conn_result - )); + auto conn = std::move( + std::get>(conn_result) + ); spider::core::StorageErr const err = metadata_store->update_heartbeat(*conn, driver.get_id()); @@ -141,10 +141,9 @@ auto heartbeat_loop( constexpr int cFetchTaskTimeout = 100; -auto fetch_task( - spider::worker::WorkerClient& client, - std::optional fail_task_id -) -> std::tuple { +auto +fetch_task(spider::worker::WorkerClient& client, std::optional fail_task_id) + -> std::tuple { spdlog::debug("Fetching task"); while (true) { std::optional> const optional_task_ids @@ -158,8 +157,8 @@ auto fetch_task( } } -auto get_args_buffers(spider::core::Task const& task -) -> std::optional> { +auto get_args_buffers(spider::core::Task const& task) + -> std::optional> { std::vector args_buffers; size_t const num_inputs = task.get_num_inputs(); for (size_t i = 0; i < num_inputs; ++i) { @@ -189,10 +188,9 @@ auto get_args_buffers(spider::core::Task const& task return args_buffers; } -auto parse_outputs( - spider::core::Task const& task, - std::vector const& result_buffers -) -> std::optional> { +auto +parse_outputs(spider::core::Task const& task, std::vector const& result_buffers) + -> std::optional> { std::vector outputs; outputs.reserve(task.get_num_outputs()); for (size_t i = 0; i < task.get_num_outputs(); ++i) { @@ -303,8 +301,9 @@ auto task_loop( ); continue; } - auto conn = std::move(std::get>(conn_result - )); + auto conn = std::move( + std::get>(conn_result) + ); if (!executor.succeed()) { spdlog::warn("Task {} failed", task.get_function_name()); @@ -373,7 +372,6 @@ auto task_loop( } // NOLINTEND(clang-analyzer-unix.BlockInCriticalSection) - } // namespace // NOLINTNEXTLINE(bugprone-exception-escape) @@ -436,8 +434,9 @@ auto main(int argc, char** argv) -> int { ); return cStorageErr; } - auto conn = std::move(std::get>(conn_result - )); + auto conn = std::move( + std::get>(conn_result) + ); spider::core::StorageErr const err = metadata_store->add_driver(*conn, driver); if (!err.success()) { diff --git a/tests/client/client-test.cpp b/tests/client/client-test.cpp index a390e6160..a5d49b0e8 100644 --- a/tests/client/client-test.cpp +++ b/tests/client/client-test.cpp @@ -206,7 +206,6 @@ auto test_large_input_output( } constexpr size_t cLargeInputSize = 300; - } // namespace // NOLINTNEXTLINE(bugprone-exception-escape) diff --git a/tests/client/test-Driver.cpp b/tests/client/test-Driver.cpp index dd245a35f..2dbbebafd 100644 --- a/tests/client/test-Driver.cpp +++ b/tests/client/test-Driver.cpp @@ -75,7 +75,6 @@ TEMPLATE_LIST_TEST_CASE( spider::TaskGraph const graph_1 = driver.bind(&test_driver, data); spider::TaskGraph const graph_2 = driver.bind(&sum, &sum, graph_1); } - } // namespace // NOLINTEND(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) diff --git a/tests/io/test-MsgpackMessage.cpp b/tests/io/test-MsgpackMessage.cpp index e01c8dd29..4885628b9 100644 --- a/tests/io/test-MsgpackMessage.cpp +++ b/tests/io/test-MsgpackMessage.cpp @@ -15,7 +15,6 @@ #include "../../src/spider/io/msgpack_message.hpp" namespace { - using namespace boost::asio::ip; constexpr std::array cBufferSizes{1, 2, 3, 4, 5, 6, 7, 8, 9, 17, 257, 65'537}; @@ -129,5 +128,4 @@ TEST_CASE("Async socket msgpack", "[io]") { } // NOLINTEND(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) - } // namespace diff --git a/tests/scheduler/test-SchedulerServer.cpp b/tests/scheduler/test-SchedulerServer.cpp index c3df2bf41..3e3be219b 100644 --- a/tests/scheduler/test-SchedulerServer.cpp +++ b/tests/scheduler/test-SchedulerServer.cpp @@ -30,7 +30,6 @@ #include "../storage/StorageTestHelper.hpp" namespace { - constexpr int cServerWarmupTime = 5; TEMPLATE_LIST_TEST_CASE( diff --git a/tests/storage/StorageTestHelper.hpp b/tests/storage/StorageTestHelper.hpp index 3dd472188..0f9406ae2 100644 --- a/tests/storage/StorageTestHelper.hpp +++ b/tests/storage/StorageTestHelper.hpp @@ -27,7 +27,6 @@ requires std::same_as auto get_storage_url() -> std::string { return cMySqlStorageUrl; } - } // namespace spider::test // NOLINTEND(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity) diff --git a/tests/storage/test-DataStorage.cpp b/tests/storage/test-DataStorage.cpp index 21da3bb62..f47e4ee30 100644 --- a/tests/storage/test-DataStorage.cpp +++ b/tests/storage/test-DataStorage.cpp @@ -22,7 +22,6 @@ #include "StorageTestHelper.hpp" namespace { - TEMPLATE_LIST_TEST_CASE( "Add, get and remove data", "[storage]", diff --git a/tests/storage/test-MetadataStorage.cpp b/tests/storage/test-MetadataStorage.cpp index 22b8b2003..4c38fb49c 100644 --- a/tests/storage/test-MetadataStorage.cpp +++ b/tests/storage/test-MetadataStorage.cpp @@ -26,7 +26,6 @@ #include "StorageTestHelper.hpp" namespace { - TEMPLATE_LIST_TEST_CASE("Driver heartbeat", "[storage]", spider::test::StorageFactoryTypeList) { std::unique_ptr storage_factory = spider::test::create_storage_factory(); @@ -434,7 +433,8 @@ TEMPLATE_LIST_TEST_CASE("Task finish", "[storage]", spider::test::StorageFactory *conn, parent_1_instance, {spider::core::TaskOutput{"1.1", "float"}} - ).success()); + ) + .success()); // Parent 1 finish should not update state of any other tasks spider::core::Task res_task{""}; REQUIRE(storage->get_task(*conn, parent_2.get_id(), &res_task).success()); @@ -510,7 +510,8 @@ TEMPLATE_LIST_TEST_CASE("Job reset", "[storage]", spider::test::StorageFactoryTy *conn, parent_1_instance, {spider::core::TaskOutput{"1.1", "float"}} - ).success()); + ) + .success()); // Task finish for parent 2 should success spider::core::TaskInstance const parent_2_instance{gen(), parent_2.get_id()}; REQUIRE(storage->set_task_state(*conn, parent_2.get_id(), spider::core::TaskState::Running) @@ -555,7 +556,6 @@ TEMPLATE_LIST_TEST_CASE("Job reset", "[storage]", spider::test::StorageFactoryTy // Clean up REQUIRE(storage->remove_job(*conn, job_id).success()); } - } // namespace // NOLINTEND(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) diff --git a/tests/utils/CoreDataUtils.hpp b/tests/utils/CoreDataUtils.hpp index 8354db8be..0de998280 100644 --- a/tests/utils/CoreDataUtils.hpp +++ b/tests/utils/CoreDataUtils.hpp @@ -3,7 +3,6 @@ #include "../../src/spider/core/Data.hpp" namespace spider::test { - inline auto data_equal(core::Data const& d1, core::Data const& d2) -> bool { if (d1.get_id() != d2.get_id()) { return false; @@ -23,7 +22,6 @@ inline auto data_equal(core::Data const& d1, core::Data const& d2) -> bool { return true; } - } // namespace spider::test #endif // SPIDER_TESTS_COREDATAUTILS_HPP diff --git a/tests/utils/CoreTaskUtils.cpp b/tests/utils/CoreTaskUtils.cpp index bfa0b96c1..ded571f1b 100644 --- a/tests/utils/CoreTaskUtils.cpp +++ b/tests/utils/CoreTaskUtils.cpp @@ -17,7 +17,6 @@ #include "../../src/spider/core/TaskGraph.hpp" namespace spider::test { - namespace { constexpr double cEpsilon = 0.0001; @@ -112,7 +111,6 @@ auto hash_map_equal( return true; } - } // namespace auto task_graph_equal(core::TaskGraph const& graph_1, core::TaskGraph const& graph_2) -> bool { @@ -185,5 +183,4 @@ auto task_output_equal(core::TaskOutput const& output_1, core::TaskOutput const& } return true; } - } // namespace spider::test diff --git a/tests/utils/CoreTaskUtils.hpp b/tests/utils/CoreTaskUtils.hpp index a3661d517..7926a8e2d 100644 --- a/tests/utils/CoreTaskUtils.hpp +++ b/tests/utils/CoreTaskUtils.hpp @@ -5,7 +5,6 @@ #include "../../src/spider/core/TaskGraph.hpp" namespace spider::test { - auto task_equal(core::Task const& t1, core::Task const& t2) -> bool; auto task_input_equal(core::TaskInput const& input_1, core::TaskInput const& input_2) -> bool; @@ -13,7 +12,6 @@ auto task_input_equal(core::TaskInput const& input_1, core::TaskInput const& inp auto task_output_equal(core::TaskOutput const& output_1, core::TaskOutput const& output_2) -> bool; auto task_graph_equal(core::TaskGraph const& graph_1, core::TaskGraph const& graph_2) -> bool; - } // namespace spider::test #endif // SPIDER_TESTS_CORETASKUTILS_HPP diff --git a/tests/worker/test-Process.cpp b/tests/worker/test-Process.cpp index 57d070a36..e7eaf69b3 100644 --- a/tests/worker/test-Process.cpp +++ b/tests/worker/test-Process.cpp @@ -11,7 +11,6 @@ #include "../../src/spider/worker/Process.hpp" namespace { - TEST_CASE("Process exit", "[worker]") { spider::worker::Process const true_process = spider::worker::Process::spawn("true", {}, std::nullopt, std::nullopt, std::nullopt); @@ -68,7 +67,6 @@ TEST_CASE("Process pipe", "[worker]") { close(read_pipe_fd[0]); REQUIRE(echo_process.wait() == 0); } - } // namespace // NOLINTEND(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) diff --git a/tests/worker/test-TaskExecutor.cpp b/tests/worker/test-TaskExecutor.cpp index bbea86504..2df25cf23 100644 --- a/tests/worker/test-TaskExecutor.cpp +++ b/tests/worker/test-TaskExecutor.cpp @@ -33,8 +33,8 @@ namespace { auto get_environment_variable() -> absl::flat_hash_map< - boost::process::v2::environment::key, - boost::process::v2::environment::value> { + boost::process::v2::environment::key, + boost::process::v2::environment::value> { boost::filesystem::path const executable_dir = boost::dll::program_location().parent_path(); boost::filesystem::path const src_dir = executable_dir.parent_path() / "src" / "spider"; @@ -244,7 +244,6 @@ TEMPLATE_LIST_TEST_CASE( REQUIRE(result_option.has_value()); REQUIRE(input_1 + input_2 == result_option.value_or("")); } - } // namespace // NOLINTEND(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays,clang-analyzer-unix.BlockInCriticalSection) From 7e7502549549dfe1751a9573a022dd5538024670 Mon Sep 17 00:00:00 2001 From: sitao Date: Fri, 4 Apr 2025 19:10:52 -0400 Subject: [PATCH 7/7] Fix the venv in test tasks using lint --- test-tasks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-tasks.yaml b/test-tasks.yaml index 8b082dd4e..116ae85b6 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -68,9 +68,9 @@ tasks: cmds: - task: ":utils:misc:create-venv" vars: - LABEL: "lint" + LABEL: "test" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: