From 27c78d33c5ff7849c0595506107668f65195e912 Mon Sep 17 00:00:00 2001 From: sitao Date: Sun, 2 Mar 2025 23:36:20 -0500 Subject: [PATCH 01/21] Add storage connection in storage interface --- src/spider/CMakeLists.txt | 1 + src/spider/storage/DataStorage.hpp | 60 +- src/spider/storage/MetadataStorage.hpp | 125 ++- src/spider/storage/MySqlConnection.hpp | 5 +- src/spider/storage/MySqlStorage.cpp | 1250 +++++++++++----------- src/spider/storage/MySqlStorage.hpp | 161 ++- src/spider/storage/StorageConnection.hpp | 12 + 7 files changed, 884 insertions(+), 730 deletions(-) create mode 100644 src/spider/storage/StorageConnection.hpp diff --git a/src/spider/CMakeLists.txt b/src/spider/CMakeLists.txt index 80b4b23b6..a17adcaa7 100644 --- a/src/spider/CMakeLists.txt +++ b/src/spider/CMakeLists.txt @@ -24,6 +24,7 @@ set(SPIDER_CORE_HEADERS utils/LruCache.hpp storage/MetadataStorage.hpp storage/DataStorage.hpp + storage/StorageConnection.hpp storage/MySqlConnection.hpp storage/MySqlStorage.hpp worker/FunctionManager.hpp diff --git a/src/spider/storage/DataStorage.hpp b/src/spider/storage/DataStorage.hpp index 8352c40b0..7d221581c 100644 --- a/src/spider/storage/DataStorage.hpp +++ b/src/spider/storage/DataStorage.hpp @@ -8,6 +8,7 @@ #include "../core/Data.hpp" #include "../core/Error.hpp" #include "../core/KeyValueData.hpp" +#include "StorageConnection.hpp" namespace spider::core { class DataStorage { @@ -19,31 +20,56 @@ class DataStorage { auto operator=(DataStorage&&) -> DataStorage& = delete; virtual ~DataStorage() = default; - virtual auto initialize() -> StorageErr = 0; + virtual auto initialize(StorageConnection& conn) -> StorageErr = 0; - virtual auto add_driver_data(boost::uuids::uuid driver_id, Data const& data) -> StorageErr = 0; - virtual auto add_task_data(boost::uuids::uuid task_id, Data const& data) -> StorageErr = 0; - virtual auto get_data(boost::uuids::uuid id, Data* data) -> StorageErr = 0; - virtual auto set_data_locality(Data const& data) -> StorageErr = 0; - virtual auto remove_data(boost::uuids::uuid id) -> StorageErr = 0; - virtual auto add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr + virtual auto add_driver_data( + StorageConnection& conn, + boost::uuids::uuid driver_id, + Data const& data + ) -> StorageErr = 0; + virtual auto add_task_data( + StorageConnection& conn, + boost::uuids::uuid task_id, + Data const& data + ) -> StorageErr = 0; + virtual auto get_data(StorageConnection& conn, boost::uuids::uuid id, Data* data) -> StorageErr + = 0; + virtual auto set_data_locality(StorageConnection& conn, Data const& data) -> StorageErr = 0; + virtual auto remove_data(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; + virtual auto add_task_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid task_id + ) -> StorageErr = 0; + virtual auto remove_task_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid task_id + ) -> StorageErr = 0; + virtual auto add_driver_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid driver_id + ) -> StorageErr = 0; + virtual auto remove_driver_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid driver_id + ) -> StorageErr = 0; + virtual auto remove_dangling_data(StorageConnection& conn) -> StorageErr = 0; + + virtual auto add_client_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr = 0; - virtual auto - remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr = 0; - virtual auto - add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) -> StorageErr = 0; - virtual auto - remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) -> StorageErr = 0; - virtual auto remove_dangling_data() -> StorageErr = 0; - - virtual auto add_client_kv_data(KeyValueData const& data) -> StorageErr = 0; - virtual auto add_task_kv_data(KeyValueData const& data) -> StorageErr = 0; + virtual auto add_task_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr + = 0; virtual auto get_client_kv_data( + StorageConnection& conn, boost::uuids::uuid const& client_id, std::string const& key, std::string* value ) -> StorageErr = 0; virtual auto get_task_kv_data( + StorageConnection& conn, boost::uuids::uuid const& task_id, std::string const& key, std::string* value diff --git a/src/spider/storage/MetadataStorage.hpp b/src/spider/storage/MetadataStorage.hpp index 05d06b233..1ff5e8bf5 100644 --- a/src/spider/storage/MetadataStorage.hpp +++ b/src/spider/storage/MetadataStorage.hpp @@ -23,58 +23,109 @@ class MetadataStorage { auto operator=(MetadataStorage&&) -> MetadataStorage& = delete; virtual ~MetadataStorage() = default; - virtual auto initialize() -> StorageErr = 0; + virtual auto initialize(StorageConnection& conn) -> StorageErr = 0; - virtual auto add_driver(Driver const& driver) -> StorageErr = 0; - virtual auto add_scheduler(Scheduler const& scheduler) -> StorageErr = 0; - virtual auto get_active_scheduler(std::vector* schedulers) -> StorageErr = 0; + virtual auto add_driver(StorageConnection& conn, Driver const& driver) -> StorageErr = 0; + virtual auto add_scheduler(StorageConnection& conn, Scheduler const& scheduler) -> StorageErr + = 0; + virtual auto get_active_scheduler(StorageConnection& conn, std::vector* schedulers) + -> StorageErr = 0; - virtual auto - add_job(boost::uuids::uuid job_id, boost::uuids::uuid client_id, TaskGraph const& task_graph + virtual auto add_job( + StorageConnection& conn, + boost::uuids::uuid job_id, + boost::uuids::uuid client_id, + TaskGraph const& task_graph ) -> StorageErr = 0; - virtual auto get_job_metadata(boost::uuids::uuid id, JobMetadata* job) -> StorageErr = 0; - virtual auto get_job_complete(boost::uuids::uuid id, bool* complete) -> StorageErr = 0; - virtual auto get_job_status(boost::uuids::uuid id, JobStatus* status) -> StorageErr = 0; + virtual auto get_job_metadata(StorageConnection& conn, boost::uuids::uuid id, JobMetadata* job) + -> StorageErr = 0; + virtual auto get_job_complete(StorageConnection& conn, boost::uuids::uuid id, bool* complete) + -> StorageErr = 0; + virtual auto get_job_status(StorageConnection& conn, boost::uuids::uuid id, JobStatus* status) + -> StorageErr = 0; virtual auto get_job_output_tasks( + StorageConnection& conn, boost::uuids::uuid id, std::vector* task_ids ) -> StorageErr = 0; - virtual auto get_task_graph(boost::uuids::uuid id, TaskGraph* task_graph) -> StorageErr = 0; + virtual auto get_task_graph( + StorageConnection& conn, + boost::uuids::uuid id, + TaskGraph* task_graph + ) -> StorageErr = 0; virtual auto get_jobs_by_client_id( + StorageConnection& conn, boost::uuids::uuid client_id, std::vector* job_ids ) -> StorageErr = 0; - virtual auto remove_job(boost::uuids::uuid id) -> StorageErr = 0; - virtual auto reset_job(boost::uuids::uuid id) -> StorageErr = 0; - virtual auto add_child(boost::uuids::uuid parent_id, Task const& child) -> StorageErr = 0; - virtual auto get_task(boost::uuids::uuid id, Task* task) -> StorageErr = 0; - virtual auto get_task_job_id(boost::uuids::uuid id, boost::uuids::uuid* job_id) -> StorageErr - = 0; - virtual auto get_ready_tasks(std::vector* tasks) -> StorageErr = 0; - virtual auto set_task_state(boost::uuids::uuid id, TaskState state) -> StorageErr = 0; - virtual auto set_task_running(boost::uuids::uuid id) -> StorageErr = 0; - virtual auto add_task_instance(TaskInstance const& instance) -> StorageErr = 0; - // Set task state and add new task instance if task is ready or all instances timed out - virtual auto create_task_instance(TaskInstance const& instance) -> StorageErr = 0; - virtual auto task_finish(TaskInstance const& instance, std::vector const& outputs) + virtual auto remove_job(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; + virtual auto reset_job(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; + virtual auto add_child(StorageConnection& conn, boost::uuids::uuid parent_id, Task const& child) -> StorageErr = 0; - virtual auto task_fail(TaskInstance const& instance, std::string const& error) -> StorageErr - = 0; - virtual auto get_task_timeout(std::vector>* tasks) -> StorageErr + virtual auto get_task(StorageConnection& conn, boost::uuids::uuid id, Task* task) -> StorageErr = 0; - virtual auto get_child_tasks(boost::uuids::uuid id, std::vector* children) -> StorageErr - = 0; - virtual auto get_parent_tasks(boost::uuids::uuid id, std::vector* tasks) -> StorageErr - = 0; - - virtual auto update_heartbeat(boost::uuids::uuid id) -> StorageErr = 0; + virtual auto get_task_job_id( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid* job_id + ) -> StorageErr = 0; + virtual auto get_ready_tasks(StorageConnection& conn, std::vector* tasks) -> StorageErr + = 0; + virtual auto set_task_state(StorageConnection& conn, boost::uuids::uuid id, TaskState state) + -> StorageErr = 0; + virtual auto set_task_running(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; virtual auto - heartbeat_timeout(double timeout, std::vector* ids) -> StorageErr = 0; - virtual auto get_scheduler_state(boost::uuids::uuid id, std::string* state) -> StorageErr = 0; + add_task_instance(StorageConnection& conn, TaskInstance const& instance) -> StorageErr = 0; + // Set task state and add new task instance if task is ready or all instances timed out virtual auto - get_scheduler_addr(boost::uuids::uuid id, std::string* addr, int* port) -> StorageErr = 0; - virtual auto set_scheduler_state(boost::uuids::uuid id, std::string const& state) -> StorageErr - = 0; + create_task_instance(StorageConnection& conn, TaskInstance const& instance) -> StorageErr = 0; + virtual auto task_finish( + StorageConnection& conn, + TaskInstance const& instance, + std::vector const& outputs + ) -> StorageErr = 0; + virtual auto task_fail( + StorageConnection& conn, + TaskInstance const& instance, + std::string const& error + ) -> StorageErr = 0; + virtual auto get_task_timeout( + StorageConnection& conn, + std::vector>* tasks + ) -> StorageErr = 0; + virtual auto get_child_tasks( + StorageConnection& conn, + boost::uuids::uuid id, + std::vector* children + ) -> StorageErr = 0; + virtual auto get_parent_tasks( + StorageConnection& conn, + boost::uuids::uuid id, + std::vector* tasks + ) -> StorageErr = 0; + + virtual auto update_heartbeat(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr = 0; + virtual auto heartbeat_timeout( + StorageConnection& conn, + double timeout, + std::vector* ids + ) -> StorageErr = 0; + virtual auto get_scheduler_state( + StorageConnection& conn, + boost::uuids::uuid id, + std::string* state + ) -> StorageErr = 0; + virtual auto get_scheduler_addr( + StorageConnection& conn, + boost::uuids::uuid id, + std::string* addr, + int* port + ) -> StorageErr = 0; + virtual auto set_scheduler_state( + StorageConnection& conn, + boost::uuids::uuid id, + std::string const& state + ) -> StorageErr = 0; }; } // namespace spider::core diff --git a/src/spider/storage/MySqlConnection.hpp b/src/spider/storage/MySqlConnection.hpp index c7bafe2fa..4df1d03fa 100644 --- a/src/spider/storage/MySqlConnection.hpp +++ b/src/spider/storage/MySqlConnection.hpp @@ -9,11 +9,12 @@ #include #include "../core/Error.hpp" +#include "StorageConnection.hpp" namespace spider::core { // RAII class for MySQL connection -class MySqlConnection { +class MySqlConnection : public StorageConnection { public: static auto create(std::string const& url) -> std::variant; @@ -24,7 +25,7 @@ class MySqlConnection { MySqlConnection(MySqlConnection&&) = default; auto operator=(MySqlConnection&&) -> MySqlConnection& = default; - ~MySqlConnection(); + ~MySqlConnection() override; auto operator*() const -> sql::Connection&; auto operator->() const -> sql::Connection*; diff --git a/src/spider/storage/MySqlStorage.cpp b/src/spider/storage/MySqlStorage.cpp index a1b7ed8e4..114ac3f88 100644 --- a/src/spider/storage/MySqlStorage.cpp +++ b/src/spider/storage/MySqlStorage.cpp @@ -37,6 +37,7 @@ #include "../core/Task.hpp" #include "../core/TaskGraph.hpp" #include "MySqlConnection.hpp" +#include "StorageConnection.hpp" // mariadb-connector-cpp does not define SQL errcode. Just include some useful ones. enum MariadbErr : uint16_t { @@ -284,90 +285,84 @@ auto string_to_task_state(std::string const& state) -> spider::core::TaskState { } } // namespace -auto MySqlMetadataStorage::initialize() -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::initialize(StorageConnection& conn) -> StorageErr { try { for (char const* create_table_str : cCreateStorage) { - std::unique_ptr statement(conn->createStatement()); + std::unique_ptr statement( + static_cast(conn)->createStatement() + ); statement->executeUpdate(create_table_str); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::add_driver(Driver const& driver) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::add_driver(StorageConnection& conn, Driver const& driver) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("INSERT INTO `drivers` (`id`) VALUES (?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `drivers` (`id`) VALUES (?)" + ) ); sql::bytes id_bytes = uuid_get_bytes(driver.get_id()); statement->setBytes(1, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::add_scheduler(Scheduler const& scheduler) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::add_scheduler(StorageConnection& conn, Scheduler const& scheduler) + -> StorageErr { try { std::unique_ptr driver_statement( - conn->prepareStatement("INSERT INTO `drivers` (`id`) VALUES (?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `drivers` (`id`) VALUES (?)" + ) ); sql::bytes id_bytes = uuid_get_bytes(scheduler.get_id()); driver_statement->setBytes(1, &id_bytes); driver_statement->executeUpdate(); - std::unique_ptr scheduler_statement(conn->prepareStatement( - "INSERT INTO `schedulers` (`id`, `address`, `port`, `state`) " - "VALUES (?, ?, ?, 'normal')" - )); + std::unique_ptr scheduler_statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `schedulers` (`id`, `address`, `port`, `state`) VALUES (?, ?, " + "?, 'normal')" + ) + ); scheduler_statement->setBytes(1, &id_bytes); scheduler_statement->setString(2, scheduler.get_addr()); scheduler_statement->setInt(3, scheduler.get_port()); scheduler_statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_active_scheduler(std::vector* schedulers) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_active_scheduler( + StorageConnection& conn, + std::vector* schedulers +) -> StorageErr { try { - std::unique_ptr statement(conn->createStatement()); + std::unique_ptr statement( + static_cast(conn)->createStatement() + ); std::unique_ptr res(statement->executeQuery( "SELECT `schedulers`.`id`, `address`, `port` FROM `schedulers` JOIN `drivers` ON " "`schedulers`.`id` = `drivers`.`id` WHERE `state` = 'normal'" @@ -379,10 +374,10 @@ auto MySqlMetadataStorage::get_active_scheduler(std::vector* schedule schedulers->emplace_back(id, addr, port); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } @@ -464,21 +459,19 @@ void MySqlMetadataStorage::add_task(MySqlConnection& conn, sql::bytes job_id, Ta // NOLINTBEGIN(readability-function-cognitive-complexity) auto MySqlMetadataStorage::add_job( + StorageConnection& conn, boost::uuids::uuid job_id, boost::uuids::uuid client_id, TaskGraph const& task_graph ) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); try { sql::bytes job_id_bytes = uuid_get_bytes(job_id); sql::bytes client_id_bytes = uuid_get_bytes(client_id); { std::unique_ptr statement{ - conn->prepareStatement("INSERT INTO `jobs` (`id`, `client_id`) VALUES (?, ?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `jobs` (`id`, `client_id`) VALUES (?, ?)" + ) }; statement->setBytes(1, &job_id_bytes); statement->setBytes(2, &client_id_bytes); @@ -496,14 +489,14 @@ auto MySqlMetadataStorage::add_job( for (boost::uuids::uuid const task_id : heads) { std::optional const task_option = task_graph.get_task(task_id); if (!task_option.has_value()) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{ StorageErrType::KeyNotFoundErr, "Task graph inconsistent: head task not found" }; } Task const* task = task_option.value(); - add_task(conn, job_id_bytes, *task); + add_task(static_cast(conn), job_id_bytes, *task); for (boost::uuids::uuid const id : task_graph.get_child_tasks(task_id)) { std::vector const parents = task_graph.get_parent_tasks(id); if (std::ranges::all_of(parents, [&](boost::uuids::uuid const& parent) { @@ -522,11 +515,11 @@ auto MySqlMetadataStorage::add_job( heads.insert(task_id); std::optional const task_option = task_graph.get_task(task_id); if (!task_option.has_value()) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::KeyNotFoundErr, "Task graph inconsistent"}; } Task const* task = task_option.value(); - add_task(conn, job_id_bytes, *task); + add_task(static_cast(conn), job_id_bytes, *task); for (boost::uuids::uuid const id : task_graph.get_child_tasks(task_id)) { std::vector const parents = task_graph.get_parent_tasks(id); if (std::ranges::all_of(parents, [&](boost::uuids::uuid const& parent) { @@ -543,9 +536,11 @@ auto MySqlMetadataStorage::add_job( for (std::pair const& pair : task_graph.get_dependencies()) { - std::unique_ptr dep_statement{conn->prepareStatement( - "INSERT INTO `task_dependencies` (parent, child) VALUES (?, ?)" - )}; + std::unique_ptr dep_statement{ + static_cast(conn)->prepareStatement( + "INSERT INTO `task_dependencies` (parent, child) VALUES (?, ?)" + ) + }; sql::bytes parent_id_bytes = uuid_get_bytes(pair.first); sql::bytes child_id_bytes = uuid_get_bytes(pair.second); dep_statement->setBytes(1, &parent_id_bytes); @@ -555,9 +550,12 @@ auto MySqlMetadataStorage::add_job( // Add input tasks for (size_t i = 0; i < input_task_ids.size(); i++) { - std::unique_ptr input_statement{conn->prepareStatement( - "INSERT INTO `input_tasks` (`job_id`, `task_id`, `position`) VALUES (?, ?, ?)" - )}; + std::unique_ptr input_statement{ + static_cast(conn)->prepareStatement( + "INSERT INTO `input_tasks` (`job_id`, `task_id`, `position`) VALUES " + "(?, ?, ?)" + ) + }; input_statement->setBytes(1, &job_id_bytes); sql::bytes task_id_bytes = uuid_get_bytes(input_task_ids[i]); input_statement->setBytes(2, &task_id_bytes); @@ -567,9 +565,12 @@ auto MySqlMetadataStorage::add_job( // Add output tasks std::vector const& output_task_ids = task_graph.get_output_tasks(); for (size_t i = 0; i < output_task_ids.size(); i++) { - std::unique_ptr output_statement{conn->prepareStatement( - "INSERT INTO `output_tasks` (`job_id`, `task_id`, `position`) VALUES (?, ?, ?)" - )}; + std::unique_ptr output_statement{ + static_cast(conn)->prepareStatement( + "INSERT INTO `output_tasks` (`job_id`, `task_id`, `position`) VALUES " + "(?, ?, ?)" + ) + }; output_statement->setBytes(1, &job_id_bytes); sql::bytes task_id_bytes = uuid_get_bytes(output_task_ids[i]); output_statement->setBytes(2, &task_id_bytes); @@ -580,7 +581,9 @@ auto MySqlMetadataStorage::add_job( // Mark head tasks as ready for (boost::uuids::uuid const& task_id : task_graph.get_input_tasks()) { std::unique_ptr statement( - conn->prepareStatement("UPDATE `tasks` SET `state` = 'ready' WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `state` = 'ready' WHERE `id` = ?" + ) ); sql::bytes task_id_bytes = uuid_get_bytes(task_id); statement->setBytes(1, &task_id_bytes); @@ -588,13 +591,13 @@ auto MySqlMetadataStorage::add_job( } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } @@ -704,8 +707,7 @@ auto MySqlMetadataStorage::fetch_full_task( // Get task inputs std::unique_ptr input_statement{conn->prepareStatement( "SELECT `task_id`, `position`, `type`, `output_task_id`, `output_task_position`, " - "`value`, `data_id` FROM `task_inputs` " - "WHERE `task_id` = ? ORDER BY `position`" + "`value`, `data_id` FROM `task_inputs` WHERE `task_id` = ? ORDER BY `position`" )}; input_statement->setBytes(1, &id_bytes); std::unique_ptr const input_res{input_statement->executeQuery()}; @@ -714,10 +716,10 @@ auto MySqlMetadataStorage::fetch_full_task( } // Get task outputs - std::unique_ptr output_statement{conn->prepareStatement( - "SELECT `task_id`, `position`, `type`, `value`, `data_id` FROM `task_outputs` WHERE " - "`task_id` = ? ORDER BY `position`" - )}; + std::unique_ptr output_statement{ + conn->prepareStatement("SELECT `task_id`, `position`, `type`, `value`, `data_id` FROM " + "`task_outputs` WHERE `task_id` = ? ORDER BY `position`") + }; output_statement->setBytes(1, &id_bytes); std::unique_ptr const output_res{output_statement->executeQuery()}; while (output_res->next()) { @@ -726,24 +728,24 @@ auto MySqlMetadataStorage::fetch_full_task( return task; } -auto MySqlMetadataStorage::get_task_graph(boost::uuids::uuid id, TaskGraph* task_graph) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_task_graph( + StorageConnection& conn, + boost::uuids::uuid id, + TaskGraph* task_graph +) -> StorageErr { try { // Get all tasks std::unique_ptr task_statement( - conn->prepareStatement("SELECT `id`, `func_name`, `state`, `timeout` " - "FROM `tasks` WHERE `job_id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` WHERE `job_id` " + "= ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); task_statement->setBytes(1, &id_bytes); std::unique_ptr const task_res(task_statement->executeQuery()); if (task_res->rowsCount() == 0) { - conn->commit(); + static_cast(conn)->commit(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format("no task graph with id {}", boost::uuids::to_string(id)) @@ -755,47 +757,48 @@ auto MySqlMetadataStorage::get_task_graph(boost::uuids::uuid id, TaskGraph* task } // Get inputs - std::unique_ptr input_statement(conn->prepareStatement( - "SELECT `t1`.`task_id`, `t1`.`position`, `t1`.`type`, `t1`.`output_task_id`, " - "`t1`.`output_task_position`, `t1`.`value`, `t1`.`data_id` FROM `task_inputs` AS " - "`t1` JOIN " - "`tasks` " - "ON `t1`.`task_id` = `tasks`.`id` WHERE `tasks`.`job_id` = ? ORDER BY " - "`t1`.`task_id`, " - "`t1`.`position`" - )); + std::unique_ptr input_statement( + static_cast(conn)->prepareStatement( + "SELECT `t1`.`task_id`, `t1`.`position`, `t1`.`type`, " + "`t1`.`output_task_id`, `t1`.`output_task_position`, `t1`.`value`, " + "`t1`.`data_id` FROM `task_inputs` AS `t1` JOIN `tasks` ON `t1`.`task_id` " + "= `tasks`.`id` WHERE `tasks`.`job_id` = ? ORDER BY `t1`.`task_id`, " + "`t1`.`position`" + ) + ); input_statement->setBytes(1, &id_bytes); std::unique_ptr const input_res(input_statement->executeQuery()); while (input_res->next()) { if (!fetch_task_graph_task_input(task_graph, input_res)) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::KeyNotFoundErr, "Task storage inconsistent"}; } } // Get outputs std::unique_ptr output_statement( - conn->prepareStatement("SELECT `t1`.`task_id`, `t1`.`position`, `t1`.`type`, " - "`t1`.`value`, `t1`.`data_id` FROM " - "`task_outputs` " - "AS `t1` JOIN `tasks` ON `t1`.`task_id` = `tasks`.`id` " - "WHERE `tasks`.`job_id` = ? ORDER BY " - "`t1`.`task_id`, `t1`.`position`") + static_cast(conn)->prepareStatement( + "SELECT `t1`.`task_id`, `t1`.`position`, `t1`.`type`, `t1`.`value`, " + "`t1`.`data_id` FROM `task_outputs` AS `t1` JOIN `tasks` ON `t1`.`task_id` " + "= `tasks`.`id` WHERE `tasks`.`job_id` = ? ORDER BY `t1`.`task_id`, " + "`t1`.`position`" + ) ); output_statement->setBytes(1, &id_bytes); std::unique_ptr const output_res(output_statement->executeQuery()); while (output_res->next()) { if (!fetch_task_graph_task_output(task_graph, output_res)) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::KeyNotFoundErr, "Task storage inconsistent"}; } } // Get dependencies std::unique_ptr dep_statement( - conn->prepareStatement("SELECT `t1`.`parent`, `t1`.`child` FROM " - "`task_dependencies` AS `t1` JOIN `tasks` ON " - "`t1`.`parent` = `tasks`.`id` WHERE `tasks`.`job_id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `t1`.`parent`, `t1`.`child` FROM `task_dependencies` AS `t1` JOIN " + "`tasks` ON `t1`.`parent` = `tasks`.`id` WHERE `tasks`.`job_id` = ?" + ) ); dep_statement->setBytes(1, &id_bytes); std::unique_ptr const dep_res(dep_statement->executeQuery()); @@ -808,8 +811,10 @@ auto MySqlMetadataStorage::get_task_graph(boost::uuids::uuid id, TaskGraph* task // Get input tasks std::unique_ptr input_task_statement( - conn->prepareStatement("SELECT `task_id`, `position` FROM `input_tasks` WHERE " - "`job_id` = ? ORDER BY `position`") + static_cast(conn)->prepareStatement( + "SELECT `task_id`, `position` FROM `input_tasks` WHERE `job_id` = ? ORDER " + "BY `position`" + ) ); input_task_statement->setBytes(1, &id_bytes); std::unique_ptr const input_task_res(input_task_statement->executeQuery()); @@ -818,8 +823,10 @@ auto MySqlMetadataStorage::get_task_graph(boost::uuids::uuid id, TaskGraph* task } // Get output tasks std::unique_ptr output_task_statement( - conn->prepareStatement("SELECT `task_id`, `position` FROM `output_tasks` WHERE " - "`job_id` = ? ORDER BY `position`") + static_cast(conn)->prepareStatement( + "SELECT `task_id`, `position` FROM `output_tasks` WHERE `job_id` = ? ORDER " + "BY `position`" + ) ); output_task_statement->setBytes(1, &id_bytes); std::unique_ptr const output_task_res(output_task_statement->executeQuery() @@ -829,13 +836,13 @@ auto MySqlMetadataStorage::get_task_graph(boost::uuids::uuid id, TaskGraph* task } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErKeyNotFound) { return StorageErr{StorageErrType::KeyNotFoundErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } } // namespace spider::core @@ -857,21 +864,22 @@ auto parse_timestamp(std::string const& timestamp namespace spider::core { -auto MySqlMetadataStorage::get_job_metadata(boost::uuids::uuid id, JobMetadata* job) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_job_metadata( + StorageConnection& conn, + boost::uuids::uuid id, + JobMetadata* job +) -> StorageErr { try { - std::unique_ptr statement{conn->prepareStatement( - "SELECT `client_id`, `creation_time` FROM `jobs` WHERE `id` = ?" - )}; + std::unique_ptr statement{ + static_cast(conn)->prepareStatement( + "SELECT `client_id`, `creation_time` FROM `jobs` WHERE `id` = ?" + ) + }; sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr const res{statement->executeQuery()}; if (0 == res->rowsCount()) { - conn->commit(); + static_cast(conn)->commit(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format("No job with id {} ", boost::uuids::to_string(id)) @@ -882,7 +890,7 @@ auto MySqlMetadataStorage::get_job_metadata(boost::uuids::uuid id, JobMetadata* std::optional const optional_creation_time = parse_timestamp(get_sql_string(res->getString("creation_time"))); if (false == optional_creation_time.has_value()) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{ StorageErrType::OtherErr, fmt::format( @@ -893,100 +901,103 @@ auto MySqlMetadataStorage::get_job_metadata(boost::uuids::uuid id, JobMetadata* } *job = JobMetadata{id, client_id, optional_creation_time.value()}; } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_job_complete(boost::uuids::uuid const id, bool* complete) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_job_complete( + StorageConnection& conn, + boost::uuids::uuid const id, + bool* complete +) -> StorageErr { try { std::unique_ptr const statement{ - conn->prepareStatement("SELECT `state` FROM `tasks` WHERE `job_id` = ? AND " - "`state` NOT IN ('success', 'cancel', 'fail') ") + static_cast(conn)->prepareStatement( + "SELECT `state` FROM `tasks` WHERE `job_id` = ? AND `state` NOT IN " + "('success', 'cancel', 'fail') " + ) }; sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr const res{statement->executeQuery()}; *complete = 0 == res->rowsCount(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_job_status(boost::uuids::uuid const id, JobStatus* status) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_job_status( + StorageConnection& conn, + boost::uuids::uuid const id, + JobStatus* status +) -> StorageErr { try { std::unique_ptr const running_statement{ - conn->prepareStatement("SELECT `state` FROM `tasks` WHERE `job_id` = ? AND " - "`state` NOT IN ('success', 'cancel', 'fail') ") + static_cast(conn)->prepareStatement( + "SELECT `state` FROM `tasks` WHERE `job_id` = ? AND `state` NOT IN " + "('success', 'cancel', 'fail')" + ) }; sql::bytes id_bytes = uuid_get_bytes(id); running_statement->setBytes(1, &id_bytes); std::unique_ptr const running_res{running_statement->executeQuery()}; if (running_res->rowsCount() > 0) { *status = JobStatus::Running; - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } - std::unique_ptr failed_statement{conn->prepareStatement( - "SELECT `state` FROM `tasks` WHERE `job_id` = ? AND `state` = 'fail'" - )}; + std::unique_ptr failed_statement{ + static_cast(conn)->prepareStatement( + "SELECT `state` FROM `tasks` WHERE `job_id` = ? AND `state` = 'fail'" + ) + }; failed_statement->setBytes(1, &id_bytes); std::unique_ptr const failed_res{failed_statement->executeQuery()}; if (failed_res->rowsCount() > 0) { *status = JobStatus::Failed; - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } - std::unique_ptr canceled_statement{conn->prepareStatement( - "SELECT `state` FROM `tasks` WHERE `job_id` = ? AND `state` = 'cancel'" - )}; + std::unique_ptr canceled_statement{ + static_cast(conn)->prepareStatement( + "SELECT `state` FROM `tasks` WHERE `job_id` = ? AND `state` = 'cancel'" + ) + }; canceled_statement->setBytes(1, &id_bytes); std::unique_ptr const canceled_res{canceled_statement->executeQuery()}; if (canceled_res->rowsCount() > 0) { *status = JobStatus::Cancelled; - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } *status = JobStatus::Succeeded; } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } auto MySqlMetadataStorage::get_job_output_tasks( + StorageConnection& conn, boost::uuids::uuid const id, std::vector* task_ids ) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); try { task_ids->clear(); - std::unique_ptr statement{conn->prepareStatement( - "SELECT `task_id` FROM `output_tasks` WHERE `job_id` = ? ORDER BY `position`" - )}; + std::unique_ptr statement{ + static_cast(conn)->prepareStatement( + "SELECT `task_id` FROM `output_tasks` WHERE `job_id` = ? ORDER BY " + "`position`" + ) + }; sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr const res{statement->executeQuery()}; @@ -994,25 +1005,23 @@ auto MySqlMetadataStorage::get_job_output_tasks( task_ids->emplace_back(read_id(res->getBinaryStream(1))); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } auto MySqlMetadataStorage::get_jobs_by_client_id( + StorageConnection& conn, boost::uuids::uuid client_id, std::vector* job_ids ) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); try { std::unique_ptr statement{ - conn->prepareStatement("SELECT `id` FROM `jobs` WHERE `client_id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `id` FROM `jobs` WHERE `client_id` = ?" + ) }; sql::bytes client_id_bytes = uuid_get_bytes(client_id); statement->setBytes(1, &client_id_bytes); @@ -1021,168 +1030,170 @@ auto MySqlMetadataStorage::get_jobs_by_client_id( job_ids->emplace_back(read_id(res->getBinaryStream(1))); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::remove_job(boost::uuids::uuid id) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::remove_job(StorageConnection& conn, boost::uuids::uuid id) + -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("DELETE FROM `jobs` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "DELETE FROM `jobs` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::reset_job(boost::uuids::uuid const id) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::reset_job(StorageConnection& conn, boost::uuids::uuid const id) + -> StorageErr { try { // Check for retry count on all tasks - std::unique_ptr retry_statement(conn->prepareStatement( - "SELECT `id` FROM `tasks` WHERE `job_id` = ? AND `retry` >= `max_retry`" - )); + std::unique_ptr retry_statement( + static_cast(conn)->prepareStatement( + "SELECT `id` FROM `tasks` WHERE `job_id` = ? AND `retry` >= `max_retry`" + ) + ); sql::bytes job_id_bytes = uuid_get_bytes(id); retry_statement->setBytes(1, &job_id_bytes); std::unique_ptr const res(retry_statement->executeQuery()); if (res->rowsCount() > 0) { - conn->commit(); + static_cast(conn)->commit(); return StorageErr{StorageErrType::Success, "Some tasks have reached max retry count"}; } // Increment the retry count for all tasks - std::unique_ptr increment_statement(conn->prepareStatement( - "UPDATE `tasks` SET `retry` = `retry` + 1 WHERE `job_id` = ?" - )); + std::unique_ptr increment_statement( + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `retry` = `retry` + 1 WHERE `job_id` = ?" + ) + ); increment_statement->setBytes(1, &job_id_bytes); increment_statement->executeUpdate(); // Reset states for all tasks. Head tasks should be ready and other tasks should be pending - std::unique_ptr state_statement(conn->prepareStatement( - "UPDATE `tasks` SET `state` = IF(`id` NOT IN (SELECT `task_id` FROM `task_inputs` " - "WHERE `task_id` IN (SELECT `id` FROM `tasks` WHERE `job_id` = ?) AND " - "`output_task_id` IS NOT NULL), 'ready', 'pending') WHERE job_id = ?" - )); + std::unique_ptr state_statement( + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `state` = IF(`id` NOT IN (SELECT `task_id` FROM " + "`task_inputs` WHERE `task_id` IN (SELECT `id` FROM `tasks` WHERE `job_id` " + "= ?) AND `output_task_id` IS NOT NULL), 'ready', 'pending') WHERE job_id " + "= ?" + ) + ); state_statement->setBytes(1, &job_id_bytes); state_statement->setBytes(2, &job_id_bytes); state_statement->executeUpdate(); // Clear outputs for all tasks - std::unique_ptr output_statement(conn->prepareStatement( - "UPDATE `task_outputs` SET `value` = NULL, `data_id` = NULL " - "WHERE `task_id` IN (SELECT `id` FROM `tasks` WHERE `job_id` = ?)" - )); + std::unique_ptr output_statement( + static_cast(conn)->prepareStatement( + "UPDATE `task_outputs` SET `value` = NULL, `data_id` = NULL WHERE " + "`task_id` IN (SELECT `id` FROM `tasks` WHERE `job_id` = ?)" + ) + ); output_statement->setBytes(1, &job_id_bytes); output_statement->executeUpdate(); // Clear inputs for non-head tasks - std::unique_ptr input_statement(conn->prepareStatement( - "UPDATE `task_inputs` SET `value` = NULL, `data_id` = NULL " - "WHERE `task_id` IN (SELECT `id` FROM `tasks` WHERE `job_id` = ?) " - "AND `output_task_id` IS NOT NULL" - )); + std::unique_ptr input_statement( + static_cast(conn)->prepareStatement( + "UPDATE `task_inputs` SET `value` = NULL, `data_id` = NULL WHERE `task_id` " + "IN (SELECT `id` FROM `tasks` WHERE `job_id` = ?) AND `output_task_id` IS " + "NOT NULL" + ) + ); input_statement->setBytes(1, &job_id_bytes); input_statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::add_child(boost::uuids::uuid parent_id, Task const& child) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::add_child( + StorageConnection& conn, + boost::uuids::uuid parent_id, + Task const& child +) -> StorageErr { try { sql::bytes const job_id = uuid_get_bytes(child.get_id()); - add_task(conn, job_id, child); + add_task(static_cast(conn), job_id, child); // Add dependencies - std::unique_ptr statement(conn->prepareStatement( - "INSERT INTO `task_dependencies` (`parent`, `child`) VALUES (?, ?)" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `task_dependencies` (`parent`, `child`) VALUES (?, ?)" + ) + ); sql::bytes parent_id_bytes = uuid_get_bytes(parent_id); sql::bytes child_id_bytes = uuid_get_bytes(child.get_id()); statement->setBytes(1, &parent_id_bytes); statement->setBytes(2, &child_id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_task(boost::uuids::uuid id, Task* task) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_task(StorageConnection& conn, boost::uuids::uuid id, Task* task) + -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("SELECT `id`, `func_name`, `state`, `timeout` " - "FROM `tasks` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr const res(statement->executeQuery()); if (res->rowsCount() == 0) { - conn->commit(); + static_cast(conn)->commit(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format("no task with id {}", boost::uuids::to_string(id)) }; } res->next(); - *task = fetch_full_task(conn, res); + *task = fetch_full_task(static_cast(conn), res); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_task_job_id(boost::uuids::uuid id, boost::uuids::uuid* job_id) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_task_job_id( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid* job_id +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("SELECT `job_id` FROM `tasks` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `job_id` FROM `tasks` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr const res(statement->executeQuery()); if (res->rowsCount() == 0) { - conn->commit(); + static_cast(conn)->commit(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format("no task with id {}", boost::uuids::to_string(id)) @@ -1191,199 +1202,201 @@ auto MySqlMetadataStorage::get_task_job_id(boost::uuids::uuid id, boost::uuids:: res->next(); *job_id = read_id(res->getBinaryStream("job_id")); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_ready_tasks(std::vector* tasks) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_ready_tasks(StorageConnection& conn, std::vector* tasks) + -> StorageErr { try { // Get all ready tasks from job that has not failed or cancelled - std::unique_ptr statement(conn->createStatement()); + std::unique_ptr statement( + static_cast(conn)->createStatement() + ); std::unique_ptr res(statement->executeQuery( "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` WHERE `state` = 'ready' " "AND `job_id` NOT IN (SELECT `job_id` FROM `tasks` WHERE `state` = 'fail' OR " "`state` = 'cancel')" )); while (res->next()) { - tasks->emplace_back(fetch_full_task(conn, res)); + tasks->emplace_back(fetch_full_task(static_cast(conn), res)); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::set_task_state(boost::uuids::uuid id, TaskState state) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::set_task_state( + StorageConnection& conn, + boost::uuids::uuid id, + TaskState state +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("UPDATE `tasks` SET `state` = ? WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `state` = ? WHERE `id` = ?" + ) ); statement->setString(1, task_state_to_string(state)); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(2, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErKeyNotFound) { return StorageErr{StorageErrType::KeyNotFoundErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::set_task_running(boost::uuids::uuid id) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::set_task_running(StorageConnection& conn, boost::uuids::uuid id) + -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "UPDATE `tasks` SET `state` = 'running' WHERE `id` = ? AND `state` = 'ready'" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `state` = 'running' WHERE `id` = ? AND `state` = " + "'ready'" + ) + ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); int32_t const update_count = statement->executeUpdate(); if (update_count == 0) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::KeyNotFoundErr, "Task not ready"}; } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::add_task_instance(TaskInstance const& instance) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::add_task_instance(StorageConnection& conn, TaskInstance const& instance) + -> StorageErr { try { - std::unique_ptr const statement(conn->prepareStatement( - "INSERT INTO `task_instances` (`id`, `task_id`, `start_time`) " - "VALUES(?, ?, CURRENT_TIMESTAMP())" - )); + std::unique_ptr const statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `task_instances` (`id`, `task_id`, `start_time`) VALUES(?, ?, " + "CURRENT_TIMESTAMP())" + ) + ); sql::bytes id_bytes = uuid_get_bytes(instance.id); sql::bytes task_id_bytes = uuid_get_bytes(instance.task_id); statement->setBytes(1, &id_bytes); statement->setBytes(2, &task_id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::create_task_instance(TaskInstance const& instance) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::create_task_instance( + StorageConnection& conn, + TaskInstance const& instance +) -> StorageErr { try { // Check the state of the task - std::unique_ptr ready_statement(conn->prepareStatement( - "SELECT `state` FROM `tasks` WHERE `id` = ? AND `state` = 'ready'" - )); + std::unique_ptr ready_statement( + static_cast(conn)->prepareStatement( + "SELECT `state` FROM `tasks` WHERE `id` = ? AND `state` = 'ready'" + ) + ); sql::bytes id_bytes = uuid_get_bytes(instance.task_id); ready_statement->setBytes(1, &id_bytes); std::unique_ptr const ready_res(ready_statement->executeQuery()); bool const task_ready = ready_res->rowsCount() > 0; // Check all task instances have timed out - std::unique_ptr not_timeout_statement(conn->prepareStatement( - "SELECT `t1`.`id` FROM `task_instances` as `t1` JOIN `tasks` ON `t1`.`task_id` = " - "`tasks`.`id` WHERE `t1`.`task_id` = ? AND `tasks`.`timeout` < 0.0001 AND " - "TIMESTAMPDIFF(MICROSECOND, `t1`.`start_time`, CURRENT_TIMESTAMP()) < " - "`tasks`.`timeout` * 1000" - )); + std::unique_ptr not_timeout_statement( + static_cast(conn)->prepareStatement( + "SELECT `t1`.`id` FROM `task_instances` as `t1` JOIN `tasks` ON " + "`t1`.`task_id` = `tasks`.`id` WHERE `t1`.`task_id` = ? AND " + "`tasks`.`timeout` < 0.0001 AND TIMESTAMPDIFF(MICROSECOND, " + "`t1`.`start_time`, CURRENT_TIMESTAMP()) < `tasks`.`timeout` * 1000" + ) + ); not_timeout_statement->setBytes(1, &id_bytes); std::unique_ptr const not_timeout_res(not_timeout_statement->executeQuery() ); bool const all_timeout = not_timeout_res->rowsCount() == 0; if (!task_ready && !all_timeout) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, "Task not ready or timed out"}; } // Set the state to running std::unique_ptr const running_statement( - conn->prepareStatement("UPDATE `tasks` SET `state` = 'running' WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `state` = 'running' WHERE `id` = ?" + ) ); running_statement->setBytes(1, &id_bytes); running_statement->executeUpdate(); // Insert task instance - std::unique_ptr const instance_statement(conn->prepareStatement( - "INSERT INTO `task_instances` (`id`, `task_id`, `start_time`) VALUES(?, ?, " - "CURRENT_TIMESTAMP())" - )); + std::unique_ptr const instance_statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `task_instances` (`id`, `task_id`, `start_time`) VALUES(?, ?, " + "CURRENT_TIMESTAMP())" + ) + ); sql::bytes instance_id_bytes = uuid_get_bytes(instance.id); instance_statement->setBytes(1, &instance_id_bytes); instance_statement->setBytes(2, &id_bytes); instance_statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } auto MySqlMetadataStorage::task_finish( + StorageConnection& conn, TaskInstance const& instance, std::vector const& outputs ) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); try { // Try to submit task instance - std::unique_ptr const statement(conn->prepareStatement( - "UPDATE `tasks` SET `instance_id` = ?, `state` = 'success' WHERE `id` = ? AND " - "`instance_id` is NULL AND `state` = 'running'" - )); + std::unique_ptr const statement( + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `instance_id` = ?, `state` = 'success' WHERE `id` = ? " + "AND `instance_id` is NULL AND `state` = 'running'" + ) + ); sql::bytes id_bytes = uuid_get_bytes(instance.id); sql::bytes task_id_bytes = uuid_get_bytes(instance.task_id); statement->setBytes(1, &id_bytes); statement->setBytes(2, &task_id_bytes); int32_t const update_count = statement->executeUpdate(); if (update_count == 0) { - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } // Update task outputs - std::unique_ptr output_statement(conn->prepareStatement( - "UPDATE `task_outputs` SET `value` = ?, `data_id` = ? WHERE `task_id` = ? AND " - "`position` = ?" - )); + std::unique_ptr output_statement( + static_cast(conn)->prepareStatement( + "UPDATE `task_outputs` SET `value` = ?, `data_id` = ? WHERE `task_id` = ? " + "AND `position` = ?" + ) + ); for (size_t i = 0; i < outputs.size(); ++i) { TaskOutput const& output = outputs[i]; std::optional const& value = output.get_value(); @@ -1405,10 +1418,12 @@ auto MySqlMetadataStorage::task_finish( } // Update task inputs - std::unique_ptr input_statement(conn->prepareStatement( - "UPDATE `task_inputs` SET `value` = ?, `data_id` = ? WHERE `output_task_id` = ? " - "AND `output_task_position` = ?" - )); + std::unique_ptr input_statement( + static_cast(conn)->prepareStatement( + "UPDATE `task_inputs` SET `value` = ?, `data_id` = ? WHERE " + "`output_task_id` = ? AND `output_task_position` = ?" + ) + ); for (size_t i = 0; i < outputs.size(); ++i) { TaskOutput const& output = outputs[i]; std::optional const& value = output.get_value(); @@ -1430,38 +1445,40 @@ auto MySqlMetadataStorage::task_finish( } // Set task states to ready if all inputs are available - std::unique_ptr ready_statement(conn->prepareStatement( - "UPDATE `tasks` SET `state` = 'ready' WHERE `id` IN (SELECT `task_id` FROM " - "`task_inputs` WHERE `output_task_id` = ?) AND `state` = 'pending' AND NOT EXISTS " - "(SELECT `task_id` FROM `task_inputs` WHERE `task_id` IN (SELECT `task_id` FROM " - "`task_inputs` WHERE `output_task_id` = ?) AND `value` IS NULL AND `data_id` IS " - "NULL)" - )); + std::unique_ptr ready_statement( + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `state` = 'ready' WHERE `id` IN (SELECT `task_id` FROM " + "`task_inputs` WHERE `output_task_id` = ?) AND `state` = 'pending' AND NOT " + "EXISTS (SELECT `task_id` FROM `task_inputs` WHERE `task_id` IN (SELECT " + "`task_id` FROM `task_inputs` WHERE `output_task_id` = ?) AND `value` IS " + "NULL AND `data_id` IS NULL)" + ) + ); ready_statement->setBytes(1, &task_id_bytes); ready_statement->setBytes(2, &task_id_bytes); ready_statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::task_fail(TaskInstance const& instance, std::string const& /*error*/) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::task_fail( + StorageConnection& conn, + TaskInstance const& instance, + std::string const& /*error*/ +) -> StorageErr { try { // Remove task instance std::unique_ptr const statement( - conn->prepareStatement("DELETE FROM `task_instances` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "DELETE FROM `task_instances` WHERE `id` = ?" + ) ); sql::bytes instance_id_bytes = uuid_get_bytes(instance.id); statement->setBytes(1, &instance_id_bytes); @@ -1469,7 +1486,9 @@ auto MySqlMetadataStorage::task_fail(TaskInstance const& instance, std::string c // Get number of remaining instances std::unique_ptr const count_statement( - conn->prepareStatement("SELECT COUNT(*) FROM `task_instances` WHERE `task_id` = ?") + static_cast(conn)->prepareStatement( + "SELECT COUNT(*) FROM `task_instances` WHERE `task_id` = ?" + ) ); sql::bytes task_id_bytes = uuid_get_bytes(instance.task_id); count_statement->setBytes(1, &task_id_bytes); @@ -1479,45 +1498,49 @@ auto MySqlMetadataStorage::task_fail(TaskInstance const& instance, std::string c if (count == 0) { // Set the task fail if the last task instance fails std::unique_ptr const task_statement( - conn->prepareStatement("UPDATE `tasks` SET `state` = 'fail' WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "UPDATE `tasks` SET `state` = 'fail' WHERE `id` = ?" + ) ); task_statement->setBytes(1, &task_id_bytes); task_statement->executeUpdate(); } } catch (sql::SQLException& e) { spdlog::error("Task fail error: {}", e.what()); - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_task_timeout(std::vector>* tasks +auto MySqlMetadataStorage::get_task_timeout( + StorageConnection& conn, + std::vector>* tasks ) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); try { - std::unique_ptr statement(conn->createStatement()); + std::unique_ptr statement( + static_cast(conn)->createStatement() + ); std::unique_ptr res(statement->executeQuery( "SELECT `t1`.`id`, `t1`.`task_id` FROM `task_instances` as `t1` JOIN `tasks` ON " - "`t1`.`task_id` = " - "`tasks`.`id` WHERE `tasks`.`timeout` > 0.0001 AND TIMESTAMPDIFF(MICROSECOND, " - "`t1`.`start_time`, CURRENT_TIMESTAMP()) > `tasks`.`timeout` * 1000" + "`t1`.`task_id` = `tasks`.`id` WHERE `tasks`.`timeout` > 0.0001 AND " + "TIMESTAMPDIFF(MICROSECOND, `t1`.`start_time`, CURRENT_TIMESTAMP()) > " + "`tasks`.`timeout` * 1000" )); std::unique_ptr not_timeout_statement( - conn->prepareStatement("SELECT FROM `task_instances` as `t1` JOIN `tasks` ON" - "`t1`.`task_id` = `tasks`.`id` WHERE `t1.task_id` = ? AND " - "TIMESTAMPDIFF(MICROSECOND, `t1`.`start_time`, " - "CURRENT_TIMESTAMP()) < `tasks`.`timeout` * 1000") + static_cast(conn)->prepareStatement( + "SELECT FROM `task_instances` as `t1` JOIN `tasks` ON`t1`.`task_id` = " + "`tasks`.`id` WHERE `t1.task_id` = ? AND TIMESTAMPDIFF(MICROSECOND, " + "`t1`.`start_time`, CURRENT_TIMESTAMP()) < `tasks`.`timeout` * 1000" + ) ); - std::unique_ptr task_statement(conn->prepareStatement( - "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` WHERE `id` = ?" - )); + std::unique_ptr task_statement( + static_cast(conn)->prepareStatement( + "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` WHERE `id` = ?" + ) + ); while (res->next()) { boost::uuids::uuid const task_instance_id = read_id(res->getBinaryStream("id")); boost::uuids::uuid const task_id = read_id(res->getBinaryStream("task_id")); @@ -1533,90 +1556,88 @@ auto MySqlMetadataStorage::get_task_timeout(std::vectorsetBytes(1, &task_id_bytes); std::unique_ptr task_res(task_statement->executeQuery()); if (task_res->next()) { - Task const task = fetch_full_task(conn, task_res); + Task const task = fetch_full_task(static_cast(conn), task_res); tasks->emplace_back(TaskInstance{task_instance_id, task_id}, task); } } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_child_tasks(boost::uuids::uuid id, std::vector* children) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_child_tasks( + StorageConnection& conn, + boost::uuids::uuid id, + std::vector* children +) -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` JOIN " - "`task_dependencies` " - "as `t2` WHERE `tasks`.`id` = `t2`.`child` AND `t2`.`parent` = ?" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` JOIN " + "`task_dependencies` as `t2` WHERE `tasks`.`id` = `t2`.`child` AND " + "`t2`.`parent` = ?" + ) + ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr res(statement->executeQuery()); while (res->next()) { - children->emplace_back(fetch_full_task(conn, res)); + children->emplace_back(fetch_full_task(static_cast(conn), res)); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_parent_tasks(boost::uuids::uuid id, std::vector* tasks) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_parent_tasks( + StorageConnection& conn, + boost::uuids::uuid id, + std::vector* tasks +) -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` JOIN " - "`task_dependencies` " - "as `t2` WHERE `tasks`.`id` = `t2`.`parent` AND `t2`.`child` = ?" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "SELECT `id`, `func_name`, `state`, `timeout` FROM `tasks` JOIN " + "`task_dependencies` as `t2` WHERE `tasks`.`id` = `t2`.`parent` AND " + "`t2`.`child` = ?" + ) + ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr const res(statement->executeQuery()); while (res->next()) { - tasks->emplace_back(fetch_full_task(conn, res)); + tasks->emplace_back(fetch_full_task(static_cast(conn), res)); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::update_heartbeat(boost::uuids::uuid id) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::update_heartbeat(StorageConnection& conn, boost::uuids::uuid id) + -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "UPDATE `drivers` SET `heartbeat` = CURRENT_TIMESTAMP() WHERE `id` = ?" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "UPDATE `drivers` SET `heartbeat` = CURRENT_TIMESTAMP() WHERE `id` = ?" + ) + ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } @@ -1624,47 +1645,47 @@ namespace { constexpr int cMillisecondToMicrosecond = 1000; } // namespace -auto MySqlMetadataStorage::heartbeat_timeout(double timeout, std::vector* ids) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::heartbeat_timeout( + StorageConnection& conn, + double timeout, + std::vector* ids +) -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "SELECT `id` FROM `drivers` WHERE TIMESTAMPDIFF(MICROSECOND, " - "`heartbeat`, CURRENT_TIMESTAMP()) > ?" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "SELECT `id` FROM `drivers` WHERE TIMESTAMPDIFF(MICROSECOND, `heartbeat`, " + "CURRENT_TIMESTAMP()) > ?" + ) + ); statement->setDouble(1, timeout * cMillisecondToMicrosecond); std::unique_ptr res(statement->executeQuery()); while (res->next()) { ids->emplace_back(read_id(res->getBinaryStream("id"))); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_scheduler_state(boost::uuids::uuid id, std::string* state) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_scheduler_state( + StorageConnection& conn, + boost::uuids::uuid id, + std::string* state +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("SELECT `state` FROM `schedulers` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `state` FROM `schedulers` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr res(statement->executeQuery()); if (res->rowsCount() == 0) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format("no scheduler with id {}", boost::uuids::to_string(id)) @@ -1673,29 +1694,30 @@ auto MySqlMetadataStorage::get_scheduler_state(boost::uuids::uuid id, std::strin res->next(); *state = get_sql_string(res->getString(1)); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::get_scheduler_addr(boost::uuids::uuid id, std::string* addr, int* port) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::get_scheduler_addr( + StorageConnection& conn, + boost::uuids::uuid id, + std::string* addr, + int* port +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("SELECT `address`, `port` FROM `schedulers` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `address`, `port` FROM `schedulers` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr res{statement->executeQuery()}; if (res->rowsCount() == 0) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format("no scheduler with id {}", boost::uuids::to_string(id)) @@ -1705,68 +1727,65 @@ auto MySqlMetadataStorage::get_scheduler_addr(boost::uuids::uuid id, std::string *addr = get_sql_string(res->getString(1)); *port = res->getInt(2); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlMetadataStorage::set_scheduler_state(boost::uuids::uuid id, std::string const& state) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlMetadataStorage::set_scheduler_state( + StorageConnection& conn, + boost::uuids::uuid id, + std::string const& state +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("UPDATE `schedulers` SET `state` = ? WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "UPDATE `schedulers` SET `state` = ? WHERE `id` = ?" + ) ); statement->setString(1, state); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(2, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::initialize() -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::initialize(StorageConnection& conn) -> StorageErr { try { // Need to initialize metadata storage first so that foreign constraint is not voilated for (char const* create_table_str : cCreateStorage) { - std::unique_ptr statement(conn->createStatement()); + std::unique_ptr statement( + static_cast(conn)->createStatement() + ); statement->executeUpdate(create_table_str); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::add_driver_data(boost::uuids::uuid const driver_id, Data const& data) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::add_driver_data( + StorageConnection& conn, + boost::uuids::uuid const driver_id, + Data const& data +) -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "INSERT INTO `data` (`id`, `value`, `hard_locality`) VALUES(?, ?, ?)" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `data` (`id`, `value`, `hard_locality`) VALUES(?, ?, ?)" + ) + ); sql::bytes id_bytes = uuid_get_bytes(data.get_id()); statement->setBytes(1, &id_bytes); statement->setString(2, data.get_value()); @@ -1775,42 +1794,46 @@ auto MySqlDataStorage::add_driver_data(boost::uuids::uuid const driver_id, Data for (std::string const& addr : data.get_locality()) { std::unique_ptr locality_statement( - conn->prepareStatement("INSERT INTO `data_locality` (`id`, " - "`address`) VALUES (?, ?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `data_locality` (`id`, " + "`address`) VALUES (?, ?)" + ) ); locality_statement->setBytes(1, &id_bytes); locality_statement->setString(2, addr); locality_statement->executeUpdate(); } - std::unique_ptr driver_ref_statement(conn->prepareStatement( - "INSERT INTO `data_ref_driver` (`id`, `driver_id`) VALUES(?, ?)" - )); + std::unique_ptr driver_ref_statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `data_ref_driver` (`id`, `driver_id`) VALUES(?, ?)" + ) + ); sql::bytes driver_id_bytes = uuid_get_bytes(driver_id); driver_ref_statement->setBytes(1, &id_bytes); driver_ref_statement->setBytes(2, &driver_id_bytes); driver_ref_statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::add_task_data(boost::uuids::uuid const task_id, Data const& data) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::add_task_data( + StorageConnection& conn, + boost::uuids::uuid const task_id, + Data const& data +) -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "INSERT INTO `data` (`id`, `value`, `hard_locality`) VALUES(?, ?, ?)" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `data` (`id`, `value`, `hard_locality`) VALUES(?, ?, ?)" + ) + ); sql::bytes id_bytes = uuid_get_bytes(data.get_id()); statement->setBytes(1, &id_bytes); statement->setString(2, data.get_value()); @@ -1819,47 +1842,47 @@ auto MySqlDataStorage::add_task_data(boost::uuids::uuid const task_id, Data cons for (std::string const& addr : data.get_locality()) { std::unique_ptr locality_statement( - conn->prepareStatement("INSERT INTO `data_locality` (`id`, " - "`address`) VALUES (?, ?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `data_locality` (`id`, `address`) VALUES (?, ?)" + ) ); locality_statement->setBytes(1, &id_bytes); locality_statement->setString(2, addr); locality_statement->executeUpdate(); } std::unique_ptr task_ref_statement( - conn->prepareStatement("INSERT INTO `data_ref_task` (`id`, `task_id`) VALUES(?, ?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `data_ref_task` (`id`, `task_id`) VALUES(?, ?)" + ) ); sql::bytes task_id_bytes = uuid_get_bytes(task_id); task_ref_statement->setBytes(1, &id_bytes); task_ref_statement->setBytes(2, &task_id_bytes); task_ref_statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::get_data(boost::uuids::uuid id, Data* data) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::get_data(StorageConnection& conn, boost::uuids::uuid id, Data* data) + -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("SELECT `id`, `value`, `hard_locality` " - "FROM `data` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `id`, `value`, `hard_locality` FROM `data` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); std::unique_ptr res(statement->executeQuery()); if (res->rowsCount() == 0) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format("no data with id {}", boost::uuids::to_string(id)) @@ -1870,7 +1893,9 @@ auto MySqlDataStorage::get_data(boost::uuids::uuid id, Data* data) -> StorageErr data->set_hard_locality(res->getBoolean(3)); std::unique_ptr locality_statement( - conn->prepareStatement("SELECT `address` FROM `data_locality` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "SELECT `address` FROM `data_locality` WHERE `id` = ?" + ) ); locality_statement->setBytes(1, &id_bytes); std::unique_ptr const locality_res(locality_statement->executeQuery()); @@ -1882,28 +1907,27 @@ auto MySqlDataStorage::get_data(boost::uuids::uuid id, Data* data) -> StorageErr data->set_locality(locality); } } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::set_data_locality(Data const& data) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::set_data_locality(StorageConnection& conn, Data const& data) -> StorageErr { try { std::unique_ptr const delete_statement( - conn->prepareStatement("DELETE FROM `data_locality` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "DELETE FROM `data_locality` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(data.get_id()); delete_statement->setBytes(1, &id_bytes); delete_statement->executeUpdate(); std::unique_ptr const insert_statement( - conn->prepareStatement("INSERT INTO `data_locality` (`id`, `address`) VALUES(?, ?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `data_locality` (`id`, `address`) VALUES(?, ?)" + ) ); for (std::string const& addr : data.get_locality()) { insert_statement->setBytes(1, &id_bytes); @@ -1911,51 +1935,49 @@ auto MySqlDataStorage::set_data_locality(Data const& data) -> StorageErr { insert_statement->executeUpdate(); } std::unique_ptr const hard_locality_statement( - conn->prepareStatement("UPDATE `data` SET `hard_locality` = ? WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "UPDATE `data` SET `hard_locality` = ? WHERE `id` = ?" + ) ); hard_locality_statement->setBoolean(1, data.is_hard_locality()); hard_locality_statement->setBytes(2, &id_bytes); hard_locality_statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::remove_data(boost::uuids::uuid id) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::remove_data(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("DELETE FROM `data` WHERE `id` = ?") + static_cast(conn)->prepareStatement( + "DELETE FROM `data` WHERE `id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::add_task_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid task_id +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("INSERT INTO `data_ref_task` (`id`, " - "`task_id`) VALUES(?, ?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `data_ref_task` (`id`, `task_id`) VALUES(?, ?)" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); @@ -1963,27 +1985,26 @@ auto MySqlDataStorage::add_task_reference(boost::uuids::uuid id, boost::uuids::u statement->setBytes(2, &task_id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::remove_task_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid task_id +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("DELETE FROM `data_ref_task` WHERE " - "`id` = ? AND `task_id` = ?") + static_cast(conn)->prepareStatement( + "DELETE FROM `data_ref_task` WHERE `id` = ? AND `task_id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); @@ -1991,24 +2012,23 @@ auto MySqlDataStorage::remove_task_reference(boost::uuids::uuid id, boost::uuids statement->setBytes(2, &task_id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::add_driver_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid driver_id +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("INSERT INTO `data_ref_driver` (`id`, " - "`driver_id`) VALUES(?, ?)") + static_cast(conn)->prepareStatement( + "INSERT INTO `data_ref_driver` (`id`, `driver_id`) VALUES(?, ?)" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); @@ -2016,27 +2036,26 @@ auto MySqlDataStorage::add_driver_reference(boost::uuids::uuid id, boost::uuids: statement->setBytes(2, &driver_id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) - -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::remove_driver_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid driver_id +) -> StorageErr { try { std::unique_ptr statement( - conn->prepareStatement("DELETE FROM `data_ref_driver` " - "WHERE `id` = ? AND `driver_id` = ?") + static_cast(conn)->prepareStatement( + "DELETE FROM `data_ref_driver` WHERE `id` = ? AND `driver_id` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(id); statement->setBytes(1, &id_bytes); @@ -2044,105 +2063,97 @@ auto MySqlDataStorage::remove_driver_reference(boost::uuids::uuid id, boost::uui statement->setBytes(2, &driver_id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::remove_dangling_data() -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::remove_dangling_data(StorageConnection& conn) -> StorageErr { try { - std::unique_ptr statement{conn->createStatement()}; + std::unique_ptr statement{ + static_cast(conn)->createStatement() + }; statement->execute("DELETE FROM `data` WHERE `id` NOT IN (SELECT driver_ref.`id` FROM " "`data_ref_driver` driver_ref) AND `id` NOT IN (SELECT task_ref.`id` " "FROM `data_ref_task` task_ref)"); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::add_client_kv_data(KeyValueData const& data) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::add_client_kv_data(StorageConnection& conn, KeyValueData const& data) + -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "INSERT INTO `client_kv_data` (`kv_key`, `value`, `client_id`) VALUES(?, ?, ?)" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `client_kv_data` (`kv_key`, `value`, `client_id`) VALUES(?, " + "?, ?)" + ) + ); statement->setString(1, data.get_key()); statement->setString(2, data.get_value()); sql::bytes id_bytes = uuid_get_bytes(data.get_id()); statement->setBytes(3, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } -auto MySqlDataStorage::add_task_kv_data(KeyValueData const& data) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); +auto MySqlDataStorage::add_task_kv_data(StorageConnection& conn, KeyValueData const& data) + -> StorageErr { try { - std::unique_ptr statement(conn->prepareStatement( - "INSERT INTO `task_kv_data` (`kv_key`, `value`, `task_id`) VALUES(?, ?, ?)" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "INSERT INTO `task_kv_data` (`kv_key`, `value`, `task_id`) VALUES(?, ?, ?)" + ) + ); statement->setString(1, data.get_key()); statement->setString(2, data.get_value()); sql::bytes id_bytes = uuid_get_bytes(data.get_id()); statement->setBytes(3, &id_bytes); statement->executeUpdate(); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); if (e.getErrorCode() == ErDupKey || e.getErrorCode() == ErDupEntry) { return StorageErr{StorageErrType::DuplicateKeyErr, e.what()}; } return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } auto MySqlDataStorage::get_client_kv_data( + StorageConnection& conn, boost::uuids::uuid const& client_id, std::string const& key, std::string* value ) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); try { - std::unique_ptr statement(conn->prepareStatement( - "SELECT `value` " - "FROM `client_kv_data` WHERE `client_id` = ? AND `kv_key` = ?" - )); + std::unique_ptr statement( + static_cast(conn)->prepareStatement( + "SELECT `value` FROM `client_kv_data` WHERE `client_id` = ? AND `kv_key` = " + "?" + ) + ); sql::bytes id_bytes = uuid_get_bytes(client_id); statement->setBytes(1, &id_bytes); statement->setString(2, key); std::unique_ptr res(statement->executeQuery()); if (res->rowsCount() == 0) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format( @@ -2155,34 +2166,31 @@ auto MySqlDataStorage::get_client_kv_data( res->next(); *value = get_sql_string(res->getString(1)); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } auto MySqlDataStorage::get_task_kv_data( + StorageConnection& conn, boost::uuids::uuid const& task_id, std::string const& key, std::string* value ) -> StorageErr { - std::variant conn_result = MySqlConnection::create(m_url); - if (std::holds_alternative(conn_result)) { - return std::get(conn_result); - } - auto& conn = std::get(conn_result); try { std::unique_ptr statement( - conn->prepareStatement("SELECT `value` " - "FROM `task_kv_data` WHERE `task_id` = ? AND `kv_key` = ?") + static_cast(conn)->prepareStatement( + "SELECT `value` FROM `task_kv_data` WHERE `task_id` = ? AND `kv_key` = ?" + ) ); sql::bytes id_bytes = uuid_get_bytes(task_id); statement->setBytes(1, &id_bytes); statement->setString(2, key); std::unique_ptr res(statement->executeQuery()); if (res->rowsCount() == 0) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{ StorageErrType::KeyNotFoundErr, fmt::format( @@ -2195,10 +2203,10 @@ auto MySqlDataStorage::get_task_kv_data( res->next(); *value = get_sql_string(res->getString(1)); } catch (sql::SQLException& e) { - conn->rollback(); + static_cast(conn)->rollback(); return StorageErr{StorageErrType::OtherErr, e.what()}; } - conn->commit(); + static_cast(conn)->commit(); return StorageErr{}; } diff --git a/src/spider/storage/MySqlStorage.hpp b/src/spider/storage/MySqlStorage.hpp index f81175f91..5f672383e 100644 --- a/src/spider/storage/MySqlStorage.hpp +++ b/src/spider/storage/MySqlStorage.hpp @@ -21,6 +21,7 @@ #include "DataStorage.hpp" #include "MetadataStorage.hpp" #include "MySqlConnection.hpp" +#include "StorageConnection.hpp" namespace spider::core { class MySqlMetadataStorage : public MetadataStorage { @@ -32,48 +33,88 @@ class MySqlMetadataStorage : public MetadataStorage { auto operator=(MySqlMetadataStorage const&) -> MySqlMetadataStorage& = delete; auto operator=(MySqlMetadataStorage&&) -> MySqlMetadataStorage& = delete; ~MySqlMetadataStorage() override = default; - auto initialize() -> StorageErr override; - auto add_driver(Driver const& driver) -> StorageErr override; - auto add_scheduler(Scheduler const& scheduler) -> StorageErr override; - auto get_active_scheduler(std::vector* schedulers) -> StorageErr override; - auto - add_job(boost::uuids::uuid job_id, boost::uuids::uuid client_id, TaskGraph const& task_graph + auto initialize(StorageConnection& conn) -> StorageErr override; + auto add_driver(StorageConnection& conn, Driver const& driver) -> StorageErr override; + auto add_scheduler(StorageConnection& conn, Scheduler const& scheduler) -> StorageErr override; + auto get_active_scheduler(StorageConnection& conn, std::vector* schedulers) + -> StorageErr override; + auto add_job( + StorageConnection& conn, + boost::uuids::uuid job_id, + boost::uuids::uuid client_id, + TaskGraph const& task_graph + ) -> StorageErr override; + auto get_job_metadata(StorageConnection& conn, boost::uuids::uuid id, JobMetadata* job) + -> StorageErr override; + auto get_job_complete(StorageConnection& conn, boost::uuids::uuid id, bool* complete) + -> StorageErr override; + auto get_job_status(StorageConnection& conn, boost::uuids::uuid id, JobStatus* status) + -> StorageErr override; + auto get_job_output_tasks( + StorageConnection& conn, + boost::uuids::uuid id, + std::vector* task_ids ) -> StorageErr override; - auto get_job_metadata(boost::uuids::uuid id, JobMetadata* job) -> StorageErr override; - auto get_job_complete(boost::uuids::uuid id, bool* complete) -> StorageErr override; - auto get_job_status(boost::uuids::uuid id, JobStatus* status) -> StorageErr override; - auto get_job_output_tasks(boost::uuids::uuid id, std::vector* task_ids) + auto get_task_graph(StorageConnection& conn, boost::uuids::uuid id, TaskGraph* task_graph) -> StorageErr override; - auto get_task_graph(boost::uuids::uuid id, TaskGraph* task_graph) -> StorageErr override; auto get_jobs_by_client_id( + StorageConnection& conn, boost::uuids::uuid client_id, std::vector* job_ids ) -> StorageErr override; - auto remove_job(boost::uuids::uuid id) -> StorageErr override; - auto reset_job(boost::uuids::uuid id) -> StorageErr override; - auto add_child(boost::uuids::uuid parent_id, Task const& child) -> StorageErr override; - auto get_task(boost::uuids::uuid id, Task* task) -> StorageErr override; - auto get_task_job_id(boost::uuids::uuid id, boost::uuids::uuid* job_id) -> StorageErr override; - auto get_ready_tasks(std::vector* tasks) -> StorageErr override; - auto set_task_state(boost::uuids::uuid id, TaskState state) -> StorageErr override; - auto set_task_running(boost::uuids::uuid id) -> StorageErr override; - auto add_task_instance(TaskInstance const& instance) -> StorageErr override; - auto create_task_instance(TaskInstance const& instance) -> StorageErr override; - auto task_finish(TaskInstance const& instance, std::vector const& outputs) + auto remove_job(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; + auto reset_job(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; + auto add_child(StorageConnection& conn, boost::uuids::uuid parent_id, Task const& child) -> StorageErr override; - auto task_fail(TaskInstance const& instance, std::string const& error) -> StorageErr override; - auto get_task_timeout(std::vector>* tasks - ) -> StorageErr override; - auto get_child_tasks(boost::uuids::uuid id, std::vector* children) -> StorageErr override; - auto get_parent_tasks(boost::uuids::uuid id, std::vector* tasks) -> StorageErr override; - auto update_heartbeat(boost::uuids::uuid id) -> StorageErr override; auto - heartbeat_timeout(double timeout, std::vector* ids) -> StorageErr override; - auto get_scheduler_state(boost::uuids::uuid id, std::string* state) -> StorageErr override; - auto - get_scheduler_addr(boost::uuids::uuid id, std::string* addr, int* port) -> StorageErr override; + get_task(StorageConnection& conn, boost::uuids::uuid id, Task* task) -> StorageErr override; + auto get_task_job_id(StorageConnection& conn, boost::uuids::uuid id, boost::uuids::uuid* job_id) + -> StorageErr override; + auto get_ready_tasks(StorageConnection& conn, std::vector* tasks) -> StorageErr override; + auto set_task_state(StorageConnection& conn, boost::uuids::uuid id, TaskState state) + -> StorageErr override; + auto set_task_running(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; auto - set_scheduler_state(boost::uuids::uuid id, std::string const& state) -> StorageErr override; + add_task_instance(StorageConnection& conn, TaskInstance const& instance) -> StorageErr override; + auto create_task_instance(StorageConnection& conn, TaskInstance const& instance) + -> StorageErr override; + auto task_finish( + StorageConnection& conn, + TaskInstance const& instance, + std::vector const& outputs + ) -> StorageErr override; + auto task_fail(StorageConnection& conn, TaskInstance const& instance, std::string const& error) + -> StorageErr override; + auto get_task_timeout( + StorageConnection& conn, + std::vector>* tasks + ) -> StorageErr override; + auto get_child_tasks( + StorageConnection& conn, + boost::uuids::uuid id, + std::vector* children + ) -> StorageErr override; + auto get_parent_tasks(StorageConnection& conn, boost::uuids::uuid id, std::vector* tasks) + -> StorageErr override; + auto update_heartbeat(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; + auto heartbeat_timeout( + StorageConnection& conn, + double timeout, + std::vector* ids + ) -> StorageErr override; + auto get_scheduler_state(StorageConnection& conn, boost::uuids::uuid id, std::string* state) + -> StorageErr override; + auto get_scheduler_addr( + StorageConnection& conn, + boost::uuids::uuid id, + std::string* addr, + int* port + ) -> StorageErr override; + auto set_scheduler_state( + StorageConnection& conn, + boost::uuids::uuid id, + std::string const& state + ) -> StorageErr override; private: std::string m_url; @@ -85,44 +126,58 @@ class MySqlMetadataStorage : public MetadataStorage { class MySqlDataStorage : public DataStorage { public: - MySqlDataStorage() = delete; - explicit MySqlDataStorage(std::string url) : m_url{std::move(url)} {}; + MySqlDataStorage() = default; MySqlDataStorage(MySqlDataStorage const&) = delete; MySqlDataStorage(MySqlDataStorage&&) = delete; auto operator=(MySqlDataStorage const&) -> MySqlDataStorage& = delete; auto operator=(MySqlDataStorage&&) -> MySqlDataStorage& = delete; ~MySqlDataStorage() override = default; - auto initialize() -> StorageErr override; - auto add_driver_data(boost::uuids::uuid driver_id, Data const& data) -> StorageErr override; - auto add_task_data(boost::uuids::uuid task_id, Data const& data) -> StorageErr override; - auto get_data(boost::uuids::uuid id, Data* data) -> StorageErr override; - auto set_data_locality(Data const& data) -> StorageErr override; - auto remove_data(boost::uuids::uuid id) -> StorageErr override; - auto - add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr override; - auto - remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr override; - auto add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) + auto initialize(StorageConnection& conn) -> StorageErr override; + auto add_driver_data(StorageConnection& conn, boost::uuids::uuid driver_id, Data const& data) -> StorageErr override; - auto remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) + auto add_task_data(StorageConnection& conn, boost::uuids::uuid task_id, Data const& data) -> StorageErr override; - auto remove_dangling_data() -> StorageErr override; + auto + get_data(StorageConnection& conn, boost::uuids::uuid id, Data* data) -> StorageErr override; + auto set_data_locality(StorageConnection& conn, Data const& data) -> StorageErr override; + auto remove_data(StorageConnection& conn, boost::uuids::uuid id) -> StorageErr override; + auto add_task_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid task_id + ) -> StorageErr override; + auto remove_task_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid task_id + ) -> StorageErr override; + auto add_driver_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid driver_id + ) -> StorageErr override; + auto remove_driver_reference( + StorageConnection& conn, + boost::uuids::uuid id, + boost::uuids::uuid driver_id + ) -> StorageErr override; + auto remove_dangling_data(StorageConnection& conn) -> StorageErr override; - auto add_client_kv_data(KeyValueData const& data) -> StorageErr override; - auto add_task_kv_data(KeyValueData const& data) -> StorageErr override; + auto + add_client_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr override; + auto add_task_kv_data(StorageConnection& conn, KeyValueData const& data) -> StorageErr override; auto get_client_kv_data( + StorageConnection& conn, boost::uuids::uuid const& client_id, std::string const& key, std::string* value ) -> StorageErr override; auto get_task_kv_data( + StorageConnection& conn, boost::uuids::uuid const& task_id, std::string const& key, std::string* value ) -> StorageErr override; - -private: - std::string m_url; }; } // namespace spider::core diff --git a/src/spider/storage/StorageConnection.hpp b/src/spider/storage/StorageConnection.hpp new file mode 100644 index 000000000..65188879c --- /dev/null +++ b/src/spider/storage/StorageConnection.hpp @@ -0,0 +1,12 @@ +#ifndef SPIDER_STORAGE_STORAGECONNECTION_HPP +#define SPIDER_STORAGE_STORAGECONNECTION_HPP + +namespace spider::core { + +class StorageConnection { + virtual ~StorageConnection() = 0; +}; + +} // namespace spider::core + +#endif From 921d28e0e8d2f248b72bb35af4fcfe800828dcac Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 00:14:17 -0500 Subject: [PATCH 02/21] Add storage connection in scheduler --- src/spider/scheduler/FifoPolicy.cpp | 17 ++++-- src/spider/scheduler/FifoPolicy.hpp | 6 ++- src/spider/scheduler/SchedulerServer.cpp | 10 ++-- src/spider/scheduler/SchedulerServer.hpp | 3 ++ src/spider/scheduler/SchedulerTaskCache.cpp | 4 +- src/spider/scheduler/SchedulerTaskCache.hpp | 4 ++ src/spider/scheduler/scheduler.cpp | 57 +++++++++++++++++---- src/spider/storage/MySqlStorage.hpp | 5 +- 8 files changed, 82 insertions(+), 24 deletions(-) diff --git a/src/spider/scheduler/FifoPolicy.cpp b/src/spider/scheduler/FifoPolicy.cpp index c17fa76e5..153b1e8de 100644 --- a/src/spider/scheduler/FifoPolicy.cpp +++ b/src/spider/scheduler/FifoPolicy.cpp @@ -23,6 +23,7 @@ namespace { auto task_locality_satisfied( std::shared_ptr const& data_store, + spider::core::StorageConnection& conn, spider::core::Task const& task, std::string const& addr ) -> bool { @@ -36,7 +37,7 @@ auto task_locality_satisfied( } boost::uuids::uuid const data_id = optional_data_id.value(); spider::core::Data data; - if (false == data_store->get_data(data_id, &data).success()) { + if (false == data_store->get_data(conn, data_id, &data).success()) { throw std::runtime_error( fmt::format("Data with id {} not exists.", boost::uuids::to_string((data_id))) ); @@ -61,13 +62,16 @@ namespace spider::scheduler { FifoPolicy::FifoPolicy( std::shared_ptr const& metadata_store, - std::shared_ptr const& data_store + std::shared_ptr const& data_store, + core::StorageConnection& conn ) : m_metadata_store{metadata_store}, m_data_store{data_store}, + m_conn{conn}, m_task_cache{ metadata_store, data_store, + conn, [&](std::vector& tasks, boost::uuids::uuid const& worker_id, std::string const& worker_addr) -> std::optional { @@ -81,7 +85,7 @@ auto FifoPolicy::get_next_task( std::string const& worker_addr ) -> std::optional { std::erase_if(tasks, [this, worker_addr](core::Task const& task) -> bool { - return !task_locality_satisfied(m_data_store, task, worker_addr); + return !task_locality_satisfied(m_data_store, m_conn, task, worker_addr); }); if (tasks.empty()) { @@ -99,7 +103,8 @@ auto FifoPolicy::get_next_task( if (optional_job_id.has_value()) { job_id = optional_job_id.value(); } else { - if (false == m_metadata_store->get_task_job_id(task_id, &job_id).success()) { + if (false + == m_metadata_store->get_task_job_id(m_conn, task_id, &job_id).success()) { throw std::runtime_error(fmt::format( "Task with id {} not exists.", boost::uuids::to_string(task_id) @@ -115,7 +120,9 @@ auto FifoPolicy::get_next_task( } core::JobMetadata job_metadata; - if (false == m_metadata_store->get_job_metadata(job_id, &job_metadata).success()) { + if (false + == m_metadata_store->get_job_metadata(m_conn, job_id, &job_metadata).success()) + { throw std::runtime_error(fmt::format( "Job with id {} not exists.", boost::uuids::to_string(job_id) diff --git a/src/spider/scheduler/FifoPolicy.hpp b/src/spider/scheduler/FifoPolicy.hpp index dfebff937..fa6bc63ec 100644 --- a/src/spider/scheduler/FifoPolicy.hpp +++ b/src/spider/scheduler/FifoPolicy.hpp @@ -12,6 +12,7 @@ #include "../core/Task.hpp" #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/StorageConnection.hpp" #include "../utils/LruCache.hpp" #include "SchedulerPolicy.hpp" #include "SchedulerTaskCache.hpp" @@ -22,7 +23,9 @@ class FifoPolicy final : public SchedulerPolicy { public: FifoPolicy( std::shared_ptr const& metadata_store, - std::shared_ptr const& data_store + std::shared_ptr const& data_store, + core::StorageConnection& conn + ); auto schedule_next(boost::uuids::uuid worker_id, std::string const& worker_addr) @@ -37,6 +40,7 @@ class FifoPolicy final : public SchedulerPolicy { std::shared_ptr m_metadata_store; std::shared_ptr m_data_store; + core::StorageConnection& m_conn; SchedulerTaskCache m_task_cache; diff --git a/src/spider/scheduler/SchedulerServer.cpp b/src/spider/scheduler/SchedulerServer.cpp index 380a7deff..48b0278c8 100644 --- a/src/spider/scheduler/SchedulerServer.cpp +++ b/src/spider/scheduler/SchedulerServer.cpp @@ -19,6 +19,7 @@ #include "../io/Serializer.hpp" // IWYU pragma: keep #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/StorageConnection.hpp" #include "../utils/StopToken.hpp" #include "SchedulerMessage.hpp" #include "SchedulerPolicy.hpp" @@ -30,12 +31,14 @@ SchedulerServer::SchedulerServer( std::shared_ptr policy, std::shared_ptr metadata_store, std::shared_ptr data_store, + core::StorageConnection& conn, core::StopToken& stop_token ) : m_port{port}, m_policy{std::move(policy)}, m_metadata_store{std::move(metadata_store)}, m_data_store{std::move(data_store)}, + m_conn{conn}, m_stop_token{stop_token} { boost::asio::co_spawn(m_context, receive_message(), boost::asio::detached); std::lock_guard const lock{m_mutex}; @@ -136,7 +139,8 @@ auto SchedulerServer::process_message(boost::asio::ip::tcp::socket socket // Reset the whole job if the task fails if (request.has_task_id()) { boost::uuids::uuid job_id; - core::StorageErr err = m_metadata_store->get_task_job_id(request.get_task_id(), &job_id); + core::StorageErr err + = m_metadata_store->get_task_job_id(m_conn, request.get_task_id(), &job_id); // It is possible the job is deleted, so we don't need to reset it if (!err.success()) { spdlog::error( @@ -144,7 +148,7 @@ auto SchedulerServer::process_message(boost::asio::ip::tcp::socket socket boost::uuids::to_string(request.get_task_id()) ); } else { - err = m_metadata_store->reset_job(job_id); + err = m_metadata_store->reset_job(m_conn, job_id); if (!err.success()) { spdlog::error("Cannot reset job {}", boost::uuids::to_string(job_id)); co_return; @@ -157,7 +161,7 @@ auto SchedulerServer::process_message(boost::asio::ip::tcp::socket socket ScheduleTaskResponse response{}; if (task_id.has_value()) { core::TaskInstance const instance{task_id.value()}; - core::StorageErr const err = m_metadata_store->create_task_instance(instance); + core::StorageErr const err = m_metadata_store->create_task_instance(m_conn, instance); if (err.success()) { response = ScheduleTaskResponse{task_id.value(), instance.id}; } else { diff --git a/src/spider/scheduler/SchedulerServer.hpp b/src/spider/scheduler/SchedulerServer.hpp index 564cb44cd..de3011243 100644 --- a/src/spider/scheduler/SchedulerServer.hpp +++ b/src/spider/scheduler/SchedulerServer.hpp @@ -8,6 +8,7 @@ #include "../io/BoostAsio.hpp" // IWYU pragma: keep #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/StorageConnection.hpp" #include "../utils/StopToken.hpp" #include "SchedulerPolicy.hpp" @@ -27,6 +28,7 @@ class SchedulerServer { std::shared_ptr policy, std::shared_ptr metadata_store, std::shared_ptr data_store, + core::StorageConnection& conn, core::StopToken& stop_token ); @@ -44,6 +46,7 @@ class SchedulerServer { std::shared_ptr m_policy; std::shared_ptr m_metadata_store; std::shared_ptr m_data_store; + core::StorageConnection& m_conn; boost::asio::io_context m_context; diff --git a/src/spider/scheduler/SchedulerTaskCache.cpp b/src/spider/scheduler/SchedulerTaskCache.cpp index 85ee697b6..7ec559855 100644 --- a/src/spider/scheduler/SchedulerTaskCache.cpp +++ b/src/spider/scheduler/SchedulerTaskCache.cpp @@ -82,12 +82,12 @@ auto SchedulerTaskCache::should_fetch_tasks() -> bool { void SchedulerTaskCache::fetch_ready_tasks() { m_tasks.clear(); std::vector tasks; - m_metadata_store->get_ready_tasks(&tasks); + m_metadata_store->get_ready_tasks(m_conn, &tasks); for (core::Task const& task : tasks) { m_tasks.emplace(std::make_pair(task.get_id(), task)); } std::vector> task_instances; - m_metadata_store->get_task_timeout(&task_instances); + m_metadata_store->get_task_timeout(m_conn, &task_instances); for (auto const& [task_instance, task] : task_instances) { m_tasks.emplace(std::make_pair(task.get_id(), task)); } diff --git a/src/spider/scheduler/SchedulerTaskCache.hpp b/src/spider/scheduler/SchedulerTaskCache.hpp index ec245021c..80e738fae 100644 --- a/src/spider/scheduler/SchedulerTaskCache.hpp +++ b/src/spider/scheduler/SchedulerTaskCache.hpp @@ -15,6 +15,7 @@ #include "../core/Task.hpp" #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/StorageConnection.hpp" namespace spider::scheduler { @@ -23,6 +24,7 @@ class SchedulerTaskCache { SchedulerTaskCache( std::shared_ptr const& metadata_store, std::shared_ptr const& data_store, + core::StorageConnection& conn, std::function( std::vector& tasks, boost::uuids::uuid const& worker_id, @@ -31,6 +33,7 @@ class SchedulerTaskCache { ) : m_metadata_store{metadata_store}, m_data_store{data_store}, + m_conn{conn}, m_get_next_task_function{get_next_task_function} {} auto get_ready_task(boost::uuids::uuid const& worker_id, std::string const& worker_addr) @@ -46,6 +49,7 @@ class SchedulerTaskCache { std::shared_ptr m_metadata_store; std::shared_ptr m_data_store; + core::StorageConnection& m_conn; // NOLINTNEXTLINE(misc-include-cleaner) absl::flat_hash_map> m_tasks; diff --git a/src/spider/scheduler/scheduler.cpp b/src/spider/scheduler/scheduler.cpp index 87b1c766e..a8688c9be 100644 --- a/src/spider/scheduler/scheduler.cpp +++ b/src/spider/scheduler/scheduler.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -23,7 +24,9 @@ #include "../io/BoostAsio.hpp" // IWYU pragma: keep #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/MySqlConnection.hpp" #include "../storage/MySqlStorage.hpp" +#include "../storage/StorageConnection.hpp" #include "../utils/StopToken.hpp" #include "FifoPolicy.hpp" #include "SchedulerPolicy.hpp" @@ -69,6 +72,7 @@ auto parse_args(int const argc, char** argv) -> boost::program_options::variable auto heartbeat_loop( std::shared_ptr const& metadata_store, + std::string const& storage_url, spider::core::Scheduler const& scheduler, spider::core::StopToken& stop_token ) -> void { @@ -76,7 +80,19 @@ auto heartbeat_loop( while (!stop_token.stop_requested()) { std::this_thread::sleep_for(std::chrono::seconds(1)); spdlog::debug("Updating heartbeat"); - spider::core::StorageErr const err = metadata_store->update_heartbeat(scheduler.get_id()); + std::variant conn_result + = spider::core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + spdlog::error( + "Failed to connection to storage: {}", + std::get(conn_result).description + ); + fail_count++; + continue; + } + spider::core::MySqlConnection& conn = std::get(conn_result); + spider::core::StorageErr const err + = metadata_store->update_heartbeat(conn, scheduler.get_id()); if (!err.success()) { spdlog::error("Failed to update scheduler heartbeat: {}", err.description); fail_count++; @@ -93,21 +109,32 @@ auto heartbeat_loop( auto cleanup_loop( std::shared_ptr const& metadata_store, std::shared_ptr const& data_store, + std::string const& storage_url, spider::core::Scheduler const& scheduler, spider::core::StopToken& stop_token ) -> void { while (!stop_token.stop_requested()) { std::this_thread::sleep_for(std::chrono::seconds(cCleanupInterval)); spdlog::debug("Starting cleanup"); + std::variant conn_result + = spider::core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + spdlog::error( + "Failed to connection to storage: {}", + std::get(conn_result).description + ); + continue; + } + spider::core::MySqlConnection& conn = std::get(conn_result); spider::core::StorageErr err - = metadata_store->set_scheduler_state(scheduler.get_id(), "gc"); + = metadata_store->set_scheduler_state(conn, scheduler.get_id(), "gc"); if (!err.success()) { spdlog::error("Failed to set scheduler state to gc: {}", err.description); continue; } - data_store->remove_dangling_data(); + data_store->remove_dangling_data(conn); for (size_t i = 0; i < cRetryCount; ++i) { - err = metadata_store->set_scheduler_state(scheduler.get_id(), "normal"); + err = metadata_store->set_scheduler_state(conn, scheduler.get_id(), "normal"); if (!err.success()) { spdlog::error("Failed to set scheduler state to normal: {}", err.description); if (i >= cRetryCount - 1) { @@ -161,17 +188,27 @@ auto main(int argc, char** argv) -> int { // Create storages std::shared_ptr const metadata_store - = std::make_shared(storage_url); + = std::make_shared(); std::shared_ptr const data_store - = std::make_shared(storage_url); + = std::make_shared(); // Initialize storages - spider::core::StorageErr err = metadata_store->initialize(); + std::variant conn_result + = spider::core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + spdlog::error( + "Failed to connection to storage: {}", + std::get(conn_result).description + ); + } + spider::core::MySqlConnection& conn = std::get(conn_result); + + spider::core::StorageErr err = metadata_store->initialize(conn); if (!err.success()) { spdlog::error("Failed to initialize metadata storage: {}", err.description); return cStorageErr; } - err = data_store->initialize(); + err = data_store->initialize(conn); if (!err.success()) { spdlog::error("Failed to initialize data storage: {}", err.description); return cStorageErr; @@ -189,7 +226,7 @@ auto main(int argc, char** argv) -> int { // Register scheduler with storage spider::core::Scheduler const scheduler{scheduler_id, scheduler_addr, port}; - err = metadata_store->add_scheduler(scheduler); + err = metadata_store->add_scheduler(conn, scheduler); if (!err.success()) { spdlog::error("Failed to register scheduler with storage server: {}", err.description); return cStorageErr; @@ -200,6 +237,7 @@ auto main(int argc, char** argv) -> int { std::thread heartbeat_thread{ heartbeat_loop, std::cref(metadata_store), + std::cref(storage_url), std::ref(scheduler), std::ref(stop_token), }; @@ -209,6 +247,7 @@ auto main(int argc, char** argv) -> int { cleanup_loop, std::cref(metadata_store), std::cref(data_store), + std::cref(storage_url), std::cref(scheduler), std::ref(stop_token) }; diff --git a/src/spider/storage/MySqlStorage.hpp b/src/spider/storage/MySqlStorage.hpp index 5f672383e..39df6b231 100644 --- a/src/spider/storage/MySqlStorage.hpp +++ b/src/spider/storage/MySqlStorage.hpp @@ -26,8 +26,7 @@ namespace spider::core { class MySqlMetadataStorage : public MetadataStorage { public: - MySqlMetadataStorage() = delete; - explicit MySqlMetadataStorage(std::string url) : m_url{std::move(url)} {}; + MySqlMetadataStorage() = default; MySqlMetadataStorage(MySqlMetadataStorage const&) = delete; MySqlMetadataStorage(MySqlMetadataStorage&&) = delete; auto operator=(MySqlMetadataStorage const&) -> MySqlMetadataStorage& = delete; @@ -117,8 +116,6 @@ class MySqlMetadataStorage : public MetadataStorage { ) -> StorageErr override; private: - std::string m_url; - static void add_task(MySqlConnection& conn, sql::bytes job_id, Task const& task); static auto fetch_full_task(MySqlConnection& conn, std::unique_ptr const& res) -> Task; From 0eec583f661c4a932c6465784ee9d7cefc04183f Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 00:25:45 -0500 Subject: [PATCH 03/21] Add storage connection in worker --- src/spider/worker/WorkerClient.cpp | 22 ++++++++-- src/spider/worker/WorkerClient.hpp | 5 ++- src/spider/worker/worker.cpp | 65 +++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 14 deletions(-) diff --git a/src/spider/worker/WorkerClient.cpp b/src/spider/worker/WorkerClient.cpp index 4d0c886bd..788891b98 100644 --- a/src/spider/worker/WorkerClient.cpp +++ b/src/spider/worker/WorkerClient.cpp @@ -12,6 +12,7 @@ #include #include +#include #include "../core/Driver.hpp" #include "../io/BoostAsio.hpp" // IWYU pragma: keep @@ -20,6 +21,8 @@ #include "../scheduler/SchedulerMessage.hpp" #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/MySqlConnection.hpp" +#include "../storage/StorageConnection.hpp" namespace spider::worker { @@ -27,18 +30,31 @@ WorkerClient::WorkerClient( boost::uuids::uuid const worker_id, std::string worker_addr, std::shared_ptr data_store, - std::shared_ptr metadata_store + std::shared_ptr metadata_store, + std::string const& storage_url ) : m_worker_id{worker_id}, m_worker_addr{std::move(worker_addr)}, m_data_store(std::move(data_store)), - m_metadata_store(std::move(metadata_store)) {} + m_metadata_store(std::move(metadata_store)), + m_storage_url{storage_url} {} auto WorkerClient::get_next_task(std::optional const& fail_task_id ) -> std::optional> { // Get schedulers std::vector schedulers; - if (!m_metadata_store->get_active_scheduler(&schedulers).success()) { + + std::variant conn_result + = spider::core::MySqlConnection::create(m_storage_url); + if (std::holds_alternative(conn_result)) { + spdlog::error( + "Failed to connection to storage: {}", + std::get(conn_result).description + ); + } + spider::core::MySqlConnection& conn = std::get(conn_result); + + if (!m_metadata_store->get_active_scheduler(conn, &schedulers).success()) { return std::nullopt; } if (schedulers.empty()) { diff --git a/src/spider/worker/WorkerClient.hpp b/src/spider/worker/WorkerClient.hpp index 523869a7b..3fd7fcdd7 100644 --- a/src/spider/worker/WorkerClient.hpp +++ b/src/spider/worker/WorkerClient.hpp @@ -26,7 +26,8 @@ class WorkerClient { boost::uuids::uuid worker_id, std::string worker_addr, std::shared_ptr data_store, - std::shared_ptr metadata_store + std::shared_ptr metadata_store, + std::string const& storage_url ); auto get_next_task(std::optional const& fail_task_id @@ -38,6 +39,8 @@ class WorkerClient { std::shared_ptr m_data_store; std::shared_ptr m_metadata_store; + + std::string m_storage_url; }; } // namespace spider::worker #endif // SPIDER_WORKER_WORKERCLIENT_HPP diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index 956548666..8986690b7 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -37,7 +38,9 @@ #include "../io/Serializer.hpp" // IWYU pragma: keep #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/MySqlConnection.hpp" #include "../storage/MySqlStorage.hpp" +#include "../storage/StorageConnection.hpp" #include "../utils/StopToken.hpp" #include "TaskExecutor.hpp" #include "WorkerClient.hpp" @@ -99,6 +102,7 @@ auto get_environment_variable() -> absl::flat_hash_map< auto heartbeat_loop( std::shared_ptr const& metadata_store, + std::string const& storage_url, spider::core::Driver const& driver, spider::core::StopToken& stop_token ) -> void { @@ -106,7 +110,20 @@ auto heartbeat_loop( while (!stop_token.stop_requested()) { std::this_thread::sleep_for(std::chrono::seconds(1)); spdlog::debug("Updating heartbeat"); - spider::core::StorageErr const err = metadata_store->update_heartbeat(driver.get_id()); + std::variant conn_result + = spider::core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + spdlog::error( + "Failed to connection to storage: {}", + std::get(conn_result).description + ); + fail_count++; + continue; + } + spider::core::MySqlConnection& conn = std::get(conn_result); + + spider::core::StorageErr const err + = metadata_store->update_heartbeat(conn, driver.get_id()); if (!err.success()) { spdlog::error("Failed to update scheduler heartbeat: {}", err.description); fail_count++; @@ -214,15 +231,26 @@ auto task_loop( spider::core::StopToken const& stop_token ) -> void { std::optional fail_task_id = std::nullopt; + boost::asio::io_context context; while (!stop_token.stop_requested()) { - boost::asio::io_context context; + std::variant conn_result + = spider::core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + spdlog::error( + "Failed to connection to storage: {}", + std::get(conn_result).description + ); + continue; + } + spider::core::MySqlConnection& conn = std::get(conn_result); + auto const [task_id, task_instance_id] = fetch_task(client, fail_task_id); spider::core::TaskInstance const instance{task_instance_id, task_id}; spdlog::debug("Fetched task {}", boost::uuids::to_string(task_id)); fail_task_id = std::nullopt; // Fetch task detail from metadata storage spider::core::Task task{""}; - spider::core::StorageErr err = metadata_store->get_task(task_id, &task); + spider::core::StorageErr err = metadata_store->get_task(conn, task_id, &task); if (!err.success()) { spdlog::error("Failed to fetch task detail: {}", err.description); continue; @@ -233,6 +261,7 @@ auto task_loop( = get_args_buffers(task); if (!optional_args_buffers.has_value()) { metadata_store->task_fail( + conn, instance, fmt::format("Task {} failed to parse arguments", task.get_function_name()) ); @@ -257,6 +286,7 @@ auto task_loop( if (!executor.succeed()) { spdlog::warn("Task {} failed", task.get_function_name()); metadata_store->task_fail( + conn, instance, fmt::format("Task {} failed", task.get_function_name()) ); @@ -270,6 +300,7 @@ auto task_loop( if (!optional_result_buffers.has_value()) { spdlog::error("Task {} failed to parse result into buffers", task.get_function_name()); metadata_store->task_fail( + conn, instance, fmt::format( "Task {} failed to parse result into buffers", @@ -284,6 +315,7 @@ auto task_loop( = parse_outputs(task, result_buffers); if (!optional_outputs.has_value()) { metadata_store->task_fail( + conn, instance, fmt::format( "Task {} failed to parse result into TaskOutput", @@ -296,7 +328,7 @@ auto task_loop( std::vector const& outputs = optional_outputs.value(); // Submit result spdlog::debug("Submitting result for task {}", boost::uuids::to_string(task_id)); - err = metadata_store->task_finish(instance, outputs); + err = metadata_store->task_finish(conn, instance, outputs); fail_task_id = std::nullopt; if (!err.success()) { spdlog::error("Submit task {} fails: {}", task.get_function_name(), err.description); @@ -348,17 +380,30 @@ auto main(int argc, char** argv) -> int { // Create storage std::shared_ptr const metadata_store - = std::make_shared(storage_url); + = std::make_shared(); std::shared_ptr const data_store - = std::make_shared(storage_url); + = std::make_shared(); boost::uuids::random_generator gen; boost::uuids::uuid const worker_id = gen(); spider::core::Driver driver{worker_id}; - spider::core::StorageErr const err = metadata_store->add_driver(driver); - if (!err.success()) { - spdlog::error("Cannot add driver to metadata storage: {}", err.description); - return cStorageErr; + + { // Keep the scope of RAII storage connection + std::variant conn_result + = spider::core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + spdlog::error( + "Failed to connection to storage: {}", + std::get(conn_result).description + ); + return cStorageErr; + } + spider::core::MySqlConnection& conn = std::get(conn_result); + spider::core::StorageErr const err = metadata_store->add_driver(conn, driver); + if (!err.success()) { + spdlog::error("Cannot add driver to metadata storage: {}", err.description); + return cStorageErr; + } } spider::core::StopToken stop_token; From a33933337bf065e4f378802d02b2ee1ff25b358e Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 00:28:17 -0500 Subject: [PATCH 04/21] Fix storage connection virtual destructor --- src/spider/storage/MySqlConnection.hpp | 2 +- src/spider/storage/StorageConnection.hpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/spider/storage/MySqlConnection.hpp b/src/spider/storage/MySqlConnection.hpp index 4df1d03fa..1bfc65001 100644 --- a/src/spider/storage/MySqlConnection.hpp +++ b/src/spider/storage/MySqlConnection.hpp @@ -25,7 +25,7 @@ class MySqlConnection : public StorageConnection { MySqlConnection(MySqlConnection&&) = default; auto operator=(MySqlConnection&&) -> MySqlConnection& = default; - ~MySqlConnection() override; + ~MySqlConnection(); auto operator*() const -> sql::Connection&; auto operator->() const -> sql::Connection*; diff --git a/src/spider/storage/StorageConnection.hpp b/src/spider/storage/StorageConnection.hpp index 65188879c..0de5dcaec 100644 --- a/src/spider/storage/StorageConnection.hpp +++ b/src/spider/storage/StorageConnection.hpp @@ -4,7 +4,6 @@ namespace spider::core { class StorageConnection { - virtual ~StorageConnection() = 0; }; } // namespace spider::core From 63b0724f7d54cbc68606652d37b2fb735aac7265 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 00:50:59 -0500 Subject: [PATCH 05/21] Add url in metadata store and data store --- src/spider/scheduler/scheduler.cpp | 16 ++++++---------- src/spider/storage/DataStorage.hpp | 2 ++ src/spider/storage/MetadataStorage.hpp | 2 ++ src/spider/storage/MySqlStorage.hpp | 15 +++++++++++++-- src/spider/worker/WorkerClient.cpp | 8 +++----- src/spider/worker/WorkerClient.hpp | 5 +---- src/spider/worker/worker.cpp | 13 +++++-------- 7 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/spider/scheduler/scheduler.cpp b/src/spider/scheduler/scheduler.cpp index a8688c9be..fc68944ae 100644 --- a/src/spider/scheduler/scheduler.cpp +++ b/src/spider/scheduler/scheduler.cpp @@ -72,7 +72,6 @@ auto parse_args(int const argc, char** argv) -> boost::program_options::variable auto heartbeat_loop( std::shared_ptr const& metadata_store, - std::string const& storage_url, spider::core::Scheduler const& scheduler, spider::core::StopToken& stop_token ) -> void { @@ -81,7 +80,7 @@ auto heartbeat_loop( std::this_thread::sleep_for(std::chrono::seconds(1)); spdlog::debug("Updating heartbeat"); std::variant conn_result - = spider::core::MySqlConnection::create(storage_url); + = spider::core::MySqlConnection::create(metadata_store->get_url()); if (std::holds_alternative(conn_result)) { spdlog::error( "Failed to connection to storage: {}", @@ -109,7 +108,6 @@ auto heartbeat_loop( auto cleanup_loop( std::shared_ptr const& metadata_store, std::shared_ptr const& data_store, - std::string const& storage_url, spider::core::Scheduler const& scheduler, spider::core::StopToken& stop_token ) -> void { @@ -117,7 +115,7 @@ auto cleanup_loop( std::this_thread::sleep_for(std::chrono::seconds(cCleanupInterval)); spdlog::debug("Starting cleanup"); std::variant conn_result - = spider::core::MySqlConnection::create(storage_url); + = spider::core::MySqlConnection::create(metadata_store->get_url()); if (std::holds_alternative(conn_result)) { spdlog::error( "Failed to connection to storage: {}", @@ -188,9 +186,9 @@ auto main(int argc, char** argv) -> int { // Create storages std::shared_ptr const metadata_store - = std::make_shared(); + = std::make_shared(storage_url); std::shared_ptr const data_store - = std::make_shared(); + = std::make_shared(storage_url); // Initialize storages std::variant conn_result @@ -221,8 +219,8 @@ auto main(int argc, char** argv) -> int { // Start scheduler server spider::core::StopToken stop_token; std::shared_ptr const policy - = std::make_shared(metadata_store, data_store); - spider::scheduler::SchedulerServer server{port, policy, metadata_store, data_store, stop_token}; + = std::make_shared(metadata_store, data_store, conn); + spider::scheduler::SchedulerServer server{port, policy, metadata_store, data_store, conn, stop_token}; // Register scheduler with storage spider::core::Scheduler const scheduler{scheduler_id, scheduler_addr, port}; @@ -237,7 +235,6 @@ auto main(int argc, char** argv) -> int { std::thread heartbeat_thread{ heartbeat_loop, std::cref(metadata_store), - std::cref(storage_url), std::ref(scheduler), std::ref(stop_token), }; @@ -247,7 +244,6 @@ auto main(int argc, char** argv) -> int { cleanup_loop, std::cref(metadata_store), std::cref(data_store), - std::cref(storage_url), std::cref(scheduler), std::ref(stop_token) }; diff --git a/src/spider/storage/DataStorage.hpp b/src/spider/storage/DataStorage.hpp index 7d221581c..ca39ee2b8 100644 --- a/src/spider/storage/DataStorage.hpp +++ b/src/spider/storage/DataStorage.hpp @@ -74,6 +74,8 @@ class DataStorage { std::string const& key, std::string* value ) -> StorageErr = 0; + + virtual auto get_url() const -> std::string const& = 0 }; } // namespace spider::core diff --git a/src/spider/storage/MetadataStorage.hpp b/src/spider/storage/MetadataStorage.hpp index 1ff5e8bf5..f2be8fc12 100644 --- a/src/spider/storage/MetadataStorage.hpp +++ b/src/spider/storage/MetadataStorage.hpp @@ -126,6 +126,8 @@ class MetadataStorage { boost::uuids::uuid id, std::string const& state ) -> StorageErr = 0; + + virtual auto get_url() const -> std::string const& = 0; }; } // namespace spider::core diff --git a/src/spider/storage/MySqlStorage.hpp b/src/spider/storage/MySqlStorage.hpp index 39df6b231..44ce37a60 100644 --- a/src/spider/storage/MySqlStorage.hpp +++ b/src/spider/storage/MySqlStorage.hpp @@ -26,7 +26,8 @@ namespace spider::core { class MySqlMetadataStorage : public MetadataStorage { public: - MySqlMetadataStorage() = default; + MySqlMetadataStorage() = delete; + explicit MySqlMetadataStorage(std::string url): m_url{std::move(url)} {} MySqlMetadataStorage(MySqlMetadataStorage const&) = delete; MySqlMetadataStorage(MySqlMetadataStorage&&) = delete; auto operator=(MySqlMetadataStorage const&) -> MySqlMetadataStorage& = delete; @@ -115,7 +116,11 @@ class MySqlMetadataStorage : public MetadataStorage { std::string const& state ) -> StorageErr override; + auto get_url() const -> std::string const& override { return m_url; } + private: + std::string m_url; + static void add_task(MySqlConnection& conn, sql::bytes job_id, Task const& task); static auto fetch_full_task(MySqlConnection& conn, std::unique_ptr const& res) -> Task; @@ -123,7 +128,8 @@ class MySqlMetadataStorage : public MetadataStorage { class MySqlDataStorage : public DataStorage { public: - MySqlDataStorage() = default; + MySqlDataStorage() = delete; + explicit MySqlDataStorage(std::string url): m_url{std::move(url)} {} MySqlDataStorage(MySqlDataStorage const&) = delete; MySqlDataStorage(MySqlDataStorage&&) = delete; auto operator=(MySqlDataStorage const&) -> MySqlDataStorage& = delete; @@ -175,6 +181,11 @@ class MySqlDataStorage : public DataStorage { std::string const& key, std::string* value ) -> StorageErr override; + + auto get_url() const -> std::string const& override { return m_url; } + +private: + std::string m_url; }; } // namespace spider::core diff --git a/src/spider/worker/WorkerClient.cpp b/src/spider/worker/WorkerClient.cpp index 788891b98..cc5aedff2 100644 --- a/src/spider/worker/WorkerClient.cpp +++ b/src/spider/worker/WorkerClient.cpp @@ -30,14 +30,12 @@ WorkerClient::WorkerClient( boost::uuids::uuid const worker_id, std::string worker_addr, std::shared_ptr data_store, - std::shared_ptr metadata_store, - std::string const& storage_url + std::shared_ptr metadata_store ) : m_worker_id{worker_id}, m_worker_addr{std::move(worker_addr)}, m_data_store(std::move(data_store)), - m_metadata_store(std::move(metadata_store)), - m_storage_url{storage_url} {} + m_metadata_store(std::move(metadata_store)) {} auto WorkerClient::get_next_task(std::optional const& fail_task_id ) -> std::optional> { @@ -45,7 +43,7 @@ auto WorkerClient::get_next_task(std::optional const& fail_t std::vector schedulers; std::variant conn_result - = spider::core::MySqlConnection::create(m_storage_url); + = spider::core::MySqlConnection::create(m_metadata_store->get_url()); if (std::holds_alternative(conn_result)) { spdlog::error( "Failed to connection to storage: {}", diff --git a/src/spider/worker/WorkerClient.hpp b/src/spider/worker/WorkerClient.hpp index 3fd7fcdd7..523869a7b 100644 --- a/src/spider/worker/WorkerClient.hpp +++ b/src/spider/worker/WorkerClient.hpp @@ -26,8 +26,7 @@ class WorkerClient { boost::uuids::uuid worker_id, std::string worker_addr, std::shared_ptr data_store, - std::shared_ptr metadata_store, - std::string const& storage_url + std::shared_ptr metadata_store ); auto get_next_task(std::optional const& fail_task_id @@ -39,8 +38,6 @@ class WorkerClient { std::shared_ptr m_data_store; std::shared_ptr m_metadata_store; - - std::string m_storage_url; }; } // namespace spider::worker #endif // SPIDER_WORKER_WORKERCLIENT_HPP diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index 8986690b7..d91859de3 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -102,7 +102,6 @@ auto get_environment_variable() -> absl::flat_hash_map< auto heartbeat_loop( std::shared_ptr const& metadata_store, - std::string const& storage_url, spider::core::Driver const& driver, spider::core::StopToken& stop_token ) -> void { @@ -111,7 +110,7 @@ auto heartbeat_loop( std::this_thread::sleep_for(std::chrono::seconds(1)); spdlog::debug("Updating heartbeat"); std::variant conn_result - = spider::core::MySqlConnection::create(storage_url); + = spider::core::MySqlConnection::create(metadata_store->get_url()); if (std::holds_alternative(conn_result)) { spdlog::error( "Failed to connection to storage: {}", @@ -223,7 +222,6 @@ auto parse_outputs( auto task_loop( std::shared_ptr const& metadata_store, spider::worker::WorkerClient& client, - std::string const& storage_url, std::vector const& libs, absl::flat_hash_map< boost::process::v2::environment::key, @@ -234,7 +232,7 @@ auto task_loop( boost::asio::io_context context; while (!stop_token.stop_requested()) { std::variant conn_result - = spider::core::MySqlConnection::create(storage_url); + = spider::core::MySqlConnection::create(metadata_store->get_url()); if (std::holds_alternative(conn_result)) { spdlog::error( "Failed to connection to storage: {}", @@ -274,7 +272,7 @@ auto task_loop( context, task.get_function_name(), task.get_id(), - storage_url, + metadata_store->get_url(), libs, environment, args_buffers @@ -380,9 +378,9 @@ auto main(int argc, char** argv) -> int { // Create storage std::shared_ptr const metadata_store - = std::make_shared(); + = std::make_shared(storage_url); std::shared_ptr const data_store - = std::make_shared(); + = std::make_shared(storage_url); boost::uuids::random_generator gen; boost::uuids::uuid const worker_id = gen(); @@ -429,7 +427,6 @@ auto main(int argc, char** argv) -> int { task_loop, std::cref(metadata_store), std::ref(client), - std::cref(storage_url), std::cref(libs), std::cref(environment_variables), std::cref(stop_token), From 11002372c65186871a43da223563e4cb1e4924a6 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 01:14:19 -0500 Subject: [PATCH 06/21] Add storage connection in client --- src/spider/client/Data.hpp | 19 +++++++-- src/spider/client/Driver.cpp | 54 +++++++++++++++++++++--- src/spider/client/Driver.hpp | 17 +++++++- src/spider/client/Job.hpp | 38 +++++++++++++---- src/spider/client/TaskContext.cpp | 27 ++++++++++-- src/spider/client/TaskContext.hpp | 9 +++- src/spider/scheduler/scheduler.cpp | 3 +- src/spider/storage/MySqlStorage.hpp | 8 +++- src/spider/storage/StorageConnection.hpp | 3 +- 9 files changed, 150 insertions(+), 28 deletions(-) diff --git a/src/spider/client/Data.hpp b/src/spider/client/Data.hpp index c3249ac80..2abab79af 100644 --- a/src/spider/client/Data.hpp +++ b/src/spider/client/Data.hpp @@ -14,6 +14,7 @@ #include "../io/MsgPack.hpp" // IWYU pragma: keep #include "../io/Serializer.hpp" #include "../storage/DataStorage.hpp" +#include "../storage/MySqlConnection.hpp" #include "Exception.hpp" namespace spider { @@ -64,7 +65,13 @@ class Data { void set_locality(std::vector const& nodes, bool hard) { m_impl->set_locality(nodes); m_impl->set_hard_locality(hard); - m_data_store->set_data_locality(*m_impl); + std::variant conn_result + = core::MySqlConnection::create(m_data_store->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + m_data_store->set_data_locality(conn, *m_impl); } class Builder { @@ -108,16 +115,22 @@ class Data { auto data = std::make_unique(std::string{buffer.data(), buffer.size()}); data->set_locality(m_nodes); data->set_hard_locality(m_hard_locality); + std::variant conn_result + = core::MySqlConnection::create(m_data_store->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); core::StorageErr err; switch (m_data_source) { case DataSource::Driver: - err = m_data_store->add_driver_data(m_source_id, *data); + err = m_data_store->add_driver_data(conn, m_source_id, *data); if (!err.success()) { throw ConnectionException(err.description); } break; case DataSource::TaskContext: - err = m_data_store->add_task_data(m_source_id, *data); + err = m_data_store->add_task_data(conn, m_source_id, *data); if (!err.success()) { throw ConnectionException(err.description); } diff --git a/src/spider/client/Driver.cpp b/src/spider/client/Driver.cpp index 9ad30469e..885e9f3fb 100644 --- a/src/spider/client/Driver.cpp +++ b/src/spider/client/Driver.cpp @@ -26,7 +26,14 @@ Driver::Driver(std::string const& storage_url) { m_metadata_storage = std::make_shared(storage_url); m_data_storage = std::make_shared(storage_url); - core::StorageErr const err = m_metadata_storage->add_driver(core::Driver{m_id}); + std::variant conn_result + = core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + + core::StorageErr const err = m_metadata_storage->add_driver(conn, core::Driver{m_id}); if (!err.success()) { if (core::StorageErrType::DuplicateKeyErr == err.type) { throw DriverIdInUseException(m_id); @@ -39,7 +46,14 @@ Driver::Driver(std::string const& storage_url) { m_heartbeat_thread = std::jthread([this](std::stop_token stoken) { while (!stoken.stop_requested()) { std::this_thread::sleep_for(std::chrono::seconds(1)); - core::StorageErr const err = m_metadata_storage->update_heartbeat(m_id); + std::variant conn_result + = core::MySqlConnection::create(m_metadata_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + + core::StorageErr const err = m_metadata_storage->update_heartbeat(conn, m_id); if (!err.success()) { throw ConnectionException(err.description); } @@ -50,8 +64,14 @@ Driver::Driver(std::string const& storage_url) { Driver::Driver(std::string const& storage_url, boost::uuids::uuid const id) : m_id{id} { m_metadata_storage = std::make_shared(storage_url); m_data_storage = std::make_shared(storage_url); + std::variant conn_result + = core::MySqlConnection::create(storage_url); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); - core::StorageErr const err = m_metadata_storage->add_driver(core::Driver{m_id}); + core::StorageErr const err = m_metadata_storage->add_driver(conn, core::Driver{m_id}); if (!err.success()) { if (core::StorageErrType::DuplicateKeyErr == err.type) { throw DriverIdInUseException(m_id); @@ -64,7 +84,14 @@ Driver::Driver(std::string const& storage_url, boost::uuids::uuid const id) : m_ m_heartbeat_thread = std::jthread([this](std::stop_token stoken) { while (!stoken.stop_requested()) { std::this_thread::sleep_for(std::chrono::seconds(1)); - core::StorageErr const err = m_metadata_storage->update_heartbeat(m_id); + std::variant conn_result + = core::MySqlConnection::create(m_metadata_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + + core::StorageErr const err = m_metadata_storage->update_heartbeat(conn, m_id); if (!err.success()) { throw ConnectionException(err.description); } @@ -74,15 +101,30 @@ Driver::Driver(std::string const& storage_url, boost::uuids::uuid const id) : m_ auto Driver::kv_store_insert(std::string const& key, std::string const& value) -> void { core::KeyValueData const kv_data{key, value, m_id}; - core::StorageErr const err = m_data_storage->add_client_kv_data(kv_data); + + std::variant conn_result + = core::MySqlConnection::create(m_data_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + + core::StorageErr const err = m_data_storage->add_client_kv_data(conn, kv_data); if (!err.success()) { throw ConnectionException(err.description); } } auto Driver::kv_store_get(std::string const& key) -> std::optional { + std::variant conn_result + = core::MySqlConnection::create(m_data_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + std::string value; - core::StorageErr const err = m_data_storage->get_client_kv_data(m_id, key, &value); + core::StorageErr const err = m_data_storage->get_client_kv_data(conn, m_id, key, &value); if (!err.success()) { if (core::StorageErrType::KeyNotFoundErr == err.type) { return std::nullopt; diff --git a/src/spider/client/Driver.hpp b/src/spider/client/Driver.hpp index bd2353cf7..0b66a26b9 100644 --- a/src/spider/client/Driver.hpp +++ b/src/spider/client/Driver.hpp @@ -173,7 +173,13 @@ class Driver { graph.add_task(new_task); graph.add_input_task(new_task.get_id()); graph.add_output_task(new_task.get_id()); - core::StorageErr err = m_metadata_storage->add_job(job_id, m_id, graph); + std::variant conn_result + = core::MySqlConnection::create(m_metadata_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + core::StorageErr err = m_metadata_storage->add_job(conn, job_id, m_id, graph); if (!err.success()) { throw ConnectionException(fmt::format("Failed to start job: {}", err.description)); } @@ -235,7 +241,14 @@ class Driver { */ auto get_jobs() -> std::vector { std::vector job_ids; - core::StorageErr const err = m_metadata_storage->get_jobs_by_client_id(m_id, &job_ids); + std::variant conn_result + = core::MySqlConnection::create(m_metadata_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + core::StorageErr const err + = m_metadata_storage->get_jobs_by_client_id(conn, m_id, &job_ids); if (!err.success()) { throw ConnectionException("Failed to get jobs."); } diff --git a/src/spider/client/Job.hpp b/src/spider/client/Job.hpp index f593a63ee..e8a0f8fab 100644 --- a/src/spider/client/Job.hpp +++ b/src/spider/client/Job.hpp @@ -60,8 +60,15 @@ class Job { * @throw spider::ConnectionException */ auto wait_complete() -> void { + std::variant conn_result + = core::MySqlConnection::create(m_data_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + bool complete = false; - core::StorageErr err = m_metadata_storage->get_job_complete(m_id, &complete); + core::StorageErr err = m_metadata_storage->get_job_complete(conn, m_id, &complete); if (!err.success()) { throw ConnectionException{ fmt::format("Failed to get job completion status: {}", err.description) @@ -70,7 +77,7 @@ class Job { while (!complete) { constexpr int cSleepMs = 10; std::this_thread::sleep_for(std::chrono::milliseconds(cSleepMs)); - err = m_metadata_storage->get_job_complete(m_id, &complete); + err = m_metadata_storage->get_job_complete(conn, m_id, &complete); if (!err.success()) { throw ConnectionException{ fmt::format("Failed to get job completion status: {}", err.description) @@ -91,8 +98,15 @@ class Job { * @throw spider::ConnectionException */ auto get_status() -> JobStatus { + std::variant conn_result + = core::MySqlConnection::create(m_data_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + core::JobStatus status = core::JobStatus::Running; - core::StorageErr const err = m_metadata_storage->get_job_status(m_id, &status); + core::StorageErr const err = m_metadata_storage->get_job_status(conn, m_id, &status); if (!err.success()) { throw ConnectionException{fmt::format("Failed to get job status: {}", err.description)}; } @@ -120,8 +134,16 @@ class Job { * @throw spider::ConnectionException */ auto get_result() -> ReturnType { + std::variant conn_result + = core::MySqlConnection::create(m_data_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + std::vector output_task_ids; - core::StorageErr err = m_metadata_storage->get_job_output_tasks(m_id, &output_task_ids); + core::StorageErr err + = m_metadata_storage->get_job_output_tasks(conn, m_id, &output_task_ids); if (!err.success()) { throw ConnectionException{ fmt::format("Failed to get job output tasks: {}", err.description) @@ -130,7 +152,7 @@ class Job { std::vector tasks; for (auto const& id : output_task_ids) { core::Task task{""}; - err = m_metadata_storage->get_task(id, &task); + err = m_metadata_storage->get_task(conn, id, &task); if (!err.success()) { throw ConnectionException{fmt::format("Failed to get task: {}", err.description)}; } @@ -161,7 +183,7 @@ class Job { if (!optional_data_id.has_value()) { throw ConnectionException{fmt::format("Output data ID is missing")}; } - err = m_data_storage->get_data(optional_data_id.value(), &data); + err = m_data_storage->get_data(conn, optional_data_id.value(), &data); if (!err.success()) { throw ConnectionException{ fmt::format("Failed to get data: {}", err.description) @@ -202,7 +224,7 @@ class Job { throw ConnectionException{fmt::format("Expected one output task for job result")}; } core::Task task{""}; - err = m_metadata_storage->get_task(output_task_ids[0], &task); + err = m_metadata_storage->get_task(conn, output_task_ids[0], &task); if (!err.success()) { throw ConnectionException{fmt::format("Failed to get task: {}", err.description)}; } @@ -220,7 +242,7 @@ class Job { if (!optional_data_id.has_value()) { throw ConnectionException{fmt::format("Output data ID is missing")}; } - err = m_data_storage->get_data(optional_data_id.value(), &data); + err = m_data_storage->get_data(conn, optional_data_id.value(), &data); if (!err.success()) { throw ConnectionException{fmt::format("Failed to get data: {}", err.description) }; diff --git a/src/spider/client/TaskContext.cpp b/src/spider/client/TaskContext.cpp index 14e77937d..7a1f17c8e 100644 --- a/src/spider/client/TaskContext.cpp +++ b/src/spider/client/TaskContext.cpp @@ -17,8 +17,15 @@ auto TaskContext::get_id() const -> boost::uuids::uuid { } auto TaskContext::kv_store_get(std::string const& key) -> std::optional { + std::variant conn_result + = core::MySqlConnection::create(m_data_store->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + std::string value; - core::StorageErr const err = m_data_store->get_task_kv_data(m_task_id, key, &value); + core::StorageErr const err = m_data_store->get_task_kv_data(conn, m_task_id, key, &value); if (!err.success()) { if (core::StorageErrType::KeyNotFoundErr == err.type) { return std::nullopt; @@ -29,16 +36,30 @@ auto TaskContext::kv_store_get(std::string const& key) -> std::optional void { + std::variant conn_result + = core::MySqlConnection::create(m_data_store->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + core::KeyValueData const kv_data{key, value, m_task_id}; - core::StorageErr const err = m_data_store->add_task_kv_data(kv_data); + core::StorageErr const err = m_data_store->add_task_kv_data(conn, kv_data); if (!err.success()) { throw ConnectionException(err.description); } } auto TaskContext::get_jobs() -> std::vector { + std::variant conn_result + = core::MySqlConnection::create(m_metadata_store->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + std::vector job_ids; - core::StorageErr const err = m_metadata_store->get_jobs_by_client_id(m_task_id, &job_ids); + core::StorageErr const err = m_metadata_store->get_jobs_by_client_id(conn, m_task_id, &job_ids); if (!err.success()) { throw ConnectionException("Failed to get jobs."); } diff --git a/src/spider/client/TaskContext.hpp b/src/spider/client/TaskContext.hpp index 0a306b93c..52f313b2f 100644 --- a/src/spider/client/TaskContext.hpp +++ b/src/spider/client/TaskContext.hpp @@ -151,7 +151,14 @@ class TaskContext { graph.add_task(new_task); graph.add_input_task(new_task.get_id()); graph.add_output_task(new_task.get_id()); - core::StorageErr err = m_metadata_store->add_job(job_id, m_task_id, graph); + + std::variant conn_result + = core::MySqlConnection::create(m_data_store->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); + core::StorageErr err = m_metadata_store->add_job(conn, job_id, m_task_id, graph); if (!err.success()) { throw ConnectionException(fmt::format("Failed to start job: {}", err.description)); } diff --git a/src/spider/scheduler/scheduler.cpp b/src/spider/scheduler/scheduler.cpp index fc68944ae..e66cd078e 100644 --- a/src/spider/scheduler/scheduler.cpp +++ b/src/spider/scheduler/scheduler.cpp @@ -220,7 +220,8 @@ auto main(int argc, char** argv) -> int { spider::core::StopToken stop_token; std::shared_ptr const policy = std::make_shared(metadata_store, data_store, conn); - spider::scheduler::SchedulerServer server{port, policy, metadata_store, data_store, conn, stop_token}; + spider::scheduler::SchedulerServer + server{port, policy, metadata_store, data_store, conn, stop_token}; // Register scheduler with storage spider::core::Scheduler const scheduler{scheduler_id, scheduler_addr, port}; diff --git a/src/spider/storage/MySqlStorage.hpp b/src/spider/storage/MySqlStorage.hpp index 44ce37a60..ef2e5e7c2 100644 --- a/src/spider/storage/MySqlStorage.hpp +++ b/src/spider/storage/MySqlStorage.hpp @@ -27,7 +27,9 @@ namespace spider::core { class MySqlMetadataStorage : public MetadataStorage { public: MySqlMetadataStorage() = delete; - explicit MySqlMetadataStorage(std::string url): m_url{std::move(url)} {} + + explicit MySqlMetadataStorage(std::string url) : m_url{std::move(url)} {} + MySqlMetadataStorage(MySqlMetadataStorage const&) = delete; MySqlMetadataStorage(MySqlMetadataStorage&&) = delete; auto operator=(MySqlMetadataStorage const&) -> MySqlMetadataStorage& = delete; @@ -129,7 +131,9 @@ class MySqlMetadataStorage : public MetadataStorage { class MySqlDataStorage : public DataStorage { public: MySqlDataStorage() = delete; - explicit MySqlDataStorage(std::string url): m_url{std::move(url)} {} + + explicit MySqlDataStorage(std::string url) : m_url{std::move(url)} {} + MySqlDataStorage(MySqlDataStorage const&) = delete; MySqlDataStorage(MySqlDataStorage&&) = delete; auto operator=(MySqlDataStorage const&) -> MySqlDataStorage& = delete; diff --git a/src/spider/storage/StorageConnection.hpp b/src/spider/storage/StorageConnection.hpp index 0de5dcaec..376aadaf1 100644 --- a/src/spider/storage/StorageConnection.hpp +++ b/src/spider/storage/StorageConnection.hpp @@ -3,8 +3,7 @@ namespace spider::core { -class StorageConnection { -}; +class StorageConnection {}; } // namespace spider::core From 9e39bef8040758c845f2da05f38f3e530511f38c Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 01:15:51 -0500 Subject: [PATCH 07/21] Fix typo --- src/spider/storage/DataStorage.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spider/storage/DataStorage.hpp b/src/spider/storage/DataStorage.hpp index ca39ee2b8..f5d5141dc 100644 --- a/src/spider/storage/DataStorage.hpp +++ b/src/spider/storage/DataStorage.hpp @@ -75,7 +75,7 @@ class DataStorage { std::string* value ) -> StorageErr = 0; - virtual auto get_url() const -> std::string const& = 0 + virtual auto get_url() const -> std::string const& = 0; }; } // namespace spider::core From 161e5cf3a6a7b00f6e6c85ac1d3deabd565d4204 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 01:26:38 -0500 Subject: [PATCH 08/21] Fix missing storage connection --- src/spider/client/TaskContext.hpp | 17 ++++++++++----- src/spider/worker/FunctionManager.hpp | 31 +++++++++++++++++++-------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/spider/client/TaskContext.hpp b/src/spider/client/TaskContext.hpp index 52f313b2f..c9f16c775 100644 --- a/src/spider/client/TaskContext.hpp +++ b/src/spider/client/TaskContext.hpp @@ -121,8 +121,8 @@ class TaskContext { * @throw spider::ConnectionException */ template - auto - start(TaskFunction const& task, Inputs&&... inputs) -> Job { + auto start(TaskFunction const& task, Inputs&&... inputs) + -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), @@ -178,8 +178,8 @@ class TaskContext { * @throw spider::ConnectionException */ template - auto - start(TaskGraph const& graph, Inputs&&... inputs) -> Job { + auto start(TaskGraph const& graph, Inputs&&... inputs) + -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), @@ -201,8 +201,15 @@ class TaskContext { graph.m_impl->reset_ids(); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); + + std::variant conn_result + = core::MySqlConnection::create(m_data_store->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); core::StorageErr const err - = m_metadata_store->add_job(job_id, m_task_id, graph.m_impl->get_graph()); + = m_metadata_store->add_job(conn, job_id, m_task_id, graph.m_impl->get_graph()); if (!err.success()) { throw ConnectionException(fmt::format("Failed to start job: {}", err.description)); } diff --git a/src/spider/worker/FunctionManager.hpp b/src/spider/worker/FunctionManager.hpp index 0b76691f0..3bd248717 100644 --- a/src/spider/worker/FunctionManager.hpp +++ b/src/spider/worker/FunctionManager.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ #include "../io/MsgPack.hpp" // IWYU pragma: keep #include "../io/Serializer.hpp" #include "../storage/DataStorage.hpp" +#include "../storage/MySqlConnection.hpp" +#include "../storage/StorageConnection.hpp" #include "TaskExecutorMessage.hpp" // NOLINTBEGIN(cppcoreguidelines-macro-usage) @@ -86,8 +89,8 @@ MSGPACK_ADD_ENUM(spider::core::FunctionInvokeError); namespace spider::core { -auto response_get_error(msgpack::sbuffer const& buffer -) -> std::optional>; +auto response_get_error(msgpack::sbuffer const& buffer) + -> std::optional>; auto create_error_response(FunctionInvokeError error, std::string const& message) -> msgpack::sbuffer; @@ -160,8 +163,8 @@ auto response_get_result(msgpack::sbuffer const& buffer) -> std::optional std::optional>; +auto response_get_result_buffers(msgpack::sbuffer const& buffer) + -> std::optional>; template auto create_result_response(T const& t) -> msgpack::sbuffer { @@ -216,8 +219,8 @@ auto create_args_request(Args&&... args) -> msgpack::sbuffer { return buffer; } -inline auto create_args_request(std::vector const& args_buffers -) -> msgpack::sbuffer { +inline auto create_args_request(std::vector const& args_buffers) + -> msgpack::sbuffer { msgpack::sbuffer buffer; msgpack::packer packer{buffer}; packer.pack_array(2); @@ -234,8 +237,8 @@ inline auto create_args_request(std::vector const& args_buffer template class FunctionInvoker { public: - static auto - apply(F const& function, TaskContext& context, ArgsBuffer const& args_buffer) -> ResultBuffer { + static auto apply(F const& function, TaskContext& context, ArgsBuffer const& args_buffer) + -> ResultBuffer { // NOLINTBEGIN(cppcoreguidelines-pro-type-union-access,cppcoreguidelines-pro-bounds-pointer-arithmetic) using ArgsTuple = signature::args_t; using ReturnType = signature::ret_t; @@ -281,6 +284,16 @@ class FunctionInvoker { // Fill args_tuple StorageErr err; std::get<0>(args_tuple) = context; + std::variant conn_result + = core::MySqlConnection::create(data_store->get_url()); + if (std::holds_alternative(conn_result)) { + err = std::get(conn_result); + return create_error_response( + FunctionInvokeError::ArgumentParsingError, + fmt::format("Cannot parse arguments: {}.", err.description) + ); + } + core::MySqlConnection& conn = std::get(conn_result); for_n - 1>([&](auto i) { if (!err.success()) { return; @@ -290,7 +303,7 @@ class FunctionInvoker { if constexpr (cIsSpecializationV) { boost::uuids::uuid const data_id = arg.as(); std::unique_ptr data = std::make_unique(); - err = data_store->get_data(data_id, data.get()); + err = data_store->get_data(conn, data_id, data.get()); if (!err.success()) { return; } From 5b86ef0f382e3bbbdcd17d34d2196f149b93b014 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 01:39:16 -0500 Subject: [PATCH 09/21] Format code --- src/spider/client/TaskContext.hpp | 8 ++++---- src/spider/worker/FunctionManager.hpp | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/spider/client/TaskContext.hpp b/src/spider/client/TaskContext.hpp index c9f16c775..5151f8a8e 100644 --- a/src/spider/client/TaskContext.hpp +++ b/src/spider/client/TaskContext.hpp @@ -121,8 +121,8 @@ class TaskContext { * @throw spider::ConnectionException */ template - auto start(TaskFunction const& task, Inputs&&... inputs) - -> Job { + auto + start(TaskFunction const& task, Inputs&&... inputs) -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), @@ -178,8 +178,8 @@ class TaskContext { * @throw spider::ConnectionException */ template - auto start(TaskGraph const& graph, Inputs&&... inputs) - -> Job { + auto + start(TaskGraph const& graph, Inputs&&... inputs) -> Job { // Check input type static_assert( sizeof...(Inputs) == sizeof...(Params), diff --git a/src/spider/worker/FunctionManager.hpp b/src/spider/worker/FunctionManager.hpp index 3bd248717..e9fbd0756 100644 --- a/src/spider/worker/FunctionManager.hpp +++ b/src/spider/worker/FunctionManager.hpp @@ -89,8 +89,8 @@ MSGPACK_ADD_ENUM(spider::core::FunctionInvokeError); namespace spider::core { -auto response_get_error(msgpack::sbuffer const& buffer) - -> std::optional>; +auto response_get_error(msgpack::sbuffer const& buffer +) -> std::optional>; auto create_error_response(FunctionInvokeError error, std::string const& message) -> msgpack::sbuffer; @@ -163,8 +163,8 @@ auto response_get_result(msgpack::sbuffer const& buffer) -> std::optional std::optional>; +auto response_get_result_buffers(msgpack::sbuffer const& buffer +) -> std::optional>; template auto create_result_response(T const& t) -> msgpack::sbuffer { @@ -219,8 +219,8 @@ auto create_args_request(Args&&... args) -> msgpack::sbuffer { return buffer; } -inline auto create_args_request(std::vector const& args_buffers) - -> msgpack::sbuffer { +inline auto create_args_request(std::vector const& args_buffers +) -> msgpack::sbuffer { msgpack::sbuffer buffer; msgpack::packer packer{buffer}; packer.pack_array(2); @@ -237,8 +237,8 @@ inline auto create_args_request(std::vector const& args_buffer template class FunctionInvoker { public: - static auto apply(F const& function, TaskContext& context, ArgsBuffer const& args_buffer) - -> ResultBuffer { + static auto + apply(F const& function, TaskContext& context, ArgsBuffer const& args_buffer) -> ResultBuffer { // NOLINTBEGIN(cppcoreguidelines-pro-type-union-access,cppcoreguidelines-pro-bounds-pointer-arithmetic) using ArgsTuple = signature::args_t; using ReturnType = signature::ret_t; From cd39b11b9a516f8e704913dee37c0b72f3bbce46 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 01:47:24 -0500 Subject: [PATCH 10/21] Fix missing storage connection in driver --- src/spider/client/Driver.hpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/spider/client/Driver.hpp b/src/spider/client/Driver.hpp index 0b66a26b9..8dfa2294d 100644 --- a/src/spider/client/Driver.hpp +++ b/src/spider/client/Driver.hpp @@ -175,8 +175,8 @@ class Driver { graph.add_output_task(new_task.get_id()); std::variant conn_result = core::MySqlConnection::create(m_metadata_storage->get_url()); - if (std::holds_alternative(conn_result)) { - throw ConnectionException(std::get(conn_result).description); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); } core::MySqlConnection& conn = std::get(conn_result); core::StorageErr err = m_metadata_storage->add_job(conn, job_id, m_id, graph); @@ -222,8 +222,14 @@ class Driver { graph.m_impl->reset_ids(); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); + std::variant conn_result + = core::MySqlConnection::create(m_metadata_storage->get_url()); + if (std::holds_alternative(conn_result)) { + throw ConnectionException(std::get(conn_result).description); + } + core::MySqlConnection& conn = std::get(conn_result); core::StorageErr const err - = m_metadata_storage->add_job(job_id, m_id, graph.m_impl->get_graph()); + = m_metadata_storage->add_job(conn, job_id, m_id, graph.m_impl->get_graph()); if (!err.success()) { throw ConnectionException(fmt::format("Failed to start job: {}", err.description)); } From a9ef96752e5dcde8a693fe549ff32254082a6c0c Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 01:54:19 -0500 Subject: [PATCH 11/21] Fix missing header --- src/spider/storage/MetadataStorage.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spider/storage/MetadataStorage.hpp b/src/spider/storage/MetadataStorage.hpp index f2be8fc12..6cd2b9ed7 100644 --- a/src/spider/storage/MetadataStorage.hpp +++ b/src/spider/storage/MetadataStorage.hpp @@ -12,6 +12,7 @@ #include "../core/JobMetadata.hpp" #include "../core/Task.hpp" #include "../core/TaskGraph.hpp" +#include "StorageConnection.hpp" namespace spider::core { class MetadataStorage { From f2be2b36f509f3f9c616cb6214ef62a31a5df101 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 02:05:47 -0500 Subject: [PATCH 12/21] Add connection in tests --- tests/scheduler/test-SchedulerPolicy.cpp | 45 +++++--- tests/scheduler/test-SchedulerServer.cpp | 13 ++- tests/storage/StorageTestHelper.hpp | 21 +++- tests/storage/test-DataStorage.cpp | 89 ++++++++++------ tests/storage/test-MetadataStorage.cpp | 129 ++++++++++++++--------- tests/worker/test-FunctionManager.cpp | 11 +- tests/worker/test-TaskExecutor.cpp | 11 +- 7 files changed, 211 insertions(+), 108 deletions(-) diff --git a/tests/scheduler/test-SchedulerPolicy.cpp b/tests/scheduler/test-SchedulerPolicy.cpp index 22c28cf5e..f47aaf0e0 100644 --- a/tests/scheduler/test-SchedulerPolicy.cpp +++ b/tests/scheduler/test-SchedulerPolicy.cpp @@ -37,6 +37,11 @@ TEMPLATE_LIST_TEST_CASE( = std::move(std::get<0>(storages)); std::shared_ptr const data_store = std::move(std::get<1>(storages)); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_store->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; boost::uuids::uuid const client_id = gen(); // Submit tasks @@ -46,7 +51,7 @@ TEMPLATE_LIST_TEST_CASE( graph_1.add_input_task(task_1.get_id()); graph_1.add_output_task(task_1.get_id()); boost::uuids::uuid const job_id_1 = gen(); - REQUIRE(metadata_store->add_job(job_id_1, client_id, graph_1).success()); + REQUIRE(metadata_store->add_job(conn, job_id_1, client_id, graph_1).success()); std::this_thread::sleep_for(std::chrono::seconds(1)); spider::core::Task const task_2{"task_2"}; spider::core::TaskGraph graph_2; @@ -54,9 +59,9 @@ TEMPLATE_LIST_TEST_CASE( graph_2.add_input_task(task_2.get_id()); graph_2.add_output_task(task_2.get_id()); boost::uuids::uuid const job_id_2 = gen(); - REQUIRE(metadata_store->add_job(job_id_2, client_id, graph_2).success()); + REQUIRE(metadata_store->add_job(conn, job_id_2, client_id, graph_2).success()); - spider::scheduler::FifoPolicy policy{metadata_store, data_store}; + spider::scheduler::FifoPolicy policy{metadata_store, data_store, conn}; // Scheduler the earlier task std::optional const optional_task_id = policy.schedule_next(gen(), ""); @@ -66,8 +71,8 @@ TEMPLATE_LIST_TEST_CASE( REQUIRE(task_id == task_1.get_id()); } - REQUIRE(metadata_store->remove_job(job_id_1).success()); - REQUIRE(metadata_store->remove_job(job_id_2).success()); + REQUIRE(metadata_store->remove_job(conn, job_id_1).success()); + REQUIRE(metadata_store->remove_job(conn, job_id_2).success()); } TEMPLATE_LIST_TEST_CASE( @@ -85,6 +90,11 @@ TEMPLATE_LIST_TEST_CASE( = std::move(std::get<0>(storages)); std::shared_ptr const data_store = std::move(std::get<1>(storages)); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_store->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); boost::uuids::uuid const client_id = gen(); @@ -93,16 +103,16 @@ TEMPLATE_LIST_TEST_CASE( spider::core::Data data{"value"}; data.set_hard_locality(true); data.set_locality({"127.0.0.1"}); - REQUIRE(metadata_store->add_driver(spider::core::Driver{client_id}).success()); - REQUIRE(data_store->add_driver_data(client_id, data).success()); + REQUIRE(metadata_store->add_driver(conn, spider::core::Driver{client_id}).success()); + REQUIRE(data_store->add_driver_data(conn, client_id, data).success()); task.add_input(spider::core::TaskInput{data.get_id()}); spider::core::TaskGraph graph; graph.add_task(task); graph.add_input_task(task.get_id()); graph.add_output_task(task.get_id()); - REQUIRE(metadata_store->add_job(job_id, client_id, graph).success()); + REQUIRE(metadata_store->add_job(conn, job_id, client_id, graph).success()); - spider::scheduler::FifoPolicy policy{metadata_store, data_store}; + spider::scheduler::FifoPolicy policy{metadata_store, data_store, conn}; // Schedule with wrong address REQUIRE_FALSE(policy.schedule_next(gen(), "").has_value()); // Schedule with correct address @@ -114,7 +124,7 @@ TEMPLATE_LIST_TEST_CASE( REQUIRE(task_id == task.get_id()); } - REQUIRE(metadata_store->remove_job(job_id).success()); + REQUIRE(metadata_store->remove_job(conn, job_id).success()); } TEMPLATE_LIST_TEST_CASE( @@ -132,6 +142,11 @@ TEMPLATE_LIST_TEST_CASE( = std::move(std::get<0>(storages)); std::shared_ptr const data_store = std::move(std::get<1>(storages)); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_store->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + // Add task boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); @@ -140,16 +155,16 @@ TEMPLATE_LIST_TEST_CASE( spider::core::Data data; data.set_hard_locality(false); data.set_locality({"127.0.0.1"}); - REQUIRE(metadata_store->add_driver(spider::core::Driver{client_id}).success()); - REQUIRE(data_store->add_driver_data(client_id, data).success()); + REQUIRE(metadata_store->add_driver(conn, spider::core::Driver{client_id}).success()); + REQUIRE(data_store->add_driver_data(conn, client_id, data).success()); task.add_input(spider::core::TaskInput{data.get_id()}); spider::core::TaskGraph graph; graph.add_task(task); graph.add_input_task(task.get_id()); graph.add_output_task(task.get_id()); - REQUIRE(metadata_store->add_job(job_id, client_id, graph).success()); + REQUIRE(metadata_store->add_job(conn, job_id, client_id, graph).success()); - spider::scheduler::FifoPolicy policy{metadata_store, data_store}; + spider::scheduler::FifoPolicy policy{metadata_store, data_store, conn}; // Schedule with wrong address std::optional const optional_task_id = policy.schedule_next(gen(), ""); REQUIRE(optional_task_id.has_value()); @@ -158,7 +173,7 @@ TEMPLATE_LIST_TEST_CASE( REQUIRE(task_id == task.get_id()); } - REQUIRE(metadata_store->remove_job(job_id).success()); + REQUIRE(metadata_store->remove_job(conn, job_id).success()); } } // namespace diff --git a/tests/scheduler/test-SchedulerServer.cpp b/tests/scheduler/test-SchedulerServer.cpp index cb90ce28b..c81422838 100644 --- a/tests/scheduler/test-SchedulerServer.cpp +++ b/tests/scheduler/test-SchedulerServer.cpp @@ -45,13 +45,18 @@ TEMPLATE_LIST_TEST_CASE( = std::move(std::get<0>(storages)); std::shared_ptr const data_store = std::move(std::get<1>(storages)); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_store->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + std::shared_ptr const policy - = std::make_shared(metadata_store, data_store); + = std::make_shared(metadata_store, data_store, conn); constexpr unsigned short cPort = 6021; spider::core::StopToken stop_token; spider::scheduler::SchedulerServer - server{cPort, policy, metadata_store, data_store, stop_token}; + server{cPort, policy, metadata_store, data_store, conn, stop_token}; // Pause and resume server server.pause(); @@ -76,7 +81,7 @@ TEMPLATE_LIST_TEST_CASE( graph.add_output_task(child_task.get_id()); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); - REQUIRE(metadata_store->add_job(job_id, gen(), graph).success()); + REQUIRE(metadata_store->add_job(conn, job_id, gen(), graph).success()); // Schedule request should succeed spider::scheduler::ScheduleTaskRequest const req{gen(), ""}; @@ -91,7 +96,7 @@ TEMPLATE_LIST_TEST_CASE( // Get response should succeed and get child task std::optional const& res_buffer = spider::core::receive_message(socket); - REQUIRE(metadata_store->remove_job(job_id).success()); + REQUIRE(metadata_store->remove_job(conn, job_id).success()); REQUIRE(res_buffer.has_value()); if (res_buffer.has_value()) { msgpack::object_handle const handle diff --git a/tests/storage/StorageTestHelper.hpp b/tests/storage/StorageTestHelper.hpp index 3ae551560..a830dbf39 100644 --- a/tests/storage/StorageTestHelper.hpp +++ b/tests/storage/StorageTestHelper.hpp @@ -25,7 +25,11 @@ template requires std::derived_from auto create_data_storage() -> std::unique_ptr { std::unique_ptr storage = std::make_unique(cStorageUrl); - REQUIRE(storage->initialize().success()); + std::variant conn_result + = core::MySqlConnection::create(cStorageUrl); + REQUIRE(std::holds_alternative(conn_result)); + core::MySqlConnection& conn = std::get(conn_result); + REQUIRE(storage->initialize(conn).success()); return storage; } @@ -33,7 +37,11 @@ template requires std::derived_from auto create_metadata_storage() -> std::unique_ptr { std::unique_ptr storage = std::make_unique(cStorageUrl); - REQUIRE(storage->initialize().success()); + std::variant conn_result + = core::MySqlConnection::create(cStorageUrl); + REQUIRE(std::holds_alternative(conn_result)); + core::MySqlConnection& conn = std::get(conn_result); + REQUIRE(storage->initialize(conn).success()); return storage; } @@ -41,10 +49,15 @@ template requires std::derived_from && std::derived_from auto create_storage( ) -> std::tuple, std::unique_ptr> { + std::variant conn_result + = core::MySqlConnection::create(cStorageUrl); + REQUIRE(std::holds_alternative(conn_result)); + core::MySqlConnection& conn = std::get(conn_result); + std::unique_ptr metadata_storage = std::make_unique(cStorageUrl); - REQUIRE(metadata_storage->initialize().success()); + REQUIRE(metadata_storage->initialize(conn).success()); std::unique_ptr data_storage = std::make_unique(cStorageUrl); - REQUIRE(data_storage->initialize().success()); + REQUIRE(data_storage->initialize(conn).success()); return std::make_tuple(std::move(metadata_storage), std::move(data_storage)); } diff --git a/tests/storage/test-DataStorage.cpp b/tests/storage/test-DataStorage.cpp index f24d88ef7..d0094e664 100644 --- a/tests/storage/test-DataStorage.cpp +++ b/tests/storage/test-DataStorage.cpp @@ -21,29 +21,34 @@ TEMPLATE_LIST_TEST_CASE("Add, get and remove data", "[storage]", spider::test::S auto [metadata_storage, data_storage] = spider::test:: create_storage, std::tuple_element_t<1, TestType>>(); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + // Add driver and data spider::core::Data const data{"value"}; boost::uuids::random_generator gen; boost::uuids::uuid const driver_id = gen(); - REQUIRE(metadata_storage->add_driver(spider::core::Driver{driver_id}).success()); - REQUIRE(data_storage->add_driver_data(driver_id, data).success()); + REQUIRE(metadata_storage->add_driver(conn, spider::core::Driver{driver_id}).success()); + REQUIRE(data_storage->add_driver_data(conn, driver_id, data).success()); // Add data with same id again should fail spider::core::Data const data_same_id{data.get_id(), "value2"}; REQUIRE(spider::core::StorageErrType::DuplicateKeyErr - == data_storage->add_driver_data(driver_id, data_same_id).type); + == data_storage->add_driver_data(conn, driver_id, data_same_id).type); // Get data should match spider::core::Data result{"temp"}; - REQUIRE(data_storage->get_data(data.get_id(), &result).success()); + REQUIRE(data_storage->get_data(conn, data.get_id(), &result).success()); REQUIRE(spider::test::data_equal(data, result)); // Remove data should succeed - REQUIRE(data_storage->remove_data(data.get_id()).success()); + REQUIRE(data_storage->remove_data(conn, data.get_id()).success()); // Get data should fail REQUIRE(spider::core::StorageErrType::KeyNotFoundErr - == data_storage->get_data(data.get_id(), &result).type); + == data_storage->get_data(conn, data.get_id(), &result).type); } TEMPLATE_LIST_TEST_CASE( @@ -54,24 +59,29 @@ TEMPLATE_LIST_TEST_CASE( auto [metadata_storage, data_storage] = spider::test:: create_storage, std::tuple_element_t<1, TestType>>(); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + // Add driver boost::uuids::random_generator gen; boost::uuids::uuid const driver_id = gen(); - REQUIRE(metadata_storage->add_driver(spider::core::Driver{driver_id}).success()); + REQUIRE(metadata_storage->add_driver(conn, spider::core::Driver{driver_id}).success()); // Add data spider::core::KeyValueData const data{"key", "value", driver_id}; - REQUIRE(data_storage->add_client_kv_data(data).success()); + REQUIRE(data_storage->add_client_kv_data(conn, data).success()); // Add data with same key and id again should fail spider::core::KeyValueData const data_same_key{"key", "value2", driver_id}; REQUIRE(spider::core::StorageErrType::DuplicateKeyErr - == data_storage->add_client_kv_data(data_same_key).type); + == data_storage->add_client_kv_data(conn, data_same_key).type); // Get data should match std::string value; - auto err = data_storage->get_client_kv_data(driver_id, "key", &value); - REQUIRE(data_storage->get_client_kv_data(driver_id, "key", &value).success()); + auto err = data_storage->get_client_kv_data(conn, driver_id, "key", &value); + REQUIRE(data_storage->get_client_kv_data(conn, driver_id, "key", &value).success()); REQUIRE(data.get_value() == value); } @@ -83,6 +93,11 @@ TEMPLATE_LIST_TEST_CASE( auto [metadata_storage, data_storage] = spider::test:: create_storage, std::tuple_element_t<1, TestType>>(); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + // Add task boost::uuids::random_generator gen; spider::core::Task const task{"func"}; @@ -91,24 +106,24 @@ TEMPLATE_LIST_TEST_CASE( graph.add_input_task(task.get_id()); graph.add_output_task(task.get_id()); boost::uuids::uuid const job_id = gen(); - REQUIRE(metadata_storage->add_job(job_id, gen(), graph).success()); + REQUIRE(metadata_storage->add_job(conn, job_id, gen(), graph).success()); // Add data spider::core::KeyValueData const data{"key", "value", task.get_id()}; - REQUIRE(data_storage->add_task_kv_data(data).success()); + REQUIRE(data_storage->add_task_kv_data(conn, data).success()); // Add data with same key and id again should fail spider::core::KeyValueData const data_same_key{"key", "value2", task.get_id()}; REQUIRE(spider::core::StorageErrType::DuplicateKeyErr - == data_storage->add_task_kv_data(data_same_key).type); + == data_storage->add_task_kv_data(conn, data_same_key).type); // Get data should match std::string value; - REQUIRE(data_storage->get_task_kv_data(task.get_id(), "key", &value).success()); + REQUIRE(data_storage->get_task_kv_data(conn, task.get_id(), "key", &value).success()); REQUIRE(data.get_value() == value); // Clean up - REQUIRE(metadata_storage->remove_job(job_id).success()); + REQUIRE(metadata_storage->remove_job(conn, job_id).success()); } TEMPLATE_LIST_TEST_CASE( @@ -119,9 +134,14 @@ TEMPLATE_LIST_TEST_CASE( auto [metadata_storage, data_storage] = spider::test:: create_storage, std::tuple_element_t<1, TestType>>(); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; // Add task reference without data and task should fail. - REQUIRE(!data_storage->add_task_reference(gen(), gen()).success()); + REQUIRE(!data_storage->add_task_reference(conn, gen(), gen()).success()); // Add task spider::core::Task const task{"func"}; @@ -133,31 +153,31 @@ TEMPLATE_LIST_TEST_CASE( graph.add_input_task(task.get_id()); graph.add_output_task(task_2.get_id()); boost::uuids::uuid const job_id = gen(); - REQUIRE(metadata_storage->add_job(job_id, gen(), graph).success()); + REQUIRE(metadata_storage->add_job(conn, job_id, gen(), graph).success()); // Add task reference without data should fail. - REQUIRE(!data_storage->add_task_reference(gen(), task.get_id()).success()); + REQUIRE(!data_storage->add_task_reference(conn, gen(), task.get_id()).success()); // Add data spider::core::Data const data{"value"}; - REQUIRE(data_storage->add_task_data(task.get_id(), data).success()); + REQUIRE(data_storage->add_task_data(conn, task.get_id(), data).success()); // Add task reference - REQUIRE(data_storage->add_task_reference(data.get_id(), task_2.get_id()).success()); + REQUIRE(data_storage->add_task_reference(conn, data.get_id(), task_2.get_id()).success()); // Remove task reference - REQUIRE(data_storage->remove_task_reference(data.get_id(), task_2.get_id()).success()); + REQUIRE(data_storage->remove_task_reference(conn, data.get_id(), task_2.get_id()).success()); // Remove job - REQUIRE(metadata_storage->remove_job(job_id).success()); + REQUIRE(metadata_storage->remove_job(conn, job_id).success()); // Clean up - REQUIRE(data_storage->remove_dangling_data().success()); + REQUIRE(data_storage->remove_dangling_data(conn).success()); // Get data should fail spider::core::Data res{"temp"}; REQUIRE(spider::core::StorageErrType::KeyNotFoundErr - == data_storage->get_data(data.get_id(), &res).type); + == data_storage->get_data(conn, data.get_id(), &res).type); } TEMPLATE_LIST_TEST_CASE( @@ -168,29 +188,34 @@ TEMPLATE_LIST_TEST_CASE( auto [metadata_storage, data_storage] = spider::test:: create_storage, std::tuple_element_t<1, TestType>>(); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; // Add driver reference without data and driver should fail - REQUIRE(!data_storage->add_driver_reference(gen(), gen()).success()); + REQUIRE(!data_storage->add_driver_reference(conn, gen(), gen()).success()); // Add driver boost::uuids::uuid const driver_id = gen(); boost::uuids::uuid const driver_id_2 = gen(); - REQUIRE(metadata_storage->add_driver(spider::core::Driver{driver_id}).success()); - REQUIRE(metadata_storage->add_driver(spider::core::Driver{driver_id_2}).success()); + REQUIRE(metadata_storage->add_driver(conn, spider::core::Driver{driver_id}).success()); + REQUIRE(metadata_storage->add_driver(conn, spider::core::Driver{driver_id_2}).success()); // Add driver reference without data should fail - REQUIRE(!data_storage->add_driver_reference(gen(), driver_id).success()); + REQUIRE(!data_storage->add_driver_reference(conn, gen(), driver_id).success()); // Add data spider::core::Data const data{"value"}; - REQUIRE(data_storage->add_driver_data(driver_id, data).success()); + REQUIRE(data_storage->add_driver_data(conn, driver_id, data).success()); // Add driver reference - REQUIRE(data_storage->add_driver_reference(data.get_id(), driver_id_2).success()); + REQUIRE(data_storage->add_driver_reference(conn, data.get_id(), driver_id_2).success()); // Remove driver reference - REQUIRE(data_storage->remove_driver_reference(data.get_id(), driver_id_2).success()); + REQUIRE(data_storage->remove_driver_reference(conn, data.get_id(), driver_id_2).success()); } } // namespace diff --git a/tests/storage/test-MetadataStorage.cpp b/tests/storage/test-MetadataStorage.cpp index 619e8a09a..d9512f77f 100644 --- a/tests/storage/test-MetadataStorage.cpp +++ b/tests/storage/test-MetadataStorage.cpp @@ -26,16 +26,21 @@ TEMPLATE_LIST_TEST_CASE("Driver heartbeat", "[storage]", spider::test::MetadataS std::unique_ptr storage = spider::test::create_metadata_storage(); + std::variant conn_result + = spider::core::MySqlConnection::create(storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + constexpr double cDuration = 100; // Add driver should succeed boost::uuids::random_generator gen; boost::uuids::uuid const driver_id = gen(); - REQUIRE(storage->add_driver(spider::core::Driver{driver_id}).success()); + REQUIRE(storage->add_driver(conn, spider::core::Driver{driver_id}).success()); std::vector ids{}; // Driver should not time out - REQUIRE(storage->heartbeat_timeout(cDuration, &ids).success()); + REQUIRE(storage->heartbeat_timeout(conn, cDuration, &ids).success()); // Because other tests may run in parallel, just check `ids` don't have `driver_id` REQUIRE(std::ranges::none_of(ids, [&driver_id](boost::uuids::uuid id) { return id == driver_id; @@ -44,7 +49,7 @@ TEMPLATE_LIST_TEST_CASE("Driver heartbeat", "[storage]", spider::test::MetadataS std::this_thread::sleep_for(std::chrono::seconds(1)); // Driver should time out - REQUIRE(storage->heartbeat_timeout(cDuration, &ids).success()); + REQUIRE(storage->heartbeat_timeout(conn, cDuration, &ids).success()); REQUIRE(!ids.empty()); REQUIRE(std::ranges::any_of(ids, [&driver_id](boost::uuids::uuid id) { return id == driver_id; @@ -52,9 +57,9 @@ TEMPLATE_LIST_TEST_CASE("Driver heartbeat", "[storage]", spider::test::MetadataS ids.clear(); // Update heartbeat - REQUIRE(storage->update_heartbeat(driver_id).success()); + REQUIRE(storage->update_heartbeat(conn, driver_id).success()); // Driver should not time out - REQUIRE(storage->heartbeat_timeout(cDuration, &ids).success()); + REQUIRE(storage->heartbeat_timeout(conn, cDuration, &ids).success()); REQUIRE(std::ranges::none_of(ids, [&driver_id](boost::uuids::uuid id) { return id == driver_id; })); @@ -68,37 +73,42 @@ TEMPLATE_LIST_TEST_CASE( std::unique_ptr storage = spider::test::create_metadata_storage(); + std::variant conn_result + = spider::core::MySqlConnection::create(storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; boost::uuids::uuid const scheduler_id = gen(); constexpr int cPort = 3306; // Add scheduler should succeed - REQUIRE(storage->add_scheduler(spider::core::Scheduler{scheduler_id, "127.0.0.1", cPort}) + REQUIRE(storage->add_scheduler(conn, spider::core::Scheduler{scheduler_id, "127.0.0.1", cPort}) .success()); // Get scheduler addr should succeed std::string addr_res; int port_res = 0; - REQUIRE(storage->get_scheduler_addr(scheduler_id, &addr_res, &port_res).success()); + REQUIRE(storage->get_scheduler_addr(conn, scheduler_id, &addr_res, &port_res).success()); REQUIRE(addr_res == "127.0.0.1"); REQUIRE(port_res == cPort); // Get non-exist scheduler should fail REQUIRE(spider::core::StorageErrType::KeyNotFoundErr - == storage->get_scheduler_addr(gen(), &addr_res, &port_res).type); + == storage->get_scheduler_addr(conn, gen(), &addr_res, &port_res).type); // Get default state std::string state_res; - REQUIRE(storage->get_scheduler_state(scheduler_id, &state_res).success()); + REQUIRE(storage->get_scheduler_state(conn, scheduler_id, &state_res).success()); REQUIRE(state_res == "normal"); state_res.clear(); // Update scheduler state should succeed std::string state = "recovery"; - REQUIRE(storage->set_scheduler_state(scheduler_id, state).success()); + REQUIRE(storage->set_scheduler_state(conn, scheduler_id, state).success()); // Get new state - REQUIRE(storage->get_scheduler_state(scheduler_id, &state_res).success()); + REQUIRE(storage->get_scheduler_state(conn, scheduler_id, &state_res).success()); REQUIRE(state_res == state); } @@ -110,6 +120,11 @@ TEMPLATE_LIST_TEST_CASE( std::unique_ptr storage = spider::test::create_metadata_storage(); + std::variant conn_result + = spider::core::MySqlConnection::create(storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); @@ -160,16 +175,16 @@ TEMPLATE_LIST_TEST_CASE( REQUIRE(heads[0] == simple_task.get_id()); // Submit job should success - REQUIRE(storage->add_job(job_id, client_id, graph).success()); - REQUIRE(storage->add_job(simple_job_id, client_id, simple_graph).success()); + REQUIRE(storage->add_job(conn, job_id, client_id, graph).success()); + REQUIRE(storage->add_job(conn, simple_job_id, client_id, simple_graph).success()); // Get job id for non-existent client id should return empty vector std::vector job_ids; - REQUIRE(storage->get_jobs_by_client_id(gen(), &job_ids).success()); + REQUIRE(storage->get_jobs_by_client_id(conn, gen(), &job_ids).success()); REQUIRE(job_ids.empty()); // Get job id for client id should get correct value - REQUIRE(storage->get_jobs_by_client_id(client_id, &job_ids).success()); + REQUIRE(storage->get_jobs_by_client_id(conn, client_id, &job_ids).success()); REQUIRE(2 == job_ids.size()); REQUIRE( ((job_ids[0] == job_id && job_ids[1] == simple_job_id) @@ -178,7 +193,7 @@ TEMPLATE_LIST_TEST_CASE( // Get job metadata should get correct value spider::core::JobMetadata job_metadata{}; - REQUIRE(storage->get_job_metadata(job_id, &job_metadata).success()); + REQUIRE(storage->get_job_metadata(conn, job_id, &job_metadata).success()); REQUIRE(job_id == job_metadata.get_id()); REQUIRE(client_id == job_metadata.get_client_id()); std::chrono::seconds const time_delta{1}; @@ -187,26 +202,26 @@ TEMPLATE_LIST_TEST_CASE( // Get task graph should succeed spider::core::TaskGraph graph_res{}; - REQUIRE(storage->get_task_graph(job_id, &graph_res).success()); + REQUIRE(storage->get_task_graph(conn, job_id, &graph_res).success()); REQUIRE(spider::test::task_graph_equal(graph, graph_res)); spider::core::TaskGraph simple_graph_res{}; - REQUIRE(storage->get_task_graph(simple_job_id, &simple_graph_res).success()); + REQUIRE(storage->get_task_graph(conn, simple_job_id, &simple_graph_res).success()); REQUIRE(spider::test::task_graph_equal(simple_graph, simple_graph_res)); // Get task should succeed spider::core::Task task_res{""}; - REQUIRE(storage->get_task(child_task.get_id(), &task_res).success()); + REQUIRE(storage->get_task(conn, child_task.get_id(), &task_res).success()); REQUIRE(spider::test::task_equal(child_task, task_res)); // Get child tasks should succeed std::vector tasks; - REQUIRE(storage->get_child_tasks(parent_1.get_id(), &tasks).success()); + REQUIRE(storage->get_child_tasks(conn, parent_1.get_id(), &tasks).success()); REQUIRE(1 == tasks.size()); REQUIRE(spider::test::task_equal(child_task, tasks[0])); tasks.clear(); // Get parent tasks should succeed - REQUIRE(storage->get_parent_tasks(child_task.get_id(), &tasks).success()); + REQUIRE(storage->get_parent_tasks(conn, child_task.get_id(), &tasks).success()); REQUIRE(2 == tasks.size()); REQUIRE( ((spider::test::task_equal(tasks[0], parent_1) @@ -216,19 +231,24 @@ TEMPLATE_LIST_TEST_CASE( ); // Remove job should succeed - REQUIRE(storage->remove_job(simple_job_id).success()); + REQUIRE(storage->remove_job(conn, simple_job_id).success()); REQUIRE(spider::core::StorageErrType::KeyNotFoundErr - == storage->get_task_graph(simple_job_id, &simple_graph_res).type); + == storage->get_task_graph(conn, simple_job_id, &simple_graph_res).type); graph_res = spider::core::TaskGraph{}; - REQUIRE(storage->get_task_graph(job_id, &graph_res).success()); + REQUIRE(storage->get_task_graph(conn, job_id, &graph_res).success()); REQUIRE(spider::test::task_graph_equal(graph, graph_res)); - REQUIRE(storage->remove_job(job_id).success()); + REQUIRE(storage->remove_job(conn, job_id).success()); } TEMPLATE_LIST_TEST_CASE("Task finish", "[storage]", spider::test::MetadataStorageTypeList) { std::unique_ptr storage = spider::test::create_metadata_storage(); + std::variant conn_result + = spider::core::MySqlConnection::create(storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); @@ -256,40 +276,50 @@ TEMPLATE_LIST_TEST_CASE("Task finish", "[storage]", spider::test::MetadataStorag graph.add_input_task(parent_2.get_id()); graph.add_output_task(child_task.get_id()); // Submit job should success - REQUIRE(storage->add_job(job_id, gen(), graph).success()); + REQUIRE(storage->add_job(conn, job_id, gen(), graph).success()); // Task finish for parent 1 should succeed spider::core::TaskInstance const parent_1_instance{gen(), parent_1.get_id()}; - REQUIRE(storage->set_task_state(parent_1.get_id(), spider::core::TaskState::Running).success()); - REQUIRE(storage->task_finish(parent_1_instance, {spider::core::TaskOutput{"1.1", "float"}}) + REQUIRE(storage->set_task_state(conn, parent_1.get_id(), spider::core::TaskState::Running) .success()); + REQUIRE(storage->task_finish( + conn, + parent_1_instance, + {spider::core::TaskOutput{"1.1", "float"}} + ).success()); // Parent 1 finish should not update state of any other tasks spider::core::Task res_task{""}; - REQUIRE(storage->get_task(parent_2.get_id(), &res_task).success()); + REQUIRE(storage->get_task(conn, parent_2.get_id(), &res_task).success()); REQUIRE(spider::test::task_equal(parent_2, res_task)); REQUIRE(res_task.get_state() == spider::core::TaskState::Ready); - REQUIRE(storage->get_task(child_task.get_id(), &res_task).success()); + REQUIRE(storage->get_task(conn, child_task.get_id(), &res_task).success()); REQUIRE(res_task.get_state() == spider::core::TaskState::Pending); // Task finish for parent 2 should success spider::core::TaskInstance const parent_2_instance{gen(), parent_2.get_id()}; - REQUIRE(storage->set_task_state(parent_2.get_id(), spider::core::TaskState::Running).success()); - REQUIRE(storage->task_finish(parent_2_instance, {spider::core::TaskOutput{"2", "int"}}) + REQUIRE(storage->set_task_state(conn, parent_2.get_id(), spider::core::TaskState::Running) + .success()); + REQUIRE(storage->task_finish(conn, parent_2_instance, {spider::core::TaskOutput{"2", "int"}}) .success()); // Parent 2 finish should update state of child - REQUIRE(storage->get_task(child_task.get_id(), &res_task).success()); + REQUIRE(storage->get_task(conn, child_task.get_id(), &res_task).success()); REQUIRE(res_task.get_input(0).get_value() == "1.1"); REQUIRE(res_task.get_input(1).get_value() == "2"); REQUIRE(res_task.get_state() == spider::core::TaskState::Ready); // Clean up - REQUIRE(storage->remove_job(job_id).success()); + REQUIRE(storage->remove_job(conn, job_id).success()); } TEMPLATE_LIST_TEST_CASE("Job reset", "[storage]", spider::test::MetadataStorageTypeList) { std::unique_ptr storage = spider::test::create_metadata_storage(); + std::variant conn_result + = spider::core::MySqlConnection::create(storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); @@ -320,46 +350,51 @@ TEMPLATE_LIST_TEST_CASE("Job reset", "[storage]", spider::test::MetadataStorageT graph.add_input_task(parent_2.get_id()); graph.add_output_task(child_task.get_id()); // Submit job should success - REQUIRE(storage->add_job(job_id, gen(), graph).success()); + REQUIRE(storage->add_job(conn, job_id, gen(), graph).success()); // Task finish for parent 1 should succeed spider::core::TaskInstance const parent_1_instance{gen(), parent_1.get_id()}; - REQUIRE(storage->set_task_state(parent_1.get_id(), spider::core::TaskState::Running).success()); - REQUIRE(storage->task_finish(parent_1_instance, {spider::core::TaskOutput{"1.1", "float"}}) + REQUIRE(storage->set_task_state(conn, parent_1.get_id(), spider::core::TaskState::Running) .success()); + REQUIRE(storage->task_finish( + conn, + parent_1_instance, + {spider::core::TaskOutput{"1.1", "float"}} + ).success()); // Task finish for parent 2 should success spider::core::TaskInstance const parent_2_instance{gen(), parent_2.get_id()}; - REQUIRE(storage->set_task_state(parent_2.get_id(), spider::core::TaskState::Running).success()); - REQUIRE(storage->task_finish(parent_2_instance, {spider::core::TaskOutput{"2", "int"}}) + REQUIRE(storage->set_task_state(conn, parent_2.get_id(), spider::core::TaskState::Running) + .success()); + REQUIRE(storage->task_finish(conn, parent_2_instance, {spider::core::TaskOutput{"2", "int"}}) .success()); // Task finish for child should success spider::core::TaskInstance const child_instance{gen(), child_task.get_id()}; - REQUIRE(storage->set_task_state(child_task.get_id(), spider::core::TaskState::Running).success() - ); - REQUIRE(storage->task_finish(child_instance, {spider::core::TaskOutput{"3.3", "float"}}) + REQUIRE(storage->set_task_state(conn, child_task.get_id(), spider::core::TaskState::Running) + .success()); + REQUIRE(storage->task_finish(conn, child_instance, {spider::core::TaskOutput{"3.3", "float"}}) .success()); // Job reset - REQUIRE(storage->reset_job(job_id).success()); + REQUIRE(storage->reset_job(conn, job_id).success()); // Parent tasks states should be ready and child task state should be waiting // Parent tasks inputs should be available and child task inputs should be empty // All tasks output should be empty spider::core::Task res_task{""}; - REQUIRE(storage->get_task(parent_1.get_id(), &res_task).success()); + REQUIRE(storage->get_task(conn, parent_1.get_id(), &res_task).success()); REQUIRE(res_task.get_state() == spider::core::TaskState::Ready); REQUIRE(res_task.get_num_inputs() == 2); REQUIRE(res_task.get_input(0).get_value() == "1"); REQUIRE(res_task.get_input(1).get_value() == "2"); REQUIRE(res_task.get_num_outputs() == 1); REQUIRE(!res_task.get_output(0).get_value().has_value()); - REQUIRE(storage->get_task(parent_2.get_id(), &res_task).success()); + REQUIRE(storage->get_task(conn, parent_2.get_id(), &res_task).success()); REQUIRE(res_task.get_state() == spider::core::TaskState::Ready); REQUIRE(res_task.get_num_inputs() == 2); REQUIRE(res_task.get_input(0).get_value() == "3"); REQUIRE(res_task.get_input(1).get_value() == "4"); REQUIRE(res_task.get_num_outputs() == 1); REQUIRE(!res_task.get_output(0).get_value().has_value()); - REQUIRE(storage->get_task(child_task.get_id(), &res_task).success()); + REQUIRE(storage->get_task(conn, child_task.get_id(), &res_task).success()); REQUIRE(res_task.get_state() == spider::core::TaskState::Pending); REQUIRE(res_task.get_num_inputs() == 2); REQUIRE(!res_task.get_input(0).get_value().has_value()); @@ -368,7 +403,7 @@ TEMPLATE_LIST_TEST_CASE("Job reset", "[storage]", spider::test::MetadataStorageT REQUIRE(!res_task.get_output(0).get_value().has_value()); // Clean up - REQUIRE(storage->remove_job(job_id).success()); + REQUIRE(storage->remove_job(conn, job_id).success()); } } // namespace diff --git a/tests/worker/test-FunctionManager.cpp b/tests/worker/test-FunctionManager.cpp index 77d8f8245..5eeec46a4 100644 --- a/tests/worker/test-FunctionManager.cpp +++ b/tests/worker/test-FunctionManager.cpp @@ -148,14 +148,19 @@ TEMPLATE_LIST_TEST_CASE( = std::move(unique_metadata_storage); std::shared_ptr const data_storage = std::move(unique_data_storage); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + msgpack::sbuffer buffer; msgpack::pack(buffer, 3); spider::core::Data const data{std::string{buffer.data(), buffer.size()}}; boost::uuids::random_generator gen; boost::uuids::uuid const driver_id = gen(); spider::core::Driver const driver{driver_id}; - REQUIRE(metadata_storage->add_driver(driver).success()); - REQUIRE(data_storage->add_driver_data(driver_id, data).success()); + REQUIRE(metadata_storage->add_driver(conn, driver).success()); + REQUIRE(data_storage->add_driver_data(conn, driver_id, data).success()); spider::TaskContext context = spider::core::TaskContextImpl::create_task_context( gen(), @@ -171,7 +176,7 @@ TEMPLATE_LIST_TEST_CASE( msgpack::sbuffer const result = (*function)(context, args_buffers); REQUIRE(3 == spider::core::response_get_result(result).value_or(0)); - REQUIRE(data_storage->remove_data(data.get_id()).success()); + REQUIRE(data_storage->remove_data(conn, data.get_id()).success()); } } // namespace diff --git a/tests/worker/test-TaskExecutor.cpp b/tests/worker/test-TaskExecutor.cpp index 15a464e75..253ba69fd 100644 --- a/tests/worker/test-TaskExecutor.cpp +++ b/tests/worker/test-TaskExecutor.cpp @@ -146,6 +146,11 @@ TEMPLATE_LIST_TEST_CASE( = std::move(unique_metadata_storage); std::shared_ptr const data_storage = std::move(unique_data_storage); + std::variant conn_result + = spider::core::MySqlConnection::create(metadata_storage->get_url()); + REQUIRE(std::holds_alternative(conn_result)); + spider::core::MySqlConnection& conn = std::get(conn_result); + // Create driver and data msgpack::sbuffer buffer; msgpack::pack(buffer, 3); @@ -153,8 +158,8 @@ TEMPLATE_LIST_TEST_CASE( boost::uuids::random_generator gen; boost::uuids::uuid const driver_id = gen(); spider::core::Driver const driver{driver_id}; - REQUIRE(metadata_storage->add_driver(driver).success()); - REQUIRE(data_storage->add_driver_data(driver_id, data).success()); + REQUIRE(metadata_storage->add_driver(conn, driver).success()); + REQUIRE(data_storage->add_driver_data(conn, driver_id, data).success()); absl::flat_hash_map< boost::process::v2::environment::key, @@ -182,7 +187,7 @@ TEMPLATE_LIST_TEST_CASE( } // Clean up - REQUIRE(data_storage->remove_data(data.get_id()).success()); + REQUIRE(data_storage->remove_data(conn, data.get_id()).success()); } } // namespace From 7d964b619ae1ebcb5708b5073e3e0408100e346a Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 02:24:21 -0500 Subject: [PATCH 13/21] Fix worker context not reset --- src/spider/worker/worker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index d91859de3..e2c40656f 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -229,8 +229,8 @@ auto task_loop( spider::core::StopToken const& stop_token ) -> void { std::optional fail_task_id = std::nullopt; - boost::asio::io_context context; while (!stop_token.stop_requested()) { + boost::asio::io_context context; std::variant conn_result = spider::core::MySqlConnection::create(metadata_store->get_url()); if (std::holds_alternative(conn_result)) { From ce247ab6024e8df9c6a4b94c0d113cfd53dc14fd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 3 Mar 2025 12:18:47 -0500 Subject: [PATCH 14/21] Fix worker client connection failure handling Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/spider/worker/WorkerClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spider/worker/WorkerClient.cpp b/src/spider/worker/WorkerClient.cpp index cc5aedff2..512f7f6a5 100644 --- a/src/spider/worker/WorkerClient.cpp +++ b/src/spider/worker/WorkerClient.cpp @@ -49,9 +49,9 @@ auto WorkerClient::get_next_task(std::optional const& fail_t "Failed to connection to storage: {}", std::get(conn_result).description ); + return std::nullopt; } spider::core::MySqlConnection& conn = std::get(conn_result); - if (!m_metadata_store->get_active_scheduler(conn, &schedulers).success()) { return std::nullopt; } From a4245f3d3e3ce11615d09abb65d148d34e989097 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 13:22:57 -0500 Subject: [PATCH 15/21] Fix clang-tidy for client --- src/spider/client/Data.hpp | 5 +++-- src/spider/client/Driver.cpp | 14 ++++++++------ src/spider/client/Driver.hpp | 8 +++++--- src/spider/client/Job.hpp | 9 ++++++--- src/spider/client/TaskContext.cpp | 8 +++++--- src/spider/client/TaskContext.hpp | 6 ++++-- 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/src/spider/client/Data.hpp b/src/spider/client/Data.hpp index 2abab79af..5ec729168 100644 --- a/src/spider/client/Data.hpp +++ b/src/spider/client/Data.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -70,7 +71,7 @@ class Data { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); m_data_store->set_data_locality(conn, *m_impl); } @@ -120,7 +121,7 @@ class Data { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr err; switch (m_data_source) { case DataSource::Driver: diff --git a/src/spider/client/Driver.cpp b/src/spider/client/Driver.cpp index 885e9f3fb..ea59e08ca 100644 --- a/src/spider/client/Driver.cpp +++ b/src/spider/client/Driver.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -14,6 +15,7 @@ #include "../core/Error.hpp" #include "../core/KeyValueData.hpp" #include "../io/BoostAsio.hpp" // IWYU pragma: keep +#include "../storage/MySqlConnection.hpp" #include "../storage/MySqlStorage.hpp" #include "Exception.hpp" @@ -31,7 +33,7 @@ Driver::Driver(std::string const& storage_url) { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_metadata_storage->add_driver(conn, core::Driver{m_id}); if (!err.success()) { @@ -51,7 +53,7 @@ Driver::Driver(std::string const& storage_url) { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_metadata_storage->update_heartbeat(conn, m_id); if (!err.success()) { @@ -69,7 +71,7 @@ Driver::Driver(std::string const& storage_url, boost::uuids::uuid const id) : m_ if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_metadata_storage->add_driver(conn, core::Driver{m_id}); if (!err.success()) { @@ -89,7 +91,7 @@ Driver::Driver(std::string const& storage_url, boost::uuids::uuid const id) : m_ if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_metadata_storage->update_heartbeat(conn, m_id); if (!err.success()) { @@ -107,7 +109,7 @@ auto Driver::kv_store_insert(std::string const& key, std::string const& value) - if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_data_storage->add_client_kv_data(conn, kv_data); if (!err.success()) { @@ -121,7 +123,7 @@ auto Driver::kv_store_get(std::string const& key) -> std::optional if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); std::string value; core::StorageErr const err = m_data_storage->get_client_kv_data(conn, m_id, key, &value); diff --git a/src/spider/client/Driver.hpp b/src/spider/client/Driver.hpp index 8dfa2294d..84d1ffb62 100644 --- a/src/spider/client/Driver.hpp +++ b/src/spider/client/Driver.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,7 @@ #include "../core/Error.hpp" #include "../core/TaskGraphImpl.hpp" #include "../io/Serializer.hpp" +#include "../storage/MySqlConnection.hpp" #include "../worker/FunctionManager.hpp" #include "../worker/FunctionNameManager.hpp" #include "Data.hpp" @@ -178,7 +180,7 @@ class Driver { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr err = m_metadata_storage->add_job(conn, job_id, m_id, graph); if (!err.success()) { throw ConnectionException(fmt::format("Failed to start job: {}", err.description)); @@ -227,7 +229,7 @@ class Driver { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_metadata_storage->add_job(conn, job_id, m_id, graph.m_impl->get_graph()); if (!err.success()) { @@ -252,7 +254,7 @@ class Driver { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_metadata_storage->get_jobs_by_client_id(conn, m_id, &job_ids); if (!err.success()) { diff --git a/src/spider/client/Job.hpp b/src/spider/client/Job.hpp index e8a0f8fab..8b0ff626f 100644 --- a/src/spider/client/Job.hpp +++ b/src/spider/client/Job.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -20,7 +21,9 @@ #include "../core/JobMetadata.hpp" #include "../io/MsgPack.hpp" // IWYU pragma: keep #include "../storage/MetadataStorage.hpp" +#include "../storage/MySqlConnection.hpp" #include "Data.hpp" +#include "Exception.hpp" #include "task.hpp" #include "type_utils.hpp" @@ -65,7 +68,7 @@ class Job { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); bool complete = false; core::StorageErr err = m_metadata_storage->get_job_complete(conn, m_id, &complete); @@ -103,7 +106,7 @@ class Job { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::JobStatus status = core::JobStatus::Running; core::StorageErr const err = m_metadata_storage->get_job_status(conn, m_id, &status); @@ -139,7 +142,7 @@ class Job { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); std::vector output_task_ids; core::StorageErr err diff --git a/src/spider/client/TaskContext.cpp b/src/spider/client/TaskContext.cpp index 7a1f17c8e..8aa4d59b2 100644 --- a/src/spider/client/TaskContext.cpp +++ b/src/spider/client/TaskContext.cpp @@ -2,12 +2,14 @@ #include #include +#include #include #include #include "../core/Error.hpp" #include "../core/KeyValueData.hpp" +#include "../storage/MySqlConnection.hpp" #include "Exception.hpp" namespace spider { @@ -22,7 +24,7 @@ auto TaskContext::kv_store_get(std::string const& key) -> std::optional(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); std::string value; core::StorageErr const err = m_data_store->get_task_kv_data(conn, m_task_id, key, &value); @@ -41,7 +43,7 @@ auto TaskContext::kv_store_insert(std::string const& key, std::string const& val if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::KeyValueData const kv_data{key, value, m_task_id}; core::StorageErr const err = m_data_store->add_task_kv_data(conn, kv_data); @@ -56,7 +58,7 @@ auto TaskContext::get_jobs() -> std::vector { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); std::vector job_ids; core::StorageErr const err = m_metadata_store->get_jobs_by_client_id(conn, m_task_id, &job_ids); diff --git a/src/spider/client/TaskContext.hpp b/src/spider/client/TaskContext.hpp index 5151f8a8e..b370f6a85 100644 --- a/src/spider/client/TaskContext.hpp +++ b/src/spider/client/TaskContext.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,7 @@ #include "../core/TaskGraph.hpp" #include "../core/TaskGraphImpl.hpp" #include "../io/Serializer.hpp" +#include "../storage/MySqlConnection.hpp" #include "Data.hpp" #include "Exception.hpp" #include "Job.hpp" @@ -157,7 +159,7 @@ class TaskContext { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr err = m_metadata_store->add_job(conn, job_id, m_task_id, graph); if (!err.success()) { throw ConnectionException(fmt::format("Failed to start job: {}", err.description)); @@ -207,7 +209,7 @@ class TaskContext { if (std::holds_alternative(conn_result)) { throw ConnectionException(std::get(conn_result).description); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); core::StorageErr const err = m_metadata_store->add_job(conn, job_id, m_task_id, graph.m_impl->get_graph()); if (!err.success()) { From aad738eb42329b014bad17e454676bfd01c9bb28 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 13:33:27 -0500 Subject: [PATCH 16/21] Fix scheduler clang tidy --- src/spider/scheduler/FifoPolicy.cpp | 1 + src/spider/scheduler/FifoPolicy.hpp | 1 + src/spider/scheduler/scheduler.cpp | 7 +++---- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/spider/scheduler/FifoPolicy.cpp b/src/spider/scheduler/FifoPolicy.cpp index 153b1e8de..bc5fd0382 100644 --- a/src/spider/scheduler/FifoPolicy.cpp +++ b/src/spider/scheduler/FifoPolicy.cpp @@ -17,6 +17,7 @@ #include "../core/Task.hpp" #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" +#include "../storage/StorageConnection.hpp" #include "SchedulerTaskCache.hpp" namespace { diff --git a/src/spider/scheduler/FifoPolicy.hpp b/src/spider/scheduler/FifoPolicy.hpp index fa6bc63ec..d114875da 100644 --- a/src/spider/scheduler/FifoPolicy.hpp +++ b/src/spider/scheduler/FifoPolicy.hpp @@ -40,6 +40,7 @@ class FifoPolicy final : public SchedulerPolicy { std::shared_ptr m_metadata_store; std::shared_ptr m_data_store; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) core::StorageConnection& m_conn; SchedulerTaskCache m_task_cache; diff --git a/src/spider/scheduler/scheduler.cpp b/src/spider/scheduler/scheduler.cpp index e66cd078e..3ac8e85ea 100644 --- a/src/spider/scheduler/scheduler.cpp +++ b/src/spider/scheduler/scheduler.cpp @@ -26,7 +26,6 @@ #include "../storage/MetadataStorage.hpp" #include "../storage/MySqlConnection.hpp" #include "../storage/MySqlStorage.hpp" -#include "../storage/StorageConnection.hpp" #include "../utils/StopToken.hpp" #include "FifoPolicy.hpp" #include "SchedulerPolicy.hpp" @@ -89,7 +88,7 @@ auto heartbeat_loop( fail_count++; continue; } - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); spider::core::StorageErr const err = metadata_store->update_heartbeat(conn, scheduler.get_id()); if (!err.success()) { @@ -123,7 +122,7 @@ auto cleanup_loop( ); continue; } - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); spider::core::StorageErr err = metadata_store->set_scheduler_state(conn, scheduler.get_id(), "gc"); if (!err.success()) { @@ -199,7 +198,7 @@ auto main(int argc, char** argv) -> int { std::get(conn_result).description ); } - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); spider::core::StorageErr err = metadata_store->initialize(conn); if (!err.success()) { From b55c7085c0085d1dd3abd555b80280dc10f41ec8 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 13:51:25 -0500 Subject: [PATCH 17/21] Fix storage clang tidy --- src/spider/storage/DataStorage.hpp | 2 +- src/spider/storage/MetadataStorage.hpp | 2 +- src/spider/storage/MySqlStorage.cpp | 4 +++- src/spider/storage/MySqlStorage.hpp | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/spider/storage/DataStorage.hpp b/src/spider/storage/DataStorage.hpp index f5d5141dc..610750ae5 100644 --- a/src/spider/storage/DataStorage.hpp +++ b/src/spider/storage/DataStorage.hpp @@ -75,7 +75,7 @@ class DataStorage { std::string* value ) -> StorageErr = 0; - virtual auto get_url() const -> std::string const& = 0; + [[nodiscard]] virtual auto get_url() const -> std::string const& = 0; }; } // namespace spider::core diff --git a/src/spider/storage/MetadataStorage.hpp b/src/spider/storage/MetadataStorage.hpp index 6cd2b9ed7..47564f918 100644 --- a/src/spider/storage/MetadataStorage.hpp +++ b/src/spider/storage/MetadataStorage.hpp @@ -128,7 +128,7 @@ class MetadataStorage { std::string const& state ) -> StorageErr = 0; - virtual auto get_url() const -> std::string const& = 0; + [[nodiscard]] virtual auto get_url() const -> std::string const& = 0; }; } // namespace spider::core diff --git a/src/spider/storage/MySqlStorage.cpp b/src/spider/storage/MySqlStorage.cpp index 114ac3f88..012eed730 100644 --- a/src/spider/storage/MySqlStorage.cpp +++ b/src/spider/storage/MySqlStorage.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -285,6 +284,7 @@ auto string_to_task_state(std::string const& state) -> spider::core::TaskState { } } // namespace +// NOLINTBEGIN(cppcoreguidelines-pro-type-static-cast-downcast) auto MySqlMetadataStorage::initialize(StorageConnection& conn) -> StorageErr { try { for (char const* create_table_str : cCreateStorage) { @@ -2210,4 +2210,6 @@ auto MySqlDataStorage::get_task_kv_data( return StorageErr{}; } +// NOLINTEND(cppcoreguidelines-pro-type-static-cast-downcast) + } // namespace spider::core diff --git a/src/spider/storage/MySqlStorage.hpp b/src/spider/storage/MySqlStorage.hpp index ef2e5e7c2..a77953b14 100644 --- a/src/spider/storage/MySqlStorage.hpp +++ b/src/spider/storage/MySqlStorage.hpp @@ -118,7 +118,7 @@ class MySqlMetadataStorage : public MetadataStorage { std::string const& state ) -> StorageErr override; - auto get_url() const -> std::string const& override { return m_url; } + [[nodiscard]] auto get_url() const -> std::string const& override { return m_url; } private: std::string m_url; @@ -186,7 +186,7 @@ class MySqlDataStorage : public DataStorage { std::string* value ) -> StorageErr override; - auto get_url() const -> std::string const& override { return m_url; } + [[nodiscard]] auto get_url() const -> std::string const& override { return m_url; } private: std::string m_url; From 7012151ce0814aab0561f0f75e53392c4db83ebe Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 14:01:35 -0500 Subject: [PATCH 18/21] Fix worker clang tidy --- src/spider/worker/FunctionManager.hpp | 3 +-- src/spider/worker/WorkerClient.cpp | 5 +++-- src/spider/worker/worker.cpp | 7 +++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/spider/worker/FunctionManager.hpp b/src/spider/worker/FunctionManager.hpp index e9fbd0756..867764e30 100644 --- a/src/spider/worker/FunctionManager.hpp +++ b/src/spider/worker/FunctionManager.hpp @@ -27,7 +27,6 @@ #include "../io/Serializer.hpp" #include "../storage/DataStorage.hpp" #include "../storage/MySqlConnection.hpp" -#include "../storage/StorageConnection.hpp" #include "TaskExecutorMessage.hpp" // NOLINTBEGIN(cppcoreguidelines-macro-usage) @@ -293,7 +292,7 @@ class FunctionInvoker { fmt::format("Cannot parse arguments: {}.", err.description) ); } - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); for_n - 1>([&](auto i) { if (!err.success()) { return; diff --git a/src/spider/worker/WorkerClient.cpp b/src/spider/worker/WorkerClient.cpp index 512f7f6a5..5d82dc35b 100644 --- a/src/spider/worker/WorkerClient.cpp +++ b/src/spider/worker/WorkerClient.cpp @@ -9,12 +9,14 @@ #include #include #include +#include #include #include #include #include "../core/Driver.hpp" +#include "../core/Error.hpp" #include "../io/BoostAsio.hpp" // IWYU pragma: keep #include "../io/MsgPack.hpp" // IWYU pragma: keep #include "../io/msgpack_message.hpp" @@ -22,7 +24,6 @@ #include "../storage/DataStorage.hpp" #include "../storage/MetadataStorage.hpp" #include "../storage/MySqlConnection.hpp" -#include "../storage/StorageConnection.hpp" namespace spider::worker { @@ -51,7 +52,7 @@ auto WorkerClient::get_next_task(std::optional const& fail_t ); return std::nullopt; } - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); if (!m_metadata_store->get_active_scheduler(conn, &schedulers).success()) { return std::nullopt; } diff --git a/src/spider/worker/worker.cpp b/src/spider/worker/worker.cpp index e2c40656f..ab70fb2cb 100644 --- a/src/spider/worker/worker.cpp +++ b/src/spider/worker/worker.cpp @@ -40,7 +40,6 @@ #include "../storage/MetadataStorage.hpp" #include "../storage/MySqlConnection.hpp" #include "../storage/MySqlStorage.hpp" -#include "../storage/StorageConnection.hpp" #include "../utils/StopToken.hpp" #include "TaskExecutor.hpp" #include "WorkerClient.hpp" @@ -119,7 +118,7 @@ auto heartbeat_loop( fail_count++; continue; } - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); spider::core::StorageErr const err = metadata_store->update_heartbeat(conn, driver.get_id()); @@ -240,7 +239,7 @@ auto task_loop( ); continue; } - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); auto const [task_id, task_instance_id] = fetch_task(client, fail_task_id); spider::core::TaskInstance const instance{task_instance_id, task_id}; @@ -396,7 +395,7 @@ auto main(int argc, char** argv) -> int { ); return cStorageErr; } - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); spider::core::StorageErr const err = metadata_store->add_driver(conn, driver); if (!err.success()) { spdlog::error("Cannot add driver to metadata storage: {}", err.description); From f86ec97925baf0557c400542ce92a5ab54752b6a Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 14:04:15 -0500 Subject: [PATCH 19/21] Fix typo --- src/spider/worker/WorkerClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spider/worker/WorkerClient.cpp b/src/spider/worker/WorkerClient.cpp index 5d82dc35b..291aac682 100644 --- a/src/spider/worker/WorkerClient.cpp +++ b/src/spider/worker/WorkerClient.cpp @@ -47,7 +47,7 @@ auto WorkerClient::get_next_task(std::optional const& fail_t = spider::core::MySqlConnection::create(m_metadata_store->get_url()); if (std::holds_alternative(conn_result)) { spdlog::error( - "Failed to connection to storage: {}", + "Failed to connect to storage: {}", std::get(conn_result).description ); return std::nullopt; From 44028fb3f79619b85856857f60012a38959557e6 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 15:06:32 -0500 Subject: [PATCH 20/21] Fix clang tidy --- src/spider/scheduler/SchedulerTaskCache.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spider/scheduler/SchedulerTaskCache.hpp b/src/spider/scheduler/SchedulerTaskCache.hpp index 80e738fae..dae3d1ffd 100644 --- a/src/spider/scheduler/SchedulerTaskCache.hpp +++ b/src/spider/scheduler/SchedulerTaskCache.hpp @@ -49,6 +49,7 @@ class SchedulerTaskCache { std::shared_ptr m_metadata_store; std::shared_ptr m_data_store; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) core::StorageConnection& m_conn; // NOLINTNEXTLINE(misc-include-cleaner) From 13d591591f83aff19e126c5566087ca46f88c027 Mon Sep 17 00:00:00 2001 From: sitao Date: Mon, 3 Mar 2025 16:57:29 -0500 Subject: [PATCH 21/21] Fix tests clang tidy --- tests/scheduler/test-SchedulerPolicy.cpp | 9 ++++++--- tests/scheduler/test-SchedulerServer.cpp | 5 ++++- tests/storage/StorageTestHelper.hpp | 9 ++++++--- tests/storage/test-DataStorage.cpp | 12 +++++++----- tests/storage/test-MetadataStorage.cpp | 12 +++++++----- tests/worker/test-FunctionManager.cpp | 5 ++++- tests/worker/test-TaskExecutor.cpp | 5 ++++- 7 files changed, 38 insertions(+), 19 deletions(-) diff --git a/tests/scheduler/test-SchedulerPolicy.cpp b/tests/scheduler/test-SchedulerPolicy.cpp index f47aaf0e0..6f4b9005e 100644 --- a/tests/scheduler/test-SchedulerPolicy.cpp +++ b/tests/scheduler/test-SchedulerPolicy.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -14,11 +15,13 @@ #include "../../src/spider/core/Data.hpp" #include "../../src/spider/core/Driver.hpp" +#include "../../src/spider/core/Error.hpp" #include "../../src/spider/core/Task.hpp" #include "../../src/spider/core/TaskGraph.hpp" #include "../../src/spider/scheduler/FifoPolicy.hpp" #include "../../src/spider/storage/DataStorage.hpp" #include "../../src/spider/storage/MetadataStorage.hpp" +#include "../../src/spider/storage/MySqlConnection.hpp" #include "../storage/StorageTestHelper.hpp" namespace { @@ -40,7 +43,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_store->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; boost::uuids::uuid const client_id = gen(); @@ -93,7 +96,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_store->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); @@ -145,7 +148,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_store->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); // Add task boost::uuids::random_generator gen; diff --git a/tests/scheduler/test-SchedulerServer.cpp b/tests/scheduler/test-SchedulerServer.cpp index c81422838..a1bc746c1 100644 --- a/tests/scheduler/test-SchedulerServer.cpp +++ b/tests/scheduler/test-SchedulerServer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,7 @@ #include #include +#include "../../src/spider/core/Error.hpp" #include "../../src/spider/core/Task.hpp" #include "../../src/spider/core/TaskGraph.hpp" #include "../../src/spider/io/BoostAsio.hpp" // IWYU pragma: keep @@ -23,6 +25,7 @@ #include "../../src/spider/scheduler/SchedulerServer.hpp" #include "../../src/spider/storage/DataStorage.hpp" #include "../../src/spider/storage/MetadataStorage.hpp" +#include "../../src/spider/storage/MySqlConnection.hpp" #include "../../src/spider/utils/StopToken.hpp" #include "../storage/StorageTestHelper.hpp" @@ -48,7 +51,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_store->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); std::shared_ptr const policy = std::make_shared(metadata_store, data_store, conn); diff --git a/tests/storage/StorageTestHelper.hpp b/tests/storage/StorageTestHelper.hpp index a830dbf39..09fc3e5a1 100644 --- a/tests/storage/StorageTestHelper.hpp +++ b/tests/storage/StorageTestHelper.hpp @@ -6,11 +6,14 @@ #include #include #include +#include #include +#include "../../src/spider/core/Error.hpp" #include "../../src/spider/storage/DataStorage.hpp" #include "../../src/spider/storage/MetadataStorage.hpp" +#include "../../src/spider/storage/MySqlConnection.hpp" #include "../../src/spider/storage/MySqlStorage.hpp" namespace spider::test { @@ -28,7 +31,7 @@ auto create_data_storage() -> std::unique_ptr { std::variant conn_result = core::MySqlConnection::create(cStorageUrl); REQUIRE(std::holds_alternative(conn_result)); - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); REQUIRE(storage->initialize(conn).success()); return storage; } @@ -40,7 +43,7 @@ auto create_metadata_storage() -> std::unique_ptr { std::variant conn_result = core::MySqlConnection::create(cStorageUrl); REQUIRE(std::holds_alternative(conn_result)); - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); REQUIRE(storage->initialize(conn).success()); return storage; } @@ -52,7 +55,7 @@ auto create_storage( std::variant conn_result = core::MySqlConnection::create(cStorageUrl); REQUIRE(std::holds_alternative(conn_result)); - core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); std::unique_ptr metadata_storage = std::make_unique(cStorageUrl); REQUIRE(metadata_storage->initialize(conn).success()); diff --git a/tests/storage/test-DataStorage.cpp b/tests/storage/test-DataStorage.cpp index d0094e664..6bf250fd3 100644 --- a/tests/storage/test-DataStorage.cpp +++ b/tests/storage/test-DataStorage.cpp @@ -1,5 +1,6 @@ // NOLINTBEGIN(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) #include +#include #include #include @@ -12,6 +13,7 @@ #include "../../src/spider/core/KeyValueData.hpp" #include "../../src/spider/core/Task.hpp" #include "../../src/spider/core/TaskGraph.hpp" +#include "../../src/spider/storage/MySqlConnection.hpp" #include "../utils/CoreDataUtils.hpp" #include "StorageTestHelper.hpp" @@ -24,7 +26,7 @@ TEMPLATE_LIST_TEST_CASE("Add, get and remove data", "[storage]", spider::test::S std::variant conn_result = spider::core::MySqlConnection::create(metadata_storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); // Add driver and data spider::core::Data const data{"value"}; @@ -62,7 +64,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); // Add driver boost::uuids::random_generator gen; @@ -96,7 +98,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); // Add task boost::uuids::random_generator gen; @@ -137,7 +139,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; // Add task reference without data and task should fail. @@ -191,7 +193,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; diff --git a/tests/storage/test-MetadataStorage.cpp b/tests/storage/test-MetadataStorage.cpp index d9512f77f..39edf306c 100644 --- a/tests/storage/test-MetadataStorage.cpp +++ b/tests/storage/test-MetadataStorage.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,7 @@ #include "../../src/spider/core/Task.hpp" #include "../../src/spider/core/TaskGraph.hpp" #include "../../src/spider/storage/MetadataStorage.hpp" +#include "../../src/spider/storage/MySqlConnection.hpp" #include "../utils/CoreTaskUtils.hpp" #include "StorageTestHelper.hpp" @@ -29,7 +31,7 @@ TEMPLATE_LIST_TEST_CASE("Driver heartbeat", "[storage]", spider::test::MetadataS std::variant conn_result = spider::core::MySqlConnection::create(storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); constexpr double cDuration = 100; @@ -76,7 +78,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; boost::uuids::uuid const scheduler_id = gen(); @@ -123,7 +125,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); @@ -247,7 +249,7 @@ TEMPLATE_LIST_TEST_CASE("Task finish", "[storage]", spider::test::MetadataStorag std::variant conn_result = spider::core::MySqlConnection::create(storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); @@ -318,7 +320,7 @@ TEMPLATE_LIST_TEST_CASE("Job reset", "[storage]", spider::test::MetadataStorageT std::variant conn_result = spider::core::MySqlConnection::create(storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); diff --git a/tests/worker/test-FunctionManager.cpp b/tests/worker/test-FunctionManager.cpp index 5eeec46a4..2cd4de1fb 100644 --- a/tests/worker/test-FunctionManager.cpp +++ b/tests/worker/test-FunctionManager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -13,8 +14,10 @@ #include "../../src/spider/client/Data.hpp" #include "../../src/spider/client/TaskContext.hpp" #include "../../src/spider/core/Driver.hpp" +#include "../../src/spider/core/Error.hpp" #include "../../src/spider/core/TaskContextImpl.hpp" #include "../../src/spider/io/MsgPack.hpp" // IWYU pragma: keep +#include "../../src/spider/storage/MySqlConnection.hpp" #include "../../src/spider/worker/FunctionManager.hpp" #include "../../src/spider/worker/FunctionNameManager.hpp" #include "../storage/StorageTestHelper.hpp" @@ -151,7 +154,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); msgpack::sbuffer buffer; msgpack::pack(buffer, 3); diff --git a/tests/worker/test-TaskExecutor.cpp b/tests/worker/test-TaskExecutor.cpp index 253ba69fd..99cf0f445 100644 --- a/tests/worker/test-TaskExecutor.cpp +++ b/tests/worker/test-TaskExecutor.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -16,10 +17,12 @@ #include "../../src/spider/core/Data.hpp" #include "../../src/spider/core/Driver.hpp" +#include "../../src/spider/core/Error.hpp" #include "../../src/spider/io/BoostAsio.hpp" // IWYU pragma: keep #include "../../src/spider/io/MsgPack.hpp" // IWYU pragma: keep #include "../../src/spider/storage/DataStorage.hpp" #include "../../src/spider/storage/MetadataStorage.hpp" +#include "../../src/spider/storage/MySqlConnection.hpp" #include "../../src/spider/worker/FunctionManager.hpp" #include "../../src/spider/worker/TaskExecutor.hpp" #include "../storage/StorageTestHelper.hpp" @@ -149,7 +152,7 @@ TEMPLATE_LIST_TEST_CASE( std::variant conn_result = spider::core::MySqlConnection::create(metadata_storage->get_url()); REQUIRE(std::holds_alternative(conn_result)); - spider::core::MySqlConnection& conn = std::get(conn_result); + auto& conn = std::get(conn_result); // Create driver and data msgpack::sbuffer buffer;