diff --git a/src/spider/CMakeLists.txt b/src/spider/CMakeLists.txt index cdcf2be45..6c08825f4 100644 --- a/src/spider/CMakeLists.txt +++ b/src/spider/CMakeLists.txt @@ -10,11 +10,9 @@ set(SPIDER_CORE_SOURCES set(SPIDER_CORE_HEADERS core/Error.hpp core/Data.hpp - core/DataImpl.hpp core/Driver.hpp core/KeyValueData.hpp core/Task.hpp - core/TaskContextImpl.hpp core/TaskGraph.hpp core/JobMetadata.hpp io/BoostAsio.hpp @@ -124,7 +122,11 @@ target_link_libraries( spdlog::spdlog ) -set(SPIDER_CLIENT_SHARED_SOURCES CACHE INTERNAL "spider client shared source files") +set(SPIDER_CLIENT_SHARED_SOURCES + client/Driver.cpp + CACHE INTERNAL + "spider client shared source files" +) set(SPIDER_CLIENT_SHARED_HEADERS client/Data.hpp @@ -134,6 +136,9 @@ set(SPIDER_CLIENT_SHARED_HEADERS client/TaskGraph.hpp client/type_utils.hpp client/Exception.hpp + core/DataImpl.hpp + core/TaskContextImpl.hpp + core/TaskGraphImpl.hpp CACHE INTERNAL "spider client shared header files" ) diff --git a/src/spider/client/Data.hpp b/src/spider/client/Data.hpp index 99d524a91..c3249ac80 100644 --- a/src/spider/client/Data.hpp +++ b/src/spider/client/Data.hpp @@ -22,7 +22,10 @@ namespace core { class Data; class DataStorage; class DataImpl; +class TaskGraphImpl; } // namespace core +class Driver; +class TaskContext; /** * A representation of data stored on external storage. This class allows the user to define: @@ -120,7 +123,7 @@ class Data { } break; } - return Data{data, m_data_store}; + return Data{std::move(data), m_data_store}; } private: @@ -129,11 +132,9 @@ class Data { TaskContext }; - explicit Builder( - std::shared_ptr data_store, + Builder(std::shared_ptr data_store, boost::uuids::uuid const source_id, - DataSource const data_source - ) + DataSource const data_source) : m_data_store{std::move(data_store)}, m_source_id{source_id}, m_data_source{data_source} {} @@ -163,6 +164,7 @@ class Data { std::shared_ptr m_data_store; friend class core::DataImpl; + friend class core::TaskGraphImpl; }; } // namespace spider diff --git a/src/spider/client/Driver.cpp b/src/spider/client/Driver.cpp new file mode 100644 index 000000000..02da9f754 --- /dev/null +++ b/src/spider/client/Driver.cpp @@ -0,0 +1,121 @@ +#include "Driver.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "../core/Driver.hpp" +#include "../core/Error.hpp" +#include "../core/KeyValueData.hpp" +#include "../io/BoostAsio.hpp" // IWYU pragma: keep +#include "../storage/MysqlStorage.hpp" +#include "Exception.hpp" + +namespace spider { + +Driver::Driver(std::string const& storage_url) { + boost::uuids::random_generator gen; + m_id = gen(); + + m_metadata_storage = std::make_shared(); + m_data_storage = std::make_shared(); + core::StorageErr err = m_metadata_storage->connect(storage_url); + if (!err.success()) { + throw ConnectionException(err.description); + } + err = m_data_storage->connect(storage_url); + if (!err.success()) { + throw ConnectionException(err.description); + } + + std::optional const optional_addr = core::get_address(); + if (!optional_addr.has_value()) { + throw ConnectionException("Cannot get machine address"); + } + std::string const& addr = optional_addr.value(); + err = m_metadata_storage->add_driver(core::Driver{m_id, addr}); + if (!err.success()) { + if (core::StorageErrType::DuplicateKeyErr == err.type) { + throw DriverIdInUseException(m_id); + } + throw ConnectionException(err.description); + } + + // Start a thread to send heartbeats + // NOLINTNEXTLINE(performance-unnecessary-value-param) + m_heartbeat_thread = std::jthread([this](std::stop_token stoken) { + while (!stoken.stop_requested()) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + core::StorageErr const err = m_metadata_storage->update_heartbeat(m_id); + if (!err.success()) { + throw ConnectionException(err.description); + } + } + }); +} + +Driver::Driver(std::string const& storage_url, boost::uuids::uuid const id) : m_id{id} { + m_metadata_storage = std::make_shared(); + m_data_storage = std::make_shared(); + core::StorageErr err = m_metadata_storage->connect(storage_url); + if (!err.success()) { + throw ConnectionException(err.description); + } + err = m_data_storage->connect(storage_url); + if (!err.success()) { + throw ConnectionException(err.description); + } + + std::optional const optional_addr = core::get_address(); + if (!optional_addr.has_value()) { + throw ConnectionException("Cannot get machine address"); + } + std::string const& addr = optional_addr.value(); + err = m_metadata_storage->add_driver(core::Driver{m_id, addr}); + if (!err.success()) { + if (core::StorageErrType::DuplicateKeyErr == err.type) { + throw DriverIdInUseException(m_id); + } + throw ConnectionException(err.description); + } + + // Start a thread to send heartbeats + // NOLINTNEXTLINE(performance-unnecessary-value-param) + m_heartbeat_thread = std::jthread([this](std::stop_token stoken) { + while (!stoken.stop_requested()) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + core::StorageErr const err = m_metadata_storage->update_heartbeat(m_id); + if (!err.success()) { + throw ConnectionException(err.description); + } + } + }); +} + +auto Driver::kv_store_insert(std::string const& key, std::string const& value) -> void { + core::KeyValueData const kv_data{key, value, m_id}; + core::StorageErr const err = m_data_storage->add_client_kv_data(kv_data); + if (!err.success()) { + throw ConnectionException(err.description); + } +} + +auto Driver::kv_store_get(std::string const& key) -> std::optional { + std::string value; + core::StorageErr const err = m_data_storage->get_client_kv_data(m_id, key, &value); + if (!err.success()) { + if (core::StorageErrType::KeyNotFoundErr == err.type) { + return std::nullopt; + } + throw ConnectionException(err.description); + } + return value; +} + +} // namespace spider diff --git a/src/spider/client/Driver.hpp b/src/spider/client/Driver.hpp index d0f43ec73..6e30a21bb 100644 --- a/src/spider/client/Driver.hpp +++ b/src/spider/client/Driver.hpp @@ -1,18 +1,23 @@ #ifndef SPIDER_CLIENT_DRIVER_HPP #define SPIDER_CLIENT_DRIVER_HPP +#include #include +#include #include +#include #include #include +#include "../core/TaskGraphImpl.hpp" #include "../io/Serializer.hpp" #include "../worker/FunctionManager.hpp" #include "Data.hpp" #include "Job.hpp" #include "task.hpp" #include "TaskGraph.hpp" +#include "utility" /** * Registers a Task function with Spider @@ -31,6 +36,10 @@ #define SPIDER_REGISTER_TASK_TIMEOUT(func, timeout) SPIDER_WORKER_REGISTER_TASK(func) namespace spider { +namespace core { +class MetadataStorage; +class DataStorage; +} // namespace core /** * An interface for a client to interact with Spider and create jobs, access the kv-store, etc. @@ -58,7 +67,10 @@ class Driver { * @return Data builder. */ template - auto get_data_builder() -> Data::Builder; + auto get_data_builder() -> Data::Builder { + using DataBuilder = typename Data::Builder; + return DataBuilder{m_data_storage, m_id, DataBuilder::DataSource::Driver}; + } /** * Inserts the given key-value pair into the key-value store, overwriting any existing value. @@ -67,7 +79,7 @@ class Driver { * @param value * @throw spider::ConnectionException */ - auto kv_store_insert(std::string const& key, std::string const& value); + auto kv_store_insert(std::string const& key, std::string const& value) -> void; /** * Gets the value corresponding to the given key. @@ -90,19 +102,24 @@ class Driver { * @tparam ReturnType Return type for both the task and the resulting `TaskGraph`. * @tparam TaskParams * @tparam Inputs - * @tparam GraphParams * @param task * @param inputs Inputs to bind to `task`. If an input is a `Task` or `TaskGraph`, their * outputs will be bound to the inputs of `task`. * @return A `TaskGraph` of the inputs bound to `task`. */ - template < - TaskIo ReturnType, - TaskIo... TaskParams, - RunnableOrTaskIo... Inputs, - TaskIo... GraphParams> + template auto bind(TaskFunction const& task, Inputs&&... inputs) - -> TaskGraph; + -> TaskGraphType { + std::optional optional_graph + = core::TaskGraphImpl::bind(task, std::forward(inputs)...); + if (!optional_graph.has_value()) { + throw std::invalid_argument("Failed to bind inputs to task."); + } + std::unique_ptr graph + = std::make_unique(std::move(optional_graph.value())); + + return TaskGraphType{std::move(graph)}; + } /** * Starts running a task with the given inputs on Spider. @@ -141,6 +158,12 @@ class Driver { * @throw spider::ConnectionException */ auto get_jobs() -> std::vector; + +private: + boost::uuids::uuid m_id; + std::shared_ptr m_metadata_storage; + std::shared_ptr m_data_storage; + std::jthread m_heartbeat_thread; }; } // namespace spider diff --git a/src/spider/client/TaskGraph.hpp b/src/spider/client/TaskGraph.hpp index 4da2c0a92..c5f4f18a8 100644 --- a/src/spider/client/TaskGraph.hpp +++ b/src/spider/client/TaskGraph.hpp @@ -1,9 +1,19 @@ #ifndef SPIDER_CLIENT_TASKGRAPH_HPP #define SPIDER_CLIENT_TASKGRAPH_HPP +#include +#include + #include "task.hpp" namespace spider { +namespace core { +class TaskGraphImpl; +} // namespace core + +class Driver; +class TaskContext; + /** * A TaskGraph represents a directed acyclic graph (DAG) of tasks. * @@ -11,7 +21,18 @@ namespace spider { * @tparam Params */ template -class TaskGraph {}; +class TaskGraph { +private: + explicit TaskGraph(std::unique_ptr impl) : m_impl{std::move(impl)} {} + + [[nodiscard]] auto get_impl() const -> core::TaskGraphImpl const& { return *m_impl; } + + std::unique_ptr m_impl; + + friend class core::TaskGraphImpl; + friend class Driver; + friend class TaskContext; +}; } // namespace spider #endif // SPIDER_CLIENT_TASKGRAPH_HPP diff --git a/src/spider/client/task.hpp b/src/spider/client/task.hpp index c89cb0aa3..5a48adc5b 100644 --- a/src/spider/client/task.hpp +++ b/src/spider/client/task.hpp @@ -1,7 +1,7 @@ #ifndef SPIDER_CLIENT_TASK_HPP #define SPIDER_CLIENT_TASK_HPP -#include +#include #include "../io/Serializer.hpp" #include "Data.hpp" @@ -28,7 +28,7 @@ class TaskContext; * @tparam TaskParams */ template -using TaskFunction = std::function; +using TaskFunction = ReturnType (*)(TaskContext&, TaskParams...); // Forward declare `TaskGraph` since `Runnable` takes `TaskGraph` as a param, and `TaskGraph` uses // `TaskIo` defined in this header as its template params. @@ -45,6 +45,54 @@ concept Runnable = cIsSpecializationV || cIsSpecializationV concept RunnableOrTaskIo = Runnable || TaskIo; + +template +struct ConcatTaskGraphType; + +template +struct ConcatTaskGraphType> { + using Type = TaskGraph; +}; + +template +struct ConcatTaskGraphType, InputType> { + using Type = TaskGraph; +}; + +template +struct ConcatTaskGraphType< + TaskGraph, + TaskFunction> { + using Type = TaskGraph; +}; + +template +struct ConcatTaskGraphType< + TaskGraph, + TaskGraph> { + using Type = TaskGraph; +}; + +template +struct MergeTaskGraphTypes; + +template +struct MergeTaskGraphTypes> { + using Type = TaskGraph; +}; + +template +struct MergeTaskGraphTypes, InputType, Inputs...> { + using Type = typename MergeTaskGraphTypes< + typename ConcatTaskGraphType< + TaskGraph, + std::remove_cvref_t>::Type, + Inputs...>::Type; +}; + +template +using TaskGraphType = typename MergeTaskGraphTypes, Inputs...>::Type; + } // namespace spider #endif // SPIDER_CLIENT_TASK_HPP diff --git a/src/spider/core/Task.hpp b/src/spider/core/Task.hpp index 06af3539c..16e61ace5 100644 --- a/src/spider/core/Task.hpp +++ b/src/spider/core/Task.hpp @@ -15,6 +15,8 @@ namespace spider::core { class TaskInput { public: + explicit TaskInput(std::string type) : m_type(std::move(type)) {}; + TaskInput(boost::uuids::uuid output_task_id, std::uint8_t position, std::string type) : m_task_output({output_task_id, position}), m_type(std::move(type)) {}; @@ -42,6 +44,11 @@ class TaskInput { void set_data_id(boost::uuids::uuid data_id) { m_data_id = data_id; } + void + set_output(boost::uuids::uuid const output_task_id, std::uint8_t const output_task_position) { + m_task_output = {output_task_id, output_task_position}; + } + private: std::optional> m_task_output; std::optional m_value; @@ -113,6 +120,8 @@ class Task { m_state(state), m_timeout(timeout) {} + void set_id(boost::uuids::uuid const id) { m_id = id; } + void set_max_retries(unsigned int num_retries) { m_max_tries = num_retries; } void add_input(TaskInput const& input) { m_inputs.emplace_back(input); } @@ -135,6 +144,8 @@ class Task { [[nodiscard]] auto get_input(uint64_t index) const -> TaskInput { return m_inputs[index]; } + [[nodiscard]] auto get_input_ref(uint64_t index) -> TaskInput& { return m_inputs[index]; } + [[nodiscard]] auto get_output(uint64_t index) const -> TaskOutput { return m_outputs[index]; } [[nodiscard]] auto get_inputs() const -> std::vector const& { return m_inputs; } diff --git a/src/spider/core/TaskGraph.hpp b/src/spider/core/TaskGraph.hpp index 01cc8d023..61cde5377 100644 --- a/src/spider/core/TaskGraph.hpp +++ b/src/spider/core/TaskGraph.hpp @@ -1,12 +1,15 @@ #ifndef SPIDER_CORE_TASKGRAPH_HPP #define SPIDER_CORE_TASKGRAPH_HPP +#include +#include #include +#include #include #include #include -#include +#include #include #include "Task.hpp" @@ -88,25 +91,72 @@ class TaskGraph { return m_tasks; } - [[nodiscard]] auto get_head_tasks() const -> absl::flat_hash_set { - absl::flat_hash_set heads; - for (auto const& pair : m_tasks) { - heads.emplace(pair.first); - } - for (auto const& pair : m_dependencies) { - heads.erase(pair.second); - } - return heads; + [[nodiscard]] auto get_input_tasks() const -> std::vector const& { + return m_input_tasks; } + [[nodiscard]] auto get_output_tasks() const -> std::vector const& { + return m_output_tasks; + } + + auto add_input_task(boost::uuids::uuid id) -> void { m_input_tasks.emplace_back(id); } + + auto add_output_task(boost::uuids::uuid id) -> void { m_output_tasks.emplace_back(id); } + [[nodiscard]] auto get_dependencies( ) const -> std::vector> const& { return m_dependencies; } + auto reset_ids() -> void { + absl::flat_hash_map new_id_map; + boost::uuids::random_generator gen; + for (auto const& [old_id, task] : m_tasks) { + boost::uuids::uuid new_id = gen(); + new_id_map.emplace(old_id, new_id); + } + // Replace all id in task map and task + absl::flat_hash_map new_tasks; + for (auto& [old_id, task] : m_tasks) { + boost::uuids::uuid new_id = new_id_map.at(old_id); + task.set_id(new_id); + for (size_t i = 0; i < task.get_num_inputs(); i++) { + TaskInput& input = task.get_input_ref(i); + std::optional> const& optional_task_output + = input.get_task_output(); + if (optional_task_output.has_value()) { + boost::uuids::uuid const task_id = std::get<0>(optional_task_output.value()); + input.set_output( + new_id_map.at(task_id), + std::get<1>(optional_task_output.value()) + ); + } + } + new_tasks.emplace(new_id, std::move(task)); + } + m_tasks = std::move(new_tasks); + + // Replace all id in dependencies + for (auto& dep : m_dependencies) { + dep.first = new_id_map.at(dep.first); + dep.second = new_id_map.at(dep.second); + } + + // Replace all id in input and output tasks + for (auto& task : m_input_tasks) { + task = new_id_map.at(task); + } + for (auto& task : m_output_tasks) { + task = new_id_map.at(task); + } + } + private: absl::flat_hash_map m_tasks; std::vector> m_dependencies; + + std::vector m_input_tasks; + std::vector m_output_tasks; }; } // namespace spider::core diff --git a/src/spider/core/TaskGraphImpl.hpp b/src/spider/core/TaskGraphImpl.hpp new file mode 100644 index 000000000..17aee657c --- /dev/null +++ b/src/spider/core/TaskGraphImpl.hpp @@ -0,0 +1,197 @@ +#ifndef SPIDER_CORE_TASKGRAPHIMPL_HPP +#define SPIDER_CORE_TASKGRAPHIMPL_HPP + +#include +#include +#include +#include +#include + +#include + +#include "../client/Data.hpp" +#include "../client/task.hpp" +#include "../client/type_utils.hpp" +#include "../core/Task.hpp" +#include "../core/TaskGraph.hpp" +#include "../io/MsgPack.hpp" // IWYU pragma: keep +#include "../io/Serializer.hpp" // IWYU pragma: keep +#include "../worker/FunctionManager.hpp" + +namespace spider::core { + +class TaskGraphImpl { +public: + // NOLINTBEGIN(readability-function-cognitive-complexity, cppcoreguidelines-missing-std-forward) + template + static auto + bind(TaskFunction const& task_function, + Inputs&&... inputs) -> std::optional { + std::optional optional_task = create_task(task_function); + if (!optional_task.has_value()) { + return std::nullopt; + } + Task& task = optional_task.value(); + TaskGraphImpl graph; + + uint64_t position = 0; + bool fail = false; + for_n([&](auto i) { + if (fail) { + return; + } + using InputType + = std::remove_cvref_t>>; + if constexpr (std::is_pointer_v + && std::is_function_v>) + { + std::optional optional_parent = add_task_input( + task, + std::get(std::forward_as_tuple(inputs...)), + position + ); + if (!optional_parent.has_value()) { + fail = true; + } + graph.m_graph.add_task(optional_parent.value()); + graph.m_graph.add_input_task(optional_parent.value().get_id()); + } else if constexpr (cIsSpecializationV) { + TaskGraph parent_graph + = std::get(std::forward_as_tuple(inputs...)).get_impl().m_graph; + parent_graph.reset_ids(); + if (!add_graph_input(task, parent_graph, position)) { + fail = true; + } + for (boost::uuids::uuid const& intput_task_id : parent_graph.get_input_tasks()) { + graph.m_graph.add_input_task(intput_task_id); + } + for (auto const& [task_id, task] : parent_graph.get_tasks()) { + graph.m_graph.add_task(task); + } + } else if constexpr (TaskIo) { + if (position >= task.get_num_inputs()) { + fail = true; + } + TaskInput& input = task.get_input_ref(position); + position++; + // Check type match + if (input.get_type() != typeid(InputType).name()) { + fail = true; + } + if constexpr (cIsSpecializationV) { + input.set_data_id(std::get(std::forward_as_tuple(inputs...)) + .get_impl() + ->get_id()); + } else if constexpr (Serializable) { + msgpack::sbuffer buffer; + msgpack::pack(buffer, std::get(std::forward_as_tuple(inputs...))); + std::string const value(buffer.data(), buffer.size()); + input.set_value(value); + } + } + }); + if (fail) { + return std::nullopt; + } + + // Check all inputs are consumed + if (position != task.get_num_inputs()) { + return std::nullopt; + } + + graph.m_graph.add_task(task); + graph.m_graph.add_output_task(task.get_id()); + return graph; + } + + // NOLINTEND(readability-function-cognitive-complexity, cppcoreguidelines-missing-std-forward) + +private: + template + static auto create_task(TaskFunction const& task_function + ) -> std::optional { + // NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast) + std::optional const function_name + = FunctionManager::get_instance().get_function_name( + reinterpret_cast(task_function) + ); + // NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast) + if (!function_name.has_value()) { + return std::nullopt; + } + Task task{function_name.value()}; + // Add task inputs + ((task.add_input(TaskInput{typeid(TaskParams).name()})), ...); + // Add task outputs + if constexpr (cIsSpecializationV) { + for_n>([&](auto i) { + task.add_output(TaskOutput{typeid(std::tuple_element_t).name() + }); + }); + } else { + task.add_output(TaskOutput{typeid(ReturnType).name()}); + } + return task; + } + + template + static auto add_task_input( + Task& task, + TaskFunction const& task_function, + uint64_t& position + ) -> std::optional { + std::optional const optional_parent = create_task(task_function); + if (!optional_parent.has_value()) { + return std::nullopt; + } + Task const& parent = optional_parent.value(); + if constexpr (cIsSpecializationV) { + for_n>([&](auto i) { + if (position >= sizeof...(TaskParams)) { + return std::nullopt; + } + TaskInput& input = task.get_input_ref(position); + position++; + input.set_output(parent.get_id(), i.cValue); + }); + } else { + if (position >= sizeof...(TaskParams)) { + return std::nullopt; + } + TaskInput& input = task.get_input_ref(position); + position++; + input.set_output(parent.get_id(), 0); + } + return parent; + } + + static auto add_graph_input(Task& task, TaskGraph const& graph, uint64_t& position) -> bool { + for (boost::uuids::uuid const& output_task_id : graph.get_output_tasks()) { + std::optional optional_output_task = graph.get_task(output_task_id); + if (!optional_output_task.has_value()) { + return false; + } + Task const& output_task = *optional_output_task.value(); + for (int64_t i = 0; i < output_task.get_num_outputs(); i++) { + if (position >= task.get_num_inputs()) { + return false; + } + TaskInput& input = task.get_input_ref(position); + position++; + // Check type match + TaskOutput const& output = output_task.get_output(i); + if (input.get_type() != output.get_type()) { + return false; + } + input.set_output(output_task_id, i); + } + } + return true; + } + + TaskGraph m_graph; +}; + +} // namespace spider::core + +#endif // SPIDER_CORE_TASKGRAPHIMPL_HPP diff --git a/src/spider/storage/MysqlStorage.cpp b/src/spider/storage/MysqlStorage.cpp index 6b2bf5555..795527aa1 100644 --- a/src/spider/storage/MysqlStorage.cpp +++ b/src/spider/storage/MysqlStorage.cpp @@ -486,7 +486,11 @@ auto MySqlMetadataStorage::add_job( } // Tasks must be added in graph order to avoid the dangling reference. - absl::flat_hash_set heads = task_graph.get_head_tasks(); + std::vector const& inputs = task_graph.get_input_tasks(); + absl::flat_hash_set heads; + for (boost::uuids::uuid const task_id : inputs) { + heads.insert(task_id); + } std::deque queue; // First go over all heads for (boost::uuids::uuid const task_id : heads) { @@ -535,7 +539,7 @@ auto MySqlMetadataStorage::add_job( } // Mark head tasks as ready - for (boost::uuids::uuid const& task_id : task_graph.get_head_tasks()) { + for (boost::uuids::uuid const& task_id : task_graph.get_input_tasks()) { std::unique_ptr statement( m_conn->prepareStatement("UPDATE `tasks` SET `state` = 'ready' WHERE `id` = ?") ); diff --git a/src/spider/worker/FunctionManager.cpp b/src/spider/worker/FunctionManager.cpp index a155bcac4..7f7f5338e 100644 --- a/src/spider/worker/FunctionManager.cpp +++ b/src/spider/worker/FunctionManager.cpp @@ -61,12 +61,19 @@ void create_error_buffer( } auto FunctionManager::get_function(std::string const& name) const -> Function const* { - if (auto const func_iter = m_map.find(name); func_iter != m_map.end()) { + if (auto const func_iter = m_function_map.find(name); func_iter != m_function_map.end()) { return &(func_iter->second); } return nullptr; } +auto FunctionManager::get_function_name(void const* ptr) const -> std::optional { + if (auto const name_iter = m_name_map.find(ptr); name_iter != m_name_map.end()) { + return name_iter->second; + } + return std::nullopt; +} + } // namespace spider::core // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/src/spider/worker/FunctionManager.hpp b/src/spider/worker/FunctionManager.hpp index 2397e8f84..3eaee6ffe 100644 --- a/src/spider/worker/FunctionManager.hpp +++ b/src/spider/worker/FunctionManager.hpp @@ -49,6 +49,8 @@ using Function = std::function; +using FunctionNameMap = absl::flat_hash_map; + template struct TemplateParameter; @@ -382,7 +384,12 @@ class FunctionManager { template auto register_function(std::string const& name, F f) -> bool { - return m_map + if (m_function_map.contains(name)) { + return false; + } + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + m_name_map.emplace(reinterpret_cast(f), name); + return m_function_map .emplace( name, std::bind( @@ -396,19 +403,27 @@ class FunctionManager { } auto register_function_invoker(std::string const& name, Function f) -> bool { - return m_map.emplace(name, f).second; + return m_function_map.emplace(name, f).second; } [[nodiscard]] auto get_function(std::string const& name) const -> Function const*; - [[nodiscard]] auto get_function_map() const -> FunctionMap const& { return m_map; } + [[nodiscard]] auto get_function_map() const -> FunctionMap const& { return m_function_map; } + + [[nodiscard]] auto get_function_name(void const* ptr) const -> std::optional; + + [[nodiscard]] auto get_function_name_map() const -> FunctionNameMap const& { + return m_name_map; + } private: FunctionManager() = default; ~FunctionManager() = default; - FunctionMap m_map; + FunctionMap m_function_map; + + FunctionNameMap m_name_map; }; } // namespace spider::core diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7efe04160..47e09198e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,6 +11,7 @@ set(SPIDER_TEST_SOURCES io/test-MsgpackMessage.cpp scheduler/test-SchedulerPolicy.cpp scheduler/test-SchedulerServer.cpp + client/test-Driver.cpp CACHE INTERNAL "spider test source files" ) diff --git a/tests/client/test-Driver.cpp b/tests/client/test-Driver.cpp new file mode 100644 index 000000000..64d87c170 --- /dev/null +++ b/tests/client/test-Driver.cpp @@ -0,0 +1,64 @@ +// NOLINTBEGIN(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) + +#include +#include + +#include + +#include "../../src/spider/client/Data.hpp" +#include "../../src/spider/client/Driver.hpp" +#include "../../src/spider/client/TaskContext.hpp" +#include "../../src/spider/client/TaskGraph.hpp" +#include "../storage/StorageTestHelper.hpp" + +namespace { +TEST_CASE("Driver kv store", "[client][storage]") { + spider::Driver driver{spider::test::cStorageUrl}; + driver.kv_store_insert("key", "value"); + + // Get value by key should succeed + std::optional const result = driver.kv_store_get("key"); + REQUIRE(result.has_value()); + if (result.has_value()) { + REQUIRE(result.value() == "value"); + } + // Get value by wrong key should fail + std::optional const fail_result = driver.kv_store_get("wrong_key"); + REQUIRE(!fail_result.has_value()); +} + +TEST_CASE("Driver data", "[client][storage]") { + spider::Driver driver{spider::test::cStorageUrl}; + spider::Data const data = driver.get_data_builder().build(1); +} + +auto sum(spider::TaskContext&, int x, int y) -> int { + return x + y; +} + +auto test_driver(spider::TaskContext&, spider::Data& x) -> int { + return x.get(); +} + +SPIDER_REGISTER_TASK(sum); +SPIDER_REGISTER_TASK(test_driver); + +TEST_CASE("Driver bind task", "[client][storage]") { + spider::Driver driver{spider::test::cStorageUrl}; + + spider::TaskGraph const graph_1 = driver.bind(&sum, &sum, 0); + spider::TaskGraph const graph_3 = driver.bind(&sum, &sum, &sum); + spider::TaskGraph const graph_4 = driver.bind(&sum, graph_1, graph_1); +} + +TEST_CASE("Driver bind task with data", "[client][storage]") { + spider::Driver driver{spider::test::cStorageUrl}; + + spider::Data data = driver.get_data_builder().build(1); + spider::TaskGraph const graph_1 = driver.bind(&test_driver, data); + spider::TaskGraph const graph_2 = driver.bind(&sum, &sum, graph_1); +} + +} // namespace + +// NOLINTEND(cert-err58-cpp,cppcoreguidelines-avoid-do-while,readability-function-cognitive-complexity,cppcoreguidelines-avoid-non-const-global-variables,cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays) diff --git a/tests/scheduler/test-SchedulerPolicy.cpp b/tests/scheduler/test-SchedulerPolicy.cpp index c6c665a45..9a81ab09a 100644 --- a/tests/scheduler/test-SchedulerPolicy.cpp +++ b/tests/scheduler/test-SchedulerPolicy.cpp @@ -43,12 +43,16 @@ TEMPLATE_LIST_TEST_CASE( spider::core::Task const task_1{"task_1"}; spider::core::TaskGraph graph_1; graph_1.add_task(task_1); + graph_1.add_input_task(task_1.get_id()); + graph_1.add_output_task(task_1.get_id()); boost::uuids::uuid const job_id_1 = gen(); REQUIRE(metadata_store->add_job(job_id_1, client_id, graph_1).success()); std::this_thread::sleep_for(std::chrono::seconds(1)); spider::core::Task const task_2{"task_2"}; spider::core::TaskGraph graph_2; graph_2.add_task(task_2); + graph_2.add_input_task(task_2.get_id()); + graph_2.add_output_task(task_2.get_id()); boost::uuids::uuid const job_id_2 = gen(); REQUIRE(metadata_store->add_job(job_id_2, client_id, graph_2).success()); @@ -95,6 +99,8 @@ TEMPLATE_LIST_TEST_CASE( task.add_input(spider::core::TaskInput{data.get_id(), "int"}); spider::core::TaskGraph graph; graph.add_task(task); + graph.add_input_task(task.get_id()); + graph.add_output_task(task.get_id()); REQUIRE(metadata_store->add_job(job_id, client_id, graph).success()); spider::scheduler::FifoPolicy policy; @@ -140,6 +146,8 @@ TEMPLATE_LIST_TEST_CASE( task.add_input(spider::core::TaskInput{data.get_id(), "int"}); spider::core::TaskGraph graph; graph.add_task(task); + graph.add_input_task(task.get_id()); + graph.add_output_task(task.get_id()); REQUIRE(metadata_store->add_job(job_id, client_id, graph).success()); spider::scheduler::FifoPolicy policy; diff --git a/tests/scheduler/test-SchedulerServer.cpp b/tests/scheduler/test-SchedulerServer.cpp index c2bafcbcc..6d4fcda53 100644 --- a/tests/scheduler/test-SchedulerServer.cpp +++ b/tests/scheduler/test-SchedulerServer.cpp @@ -72,6 +72,8 @@ TEMPLATE_LIST_TEST_CASE( graph.add_task(parent_task); graph.add_task(child_task); graph.add_dependency(parent_task.get_id(), child_task.get_id()); + graph.add_input_task(parent_task.get_id()); + graph.add_output_task(child_task.get_id()); boost::uuids::random_generator gen; boost::uuids::uuid const job_id = gen(); REQUIRE(metadata_store->add_job(job_id, gen(), graph).success()); diff --git a/tests/storage/test-DataStorage.cpp b/tests/storage/test-DataStorage.cpp index 4fa76a8aa..e19f23fa1 100644 --- a/tests/storage/test-DataStorage.cpp +++ b/tests/storage/test-DataStorage.cpp @@ -88,6 +88,8 @@ TEMPLATE_LIST_TEST_CASE( spider::core::Task const task{"func"}; spider::core::TaskGraph graph; graph.add_task(task); + graph.add_input_task(task.get_id()); + graph.add_output_task(task.get_id()); boost::uuids::uuid const job_id = gen(); REQUIRE(metadata_storage->add_job(job_id, gen(), graph).success()); @@ -127,6 +129,9 @@ TEMPLATE_LIST_TEST_CASE( spider::core::TaskGraph graph; graph.add_task(task); graph.add_task(task_2); + graph.add_dependency(task.get_id(), task_2.get_id()); + graph.add_input_task(task.get_id()); + graph.add_output_task(task_2.get_id()); boost::uuids::uuid const job_id = gen(); REQUIRE(metadata_storage->add_job(job_id, gen(), graph).success()); diff --git a/tests/storage/test-MetadataStorage.cpp b/tests/storage/test-MetadataStorage.cpp index c845e0cd5..c96ad59dc 100644 --- a/tests/storage/test-MetadataStorage.cpp +++ b/tests/storage/test-MetadataStorage.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -139,12 +138,15 @@ TEMPLATE_LIST_TEST_CASE( graph.add_task(parent_2); graph.add_dependency(parent_2.get_id(), child_task.get_id()); graph.add_dependency(parent_1.get_id(), child_task.get_id()); + graph.add_input_task(parent_1.get_id()); + graph.add_input_task(parent_2.get_id()); + graph.add_output_task(child_task.get_id()); // Get head tasks should succeed - absl::flat_hash_set heads = graph.get_head_tasks(); + std::vector heads = graph.get_input_tasks(); REQUIRE(2 == heads.size()); - REQUIRE(heads.contains(parent_1.get_id())); - REQUIRE(heads.contains(parent_2.get_id())); + REQUIRE(heads[0] == parent_1.get_id()); + REQUIRE(heads[1] == parent_2.get_id()); std::chrono::system_clock::time_point const job_creation_time = std::chrono::system_clock::now(); @@ -154,10 +156,12 @@ TEMPLATE_LIST_TEST_CASE( spider::core::Task const simple_task{"simple"}; spider::core::TaskGraph simple_graph; simple_graph.add_task(simple_task); + simple_graph.add_input_task(simple_task.get_id()); + simple_graph.add_output_task(simple_task.get_id()); - heads = simple_graph.get_head_tasks(); + heads = simple_graph.get_input_tasks(); REQUIRE(1 == heads.size()); - REQUIRE(heads.contains(simple_task.get_id())); + REQUIRE(heads[0] == simple_task.get_id()); // Submit job should success REQUIRE(storage->add_job(job_id, client_id, graph).success()); @@ -252,6 +256,9 @@ TEMPLATE_LIST_TEST_CASE("Task finish", "[storage]", spider::test::MetadataStorag graph.add_task(parent_2); graph.add_dependency(parent_2.get_id(), child_task.get_id()); graph.add_dependency(parent_1.get_id(), child_task.get_id()); + graph.add_input_task(parent_1.get_id()); + graph.add_input_task(parent_2.get_id()); + graph.add_output_task(child_task.get_id()); // Submit job should success REQUIRE(storage->add_job(job_id, gen(), graph).success()); @@ -313,6 +320,9 @@ TEMPLATE_LIST_TEST_CASE("Job reset", "[storage]", spider::test::MetadataStorageT graph.add_task(parent_2); graph.add_dependency(parent_2.get_id(), child_task.get_id()); graph.add_dependency(parent_1.get_id(), child_task.get_id()); + graph.add_input_task(parent_1.get_id()); + graph.add_input_task(parent_2.get_id()); + graph.add_output_task(child_task.get_id()); // Submit job should success REQUIRE(storage->add_job(job_id, gen(), graph).success()); diff --git a/tests/worker/test-FunctionManager.cpp b/tests/worker/test-FunctionManager.cpp index 3186ff833..af336384a 100644 --- a/tests/worker/test-FunctionManager.cpp +++ b/tests/worker/test-FunctionManager.cpp @@ -19,23 +19,39 @@ #include "../storage/StorageTestHelper.hpp" namespace { -auto int_test(spider::TaskContext const& /*context*/, int const x, int const y) -> int { +auto int_test(spider::TaskContext& /*context*/, int const x, int const y) -> int { return x + y; } -auto tuple_ret_test(spider::TaskContext const& /*context*/, std::string const& str, int const x) +auto tuple_ret_test(spider::TaskContext& /*context*/, std::string const& str, int const x) -> std::tuple { return std::make_tuple(str, x); } -auto data_test(spider::TaskContext const& /*context*/, spider::Data& data) -> int { +auto data_test(spider::TaskContext& /*context*/, spider::Data& data) -> int { return data.get(); } +auto not_registered(spider::TaskContext& /*context*/) -> int { + return 0; +} + SPIDER_WORKER_REGISTER_TASK(int_test); SPIDER_WORKER_REGISTER_TASK(tuple_ret_test); SPIDER_WORKER_REGISTER_TASK(data_test); +TEST_CASE("Register and get function name", "[core]") { + spider::core::FunctionManager const& manager = spider::core::FunctionManager::get_instance(); + + // Get the function name of non-registered function should return std::nullopt + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + REQUIRE(!manager.get_function_name(reinterpret_cast(not_registered)).has_value()); + // Get the function name of registered function should return the name + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) + REQUIRE("int_test" == manager.get_function_name(reinterpret_cast(int_test)).value_or("") + ); +} + TEMPLATE_LIST_TEST_CASE( "Register and run function with POD inputs", "[core][storage]", diff --git a/tools/scripts/lib_install/boost.sh b/tools/scripts/lib_install/boost.sh index 4c8fd3113..55b2eb0bd 100755 --- a/tools/scripts/lib_install/boost.sh +++ b/tools/scripts/lib_install/boost.sh @@ -30,10 +30,11 @@ mkdir -p $temp_dir cd $temp_dir # Download source -tar_filename=boost_${version_with_underscores}.tar.gz -curl -fsSL https://boostorg.jfrog.io/artifactory/main/release/${version}/source/${tar_filename} -o ${tar_filename} -tar xzf ${tar_filename} -cd boost_${version_with_underscores} +boost_source_dir=boost_${version_with_underscores} +git clone https://github.com/boostorg/boost.git ${boost_source_dir} +cd ${boost_source_dir} +git checkout boost-${version} +git submodule update --init --recursive # Build ./bootstrap.sh --with-libraries=filesystem,iostreams,process,program_options,regex,system