From e7cb9c69d8de1a920b80d45afee2eb100c126f2e Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Sun, 27 Oct 2024 20:32:25 +0800 Subject: [PATCH 1/6] storage: Add interface for storage. Added interface for metadata and storage interface. Added simple Task and Data class for interface to work. Add boost header for uuid and abseil for flat_hash_map. --- .gitignore | 1 + .gitmodules | 3 + CMakeLists.txt | 19 ++++- src/spider/CMakeLists.txt | 15 +++- src/spider/core/Data.hpp | 26 +++++++ src/spider/core/Error.hpp | 20 +++++ src/spider/core/Task.hpp | 102 +++++++++++++++++++++++++ src/spider/core/TaskGraph.hpp | 72 +++++++++++++++++ src/spider/storage/DataStorage.hpp | 28 +++++++ src/spider/storage/MetadataStorage.hpp | 39 +++++++++- src/spider/storage/MysqlStorage.cpp | 62 +++++++++++++++ src/spider/storage/MysqlStorage.hpp | 45 +++++++++++ submodules/abseil-cpp | 1 + 13 files changed, 426 insertions(+), 7 deletions(-) create mode 100644 src/spider/core/Data.hpp create mode 100644 src/spider/core/Error.hpp create mode 100644 src/spider/core/Task.hpp create mode 100644 src/spider/core/TaskGraph.hpp create mode 100644 src/spider/storage/DataStorage.hpp create mode 100644 src/spider/storage/MysqlStorage.cpp create mode 100644 src/spider/storage/MysqlStorage.hpp create mode 160000 submodules/abseil-cpp diff --git a/.gitignore b/.gitignore index 094bd633c..2bdf88c9b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build # IDE-related directories and files .idea +cmake-build-debug diff --git a/.gitmodules b/.gitmodules index 09604b153..f8ea5ed07 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "submodules/Catch2"] path = submodules/Catch2 url = https://github.com/catchorg/Catch2.git +[submodule "submodules/abseil-cpp"] + path = submodules/abseil-cpp + url = https://github.com/abseil/abseil-cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index d6ab6a8b9..ed5b1d1f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,17 @@ else() endif() message(STATUS "Building using ${SPIDER_LIBS_STRING} libraries") +# Find and setup Boost Library +if(SPIDER_USE_STATIC_LIBS) + set(Boost_USE_STATIC_LIBS ON) +endif() +find_package(Boost 1.74 REQUIRED) +if(Boost_FOUND) + message(STATUS "Found Boost ${Boost_VERSION}") +else() + message(FATAL_ERROR "Could not find ${SPIDER_LIBS_STRING} libraries for Boost") +endif() + # Find and setup fmt find_package(fmt 8.0.1 REQUIRED) if(fmt_FOUND) @@ -106,7 +117,7 @@ else() endif() # Find and setup MariaDBClient library -if(CLP_USE_STATIC_LIBS) +if(SPIDER_USE_STATIC_LIBS) # NOTE: We can't statically link to MariaDBClient since it's GPL message(AUTHOR_WARNING "MariaDBClient cannot be statically linked due to its license.") endif() @@ -114,9 +125,13 @@ find_package(MariaDBClient 3.1.0 REQUIRED) if(MariaDBClient_FOUND) message(STATUS "Found MariaDBClient ${MariaDBClient_VERSION}") else() - message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for MariaDBClient") + message(FATAL_ERROR "Could not find ${SPIDER_LIBS_STRING} libraries for MariaDBClient") endif() +# Add abseil-cpp +set(ABSL_PROPAGATE_CXX_STD ON) +add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL) + find_package(Threads REQUIRED) add_subdirectory(src/spider) diff --git a/src/spider/CMakeLists.txt b/src/spider/CMakeLists.txt index 5e1d2363d..cb4f4800a 100644 --- a/src/spider/CMakeLists.txt +++ b/src/spider/CMakeLists.txt @@ -1,6 +1,15 @@ # set variable as CACHE INTERNAL to access it from other scope -set(SPIDER_CORE_SOURCES storage/MetadataStorage.hpp CACHE INTERNAL "spider core source files") - +set(SPIDER_CORE_SOURCES + core/Error.hpp + core/Data.hpp + core/Task.hpp + core/TaskGraph.hpp + storage/MetadataStorage.hpp + storage/DataStorage.hpp + storage/MysqlStorage.cpp + storage/MysqlStorage.hpp + CACHE INTERNAL "spider core source files" +) set(SPIDER_WORKER_SOURCES worker/worker.cpp CACHE INTERNAL "spider worker source files") add_executable(spider_worker) @@ -11,4 +20,6 @@ target_sources( ${SPIDER_CORE_SOURCES} ${SPIDER_WORKER_SOURCES} ) +target_sources(spider_worker PRIVATE ${SPIDER_CORE_SOURCES} ${SPIDER_WORKER_SOURCES}) +target_link_libraries(spider_worker Boost::boost absl::flat_hash_map) add_executable(spider::worker ALIAS spider_worker) diff --git a/src/spider/core/Data.hpp b/src/spider/core/Data.hpp new file mode 100644 index 000000000..3ef72fa62 --- /dev/null +++ b/src/spider/core/Data.hpp @@ -0,0 +1,26 @@ +#ifndef SPIDER_CORE_DATA_HPP +#define SPIDER_CORE_DATA_HPP + +#include + +class Data { +private: + boost::uuids::uuid m_id; + std::optional m_key; + std::string m_value; +public: + explicit Data(std::string value): m_value(std::move(value)) { + boost::uuids::random_generator gen; + m_id = gen(); + } + Data(std::string key, std::string value): m_key(std::move(key)), m_value(std::move(value)) { + boost::uuids::random_generator gen; + m_id = gen(); + } + + boost::uuids::uuid get_id() { return m_id; } + std::optional get_key() { return m_key; } + std::string get_value() { return m_value; } +}; + +#endif // SPIDER_CORE_DATA_HPP diff --git a/src/spider/core/Error.hpp b/src/spider/core/Error.hpp new file mode 100644 index 000000000..31f1e1cc9 --- /dev/null +++ b/src/spider/core/Error.hpp @@ -0,0 +1,20 @@ +#ifndef SPIDER_CORE_ERROR_HPP +#define SPIDER_CORE_ERROR_HPP + +namespace spider::core { +enum class StorageErrType { + kConnectionErr = 0, + kDbNotFound, + kKeyNotFoundErr, + kDuplicateKeyErr, + kConstraintViolationErr, +}; + +struct StorageErr { + StorageErrType type; + std::string description; +}; + +} // namespace spider::core + +#endif // SPIDER_CORE_ERROR_HPP diff --git a/src/spider/core/Task.hpp b/src/spider/core/Task.hpp new file mode 100644 index 000000000..5c3f42deb --- /dev/null +++ b/src/spider/core/Task.hpp @@ -0,0 +1,102 @@ +#ifndef SPIDER_CORE_TASK_HPP +#define SPIDER_CORE_TASK_HPP + +#include +#include + +namespace spider::core { + +class TaskInput { +private: + std::optional> m_task_output; + std::optional m_value; + std::optional m_data_id; + std::string m_type; + +public: + TaskInput(boost::uuids::uuid output_task_id, uint8_t position, std::string type) + : m_task_output({output_task_id, position}), + m_type(type) {}; + TaskInput(std::string value, std::string type) : m_value(value), m_type(type) {}; + TaskInput(boost::uuids::uuid data_id, std::string type) : m_data_id(data_id), m_type(type) {}; + + std::optional> get_task_output() const { + return m_task_output; + } + std::optional get_value() const { return m_value; } + std::optional get_data_id() const { return m_data_id; } + std::string get_type() const { return m_type; } +}; + +class TaskOutput { +private: + std::optional m_value; + std::optional m_data_id; + std::string m_type; + +public: + TaskOutput(std::string value, std::string type) : m_value(value), m_type(type) {} + + TaskOutput(boost::uuids::uuid data_id, std::string type) : m_data_id(data_id), m_type(type) {} + + std::optional get_value() const { return m_value; } + std::optional get_data_id() const { return m_data_id; } + std::string get_type() const { return m_type; } +}; + +class TaskInstance {}; + +enum TaskState { + kPending = 0, + kReady, + kRunning, + kSucceed, + kFailed, + kCanceled, +}; + +enum TaskCreatorType { + kClient = 0, + kTask, +}; + +class Task { +private: + boost::uuids::uuid m_id; + std::string m_function_name; + TaskState m_state = kPending; + TaskCreatorType m_creator_type; + boost::uuids::uuid m_creator_id; + float m_timeout = 0; + std::vector m_inputs; + std::vector m_outputs; + +public: + Task(std::string function_name, TaskCreatorType creator_type, boost::uuids::uuid creator_id) + : m_function_name(std::move(function_name)), + m_creator_type(creator_type), + m_creator_id(creator_id) { + boost::uuids::random_generator gen; + m_id = gen(); + } + + void add_input(TaskInput const& input) { m_inputs.emplace_back(input); } + + void add_output(TaskOutput const& output) { m_outputs.emplace_back(output); } + + boost::uuids::uuid get_id() const { return m_id; } + std::string get_function_name() const { return m_function_name; } + TaskState get_state() const { return m_state; } + TaskCreatorType get_creator_type() const { return m_creator_type; } + boost::uuids::uuid get_creator_id() const { return m_creator_id; } + float get_timeout() const { return m_timeout; } + + uint64_t get_num_inputs() const { return m_inputs.size(); } + uint64_t get_num_outputs() const { return m_outputs.size(); } + TaskInput get_input(uint64_t index) const { return m_inputs[index]; } + TaskInput get_output(uint64_t index) const { return m_outputs[index]; } +}; + +} // namespace spider::core + +#endif // SPIDER_CORE_TASK_HPP diff --git a/src/spider/core/TaskGraph.hpp b/src/spider/core/TaskGraph.hpp new file mode 100644 index 000000000..8ecc41fca --- /dev/null +++ b/src/spider/core/TaskGraph.hpp @@ -0,0 +1,72 @@ +#ifndef SPIDER_CORE_TASKGRAPH_HPP +#define SPIDER_CORE_TASKGRAPH_HPP + +#include +#include + +namespace spider::core { + + + +class TaskGraph { +private: + absl::flat_hash_map m_tasks; + std::vector> m_dependencies; +public: + bool add_child_task(const Task& task, const std::vector& parents) { + boost::uuids::uuid task_id = task.get_id(); + for (boost::uuids::uuid const parent_id: parents) { + if (!m_tasks.contains(parent_id)) { + return false; + } + } + if (m_tasks.contains(task.get_id())) { + return false; + } + + m_tasks.emplace(task_id, task); + for (boost::uuids::uuid const parent_id: parents) { + m_dependencies.emplace_back(parent_id, task_id); + } + return true; + } + + std::optional get_task(boost::uuids::uuid id) const { + if (m_tasks.contains(id)) { + return m_tasks.at(id); + } else { + return std::nullopt; + } + } + + std::vector get_child_tasks(boost::uuids::uuid id) const { + std::vector children; + for (std::pair const dep: m_dependencies) { + if (dep.first == id) { + children.emplace_back(dep.second); + } + } + return children; + } + + std::vector get_parent_tasks(boost::uuids::uuid id) const { + std::vector parents; + for (std::pair const dep: m_dependencies) { + if (dep.second == id) { + parents.emplace_back(dep.first); + } + } + return parents; + } + + const absl::flat_hash_map& get_tasks() const { + return m_tasks; + } + + const std::vector>& get_dependencies() const { + return m_dependencies; + } +}; +} // namespace spider::core + +#endif // SPIDER_CORE_TASKGRAPH_HPP diff --git a/src/spider/storage/DataStorage.hpp b/src/spider/storage/DataStorage.hpp new file mode 100644 index 000000000..5b77116e1 --- /dev/null +++ b/src/spider/storage/DataStorage.hpp @@ -0,0 +1,28 @@ +#ifndef SPIDER_STORAGE_DATASTORAGE_HPP +#define SPIDER_STORAGE_DATASTORAGE_HPP + +#include +#include + +#include "../core/Data.hpp" +#include "../core/Error.hpp" + +namespace spider::core { +class DataStorage { +public: + virtual StorageErr connect(std::string url, boost::uuids::uuid id) = 0; + virtual void close() = 0; + virtual StorageErr initialize() = 0; + + virtual StorageErr add_data(Data const& data) = 0; + virtual StorageErr get_data(boost::uuids::uuid id, Data& data) = 0; + virtual StorageErr add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) = 0; + virtual StorageErr remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) = 0; + virtual StorageErr add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) + = 0; + virtual StorageErr remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) + = 0; +}; +} // namespace spider::core + +#endif // SPIDER_STORAGE_DATASTORAGE_HPP diff --git a/src/spider/storage/MetadataStorage.hpp b/src/spider/storage/MetadataStorage.hpp index c71dc170d..e027e1596 100644 --- a/src/spider/storage/MetadataStorage.hpp +++ b/src/spider/storage/MetadataStorage.hpp @@ -1,4 +1,37 @@ -#ifndef SPIDER_METADATASTORAGE_HPP -#define SPIDER_METADATASTORAGE_HPP +#ifndef SPIDER_STORAGE_METADATASTORAGE_HPP +#define SPIDER_STORAGE_METADATASTORAGE_HPP -#endif // SPIDER_METADATASTORAGE_HPP +#include +#include + +#include "../core/Error.hpp" +#include "../core/Task.hpp" +#include "../core/TaskGraph.hpp" + +namespace spider::core { +class MetadataStorage { +public: + virtual StorageErr connect(std::string url, boost::uuids::uuid id) = 0; + virtual void close() = 0; + virtual StorageErr initialize() = 0; + + virtual StorageErr add_task_graph(TaskGraph const& task_graph) = 0; + virtual StorageErr get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) = 0; + virtual StorageErr get_task_graphs(std::vector& task_graphs) = 0; + virtual StorageErr remove_task_graph(boost::uuids::uuid id) = 0; + virtual StorageErr add_child(boost::uuids::uuid parent_id, Task const& child) = 0; + virtual StorageErr get_task(boost::uuids::uuid id, Task& task) = 0; + virtual StorageErr get_ready_tasks(std::vector& tasks) = 0; + virtual StorageErr set_task_state(boost::uuids::uuid id, TaskState state) = 0; + virtual StorageErr add_task_instance(TaskInstance const& instance) = 0; + virtual StorageErr task_finish(TaskInstance const& instance) = 0; + virtual StorageErr get_task_timeout(std::vector& tasks) = 0; + virtual StorageErr get_child_task(boost::uuids::uuid id, Task& child) = 0; + virtual StorageErr get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) = 0; + + virtual StorageErr update_heartbeat(boost::uuids::uuid id) = 0; + virtual StorageErr hearbeat_timeout(std::vector& ids) = 0; +}; + +} // namespace spider::core +#endif // SPIDER_STORAGE_METADATASTORAGE_HPP diff --git a/src/spider/storage/MysqlStorage.cpp b/src/spider/storage/MysqlStorage.cpp new file mode 100644 index 000000000..40a7ea0b8 --- /dev/null +++ b/src/spider/storage/MysqlStorage.cpp @@ -0,0 +1,62 @@ +#include "MysqlStorage.hpp" + +namespace spider::core { +StorageErr MySqlMetadataStorage::connect(std::string url, boost::uuids::uuid id) {} + +void MySqlMetadataStorage::close() {} + +StorageErr MySqlMetadataStorage::initialize() {} + +StorageErr MySqlMetadataStorage::add_task_graph(TaskGraph const& task_graph) {} + +StorageErr MySqlMetadataStorage::get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) {} + +StorageErr MySqlMetadataStorage::get_task_graphs(std::vector& task_graphs) {} + +StorageErr MySqlMetadataStorage::remove_task_graph(boost::uuids::uuid id) {} + +StorageErr MySqlMetadataStorage::add_child(boost::uuids::uuid parent_id, Task const& child) {} + +StorageErr MySqlMetadataStorage::get_task(boost::uuids::uuid id, Task& task) {} + +StorageErr MySqlMetadataStorage::get_ready_tasks(std::vector& tasks) {} + +StorageErr MySqlMetadataStorage::set_task_state(boost::uuids::uuid id, TaskState state) {} + +StorageErr MySqlMetadataStorage::add_task_instance(TaskInstance const& instance) {} + +StorageErr MySqlMetadataStorage::task_finish(TaskInstance const& instance) {} + +StorageErr MySqlMetadataStorage::get_task_timeout(std::vector& tasks) {} + +StorageErr get_child_task(boost::uuids::uuid id, Task& child) {} + +StorageErr get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) {} + +StorageErr MySqlMetadataStorage::update_heartbeat(boost::uuids::uuid id) {} + +StorageErr MySqlMetadataStorage::hearbeat_timeout(std::vector& ids) {} + +StorageErr MysqlDataStorage::connect(std::string url, boost::uuids::uuid id) {} + +void MysqlDataStorage::close() {} + +StorageErr MysqlDataStorage::initialize() {} + +StorageErr MysqlDataStorage::add_data(Data const& data) {} + +StorageErr MysqlDataStorage::get_data(boost::uuids::uuid id, Data& data) {} + +StorageErr MysqlDataStorage::add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) { +} + +StorageErr +MysqlDataStorage::remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) {} + +StorageErr +MysqlDataStorage::add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) {} + +StorageErr +MysqlDataStorage::remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) {} + +} // namespace spider::core diff --git a/src/spider/storage/MysqlStorage.hpp b/src/spider/storage/MysqlStorage.hpp new file mode 100644 index 000000000..248cf22d0 --- /dev/null +++ b/src/spider/storage/MysqlStorage.hpp @@ -0,0 +1,45 @@ +#ifndef SPIDER_STORAGE_MYSQLSTORAGE_HPP +#define SPIDER_STORAGE_MYSQLSTORAGE_HPP + +#include "DataStorage.hpp" +#include "MetadataStorage.hpp" + +namespace spider::core { +class MySqlMetadataStorage : public MetadataStorage { +public: + StorageErr connect(std::string url, boost::uuids::uuid id) override; + void close() override; + StorageErr initialize() override; + StorageErr add_task_graph(TaskGraph const& task_graph) override; + StorageErr get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) override; + StorageErr get_task_graphs(std::vector& task_graphs) override; + StorageErr remove_task_graph(boost::uuids::uuid id) override; + StorageErr add_child(boost::uuids::uuid parent_id, Task const& child) override; + StorageErr get_task(boost::uuids::uuid id, Task& task) override; + StorageErr get_ready_tasks(std::vector& tasks) override; + StorageErr set_task_state(boost::uuids::uuid id, TaskState state) override; + StorageErr add_task_instance(TaskInstance const& instance) override; + StorageErr task_finish(TaskInstance const& instance) override; + StorageErr get_task_timeout(std::vector& tasks) override; + StorageErr get_child_task(boost::uuids::uuid id, Task& child) override; + StorageErr get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) override; + StorageErr update_heartbeat(boost::uuids::uuid id) override; + StorageErr hearbeat_timeout(std::vector& ids) override; +}; + +class MysqlDataStorage : public DataStorage { +public: + StorageErr connect(std::string url, boost::uuids::uuid id) override; + void close() override; + StorageErr initialize() override; + StorageErr add_data(Data const& data) override; + StorageErr get_data(boost::uuids::uuid id, Data& data) override; + StorageErr add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) override; + StorageErr remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) override; + StorageErr add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) override; + StorageErr + remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) override; +}; +} // namespace spider::core + +#endif // SPIDER_STORAGE_MYSQLSTORAGE_HPP diff --git a/submodules/abseil-cpp b/submodules/abseil-cpp new file mode 160000 index 000000000..07fff76ed --- /dev/null +++ b/submodules/abseil-cpp @@ -0,0 +1 @@ +Subproject commit 07fff76ed619672bb7bf00939231ba45aef2b043 From 8264e594b3230030254b3d2622d974f1ee6c00b4 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Tue, 29 Oct 2024 14:55:50 +0800 Subject: [PATCH 2/6] build: Add boost library install script --- tools/scripts/lib_install/boost.sh | 48 ++++++++++++++++++++++++++++++ tools/scripts/linux/install-lib.sh | 1 + 2 files changed, 49 insertions(+) create mode 100755 tools/scripts/lib_install/boost.sh diff --git a/tools/scripts/lib_install/boost.sh b/tools/scripts/lib_install/boost.sh new file mode 100755 index 000000000..e35005340 --- /dev/null +++ b/tools/scripts/lib_install/boost.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Exit on error +set -e + +cUsage="Usage: ${BASH_SOURCE[0]} " +if [ "$#" -lt 1 ] ; then + echo $cUsage + exit +fi +version=$1 +version_with_underscores=${version//./_} + +echo "Checking for elevated privileges..." +if [ ${EUID:-$(id -u)} -ne 0 ] ; then + sudo echo "Script can elevate privileges." +fi + +# Get number of cpu cores +num_cpus=$(grep -c ^processor /proc/cpuinfo) + +package_name=boost + +# Create temp dir for installation +temp_dir=/tmp/${package_name}-installation +mkdir -p $temp_dir + +cd $temp_dir + +# Download source +tar_filename=boost_${version_with_underscores}.tar.gz +curl -fsSL https://boostorg.jfrog.io/artifactory/main/release/${version}/source/${tar_filename} -o ${tar_filename} +tar xzf ${tar_filename} +cd boost_${version_with_underscores} + +# Build +./bootstrap.sh --with-libraries=filesystem,iostreams,program_options,regex,system +./b2 -j${num_cpus} + +# Install +if [ ${EUID:-$(id -u)} -ne 0 ] ; then + sudo ./b2 install +else + ./b2 install +fi + +# Clean up +rm -rf $temp_dir \ No newline at end of file diff --git a/tools/scripts/linux/install-lib.sh b/tools/scripts/linux/install-lib.sh index eca3c1d35..b49a20bbe 100755 --- a/tools/scripts/linux/install-lib.sh +++ b/tools/scripts/linux/install-lib.sh @@ -12,3 +12,4 @@ lib_install_scripts_dir=$script_dir/.. "$lib_install_scripts_dir"/lib_install/fmtlib.sh 11.0.2 "$lib_install_scripts_dir"/lib_install/spdlog.sh 1.14.1 "$lib_install_scripts_dir"/lib_install/mariadb-connector-c.sh 3.4.1 +"$lib_install_scripts_dir"/lib_install/boost.sh 1.86.0 From 1385311f4d15b0a71aa2fc531cfc4f28fbc823a9 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Tue, 29 Oct 2024 17:12:49 +0800 Subject: [PATCH 3/6] fix: Satisfy clang-tidy --- src/spider/CMakeLists.txt | 16 +++- src/spider/core/Data.hpp | 18 +++- src/spider/core/Error.hpp | 21 ++-- src/spider/core/Task.hpp | 103 +++++++++++++------- src/spider/core/TaskGraph.hpp | 37 ++++--- src/spider/storage/DataStorage.hpp | 28 ++++-- src/spider/storage/MetadataStorage.hpp | 42 ++++---- src/spider/storage/MysqlStorage.cpp | 127 +++++++++++++++++++------ src/spider/storage/MysqlStorage.hpp | 75 +++++++++------ 9 files changed, 318 insertions(+), 149 deletions(-) diff --git a/src/spider/CMakeLists.txt b/src/spider/CMakeLists.txt index cb4f4800a..70463e264 100644 --- a/src/spider/CMakeLists.txt +++ b/src/spider/CMakeLists.txt @@ -8,7 +8,8 @@ set(SPIDER_CORE_SOURCES storage/DataStorage.hpp storage/MysqlStorage.cpp storage/MysqlStorage.hpp - CACHE INTERNAL "spider core source files" + CACHE INTERNAL + "spider core source files" ) set(SPIDER_WORKER_SOURCES worker/worker.cpp CACHE INTERNAL "spider worker source files") @@ -20,6 +21,15 @@ target_sources( ${SPIDER_CORE_SOURCES} ${SPIDER_WORKER_SOURCES} ) -target_sources(spider_worker PRIVATE ${SPIDER_CORE_SOURCES} ${SPIDER_WORKER_SOURCES}) -target_link_libraries(spider_worker Boost::boost absl::flat_hash_map) +target_sources( + spider_worker + PRIVATE + ${SPIDER_CORE_SOURCES} + ${SPIDER_WORKER_SOURCES} +) +target_link_libraries( + spider_worker + Boost::boost + absl::flat_hash_map +) add_executable(spider::worker ALIAS spider_worker) diff --git a/src/spider/core/Data.hpp b/src/spider/core/Data.hpp index 3ef72fa62..04abe360b 100644 --- a/src/spider/core/Data.hpp +++ b/src/spider/core/Data.hpp @@ -2,25 +2,33 @@ #define SPIDER_CORE_DATA_HPP #include +#include +#include +#include +#include class Data { private: boost::uuids::uuid m_id; std::optional m_key; std::string m_value; + public: - explicit Data(std::string value): m_value(std::move(value)) { + explicit Data(std::string value) : m_value(std::move(value)) { boost::uuids::random_generator gen; m_id = gen(); } - Data(std::string key, std::string value): m_key(std::move(key)), m_value(std::move(value)) { + + Data(std::string key, std::string value) : m_key(std::move(key)), m_value(std::move(value)) { boost::uuids::random_generator gen; m_id = gen(); } - boost::uuids::uuid get_id() { return m_id; } - std::optional get_key() { return m_key; } - std::string get_value() { return m_value; } + auto get_id() -> boost::uuids::uuid { return m_id; } + + auto get_key() -> std::optional { return m_key; } + + auto get_value() -> std::string { return m_value; } }; #endif // SPIDER_CORE_DATA_HPP diff --git a/src/spider/core/Error.hpp b/src/spider/core/Error.hpp index 31f1e1cc9..e9d78cb18 100644 --- a/src/spider/core/Error.hpp +++ b/src/spider/core/Error.hpp @@ -1,18 +1,27 @@ #ifndef SPIDER_CORE_ERROR_HPP #define SPIDER_CORE_ERROR_HPP +#include +#include +#include + namespace spider::core { -enum class StorageErrType { - kConnectionErr = 0, - kDbNotFound, - kKeyNotFoundErr, - kDuplicateKeyErr, - kConstraintViolationErr, +enum class StorageErrType : std::uint8_t { + ConnectionErr, + DbNotFound, + KeyNotFoundErr, + DuplicateKeyErr, + ConstraintViolationErr, + Success }; struct StorageErr { StorageErrType type; std::string description; + + StorageErr(StorageErrType type, std::string description) + : type(type), + description(std::move(description)) {} }; } // namespace spider::core diff --git a/src/spider/core/Task.hpp b/src/spider/core/Task.hpp index 5c3f42deb..c0cd0fd3c 100644 --- a/src/spider/core/Task.hpp +++ b/src/spider/core/Task.hpp @@ -2,30 +2,45 @@ #define SPIDER_CORE_TASK_HPP #include +#include +#include +#include +#include #include +#include +#include +#include namespace spider::core { class TaskInput { private: - std::optional> m_task_output; + std::optional> m_task_output; std::optional m_value; std::optional m_data_id; std::string m_type; public: - TaskInput(boost::uuids::uuid output_task_id, uint8_t position, std::string type) + TaskInput(boost::uuids::uuid output_task_id, std::uint8_t position, std::string type) : m_task_output({output_task_id, position}), - m_type(type) {}; - TaskInput(std::string value, std::string type) : m_value(value), m_type(type) {}; - TaskInput(boost::uuids::uuid data_id, std::string type) : m_data_id(data_id), m_type(type) {}; - - std::optional> get_task_output() const { + m_type(std::move(type)) {}; + TaskInput(std::string value, std::string type) : m_value(value), m_type(std::move(type)) {}; + TaskInput(boost::uuids::uuid data_id, std::string type) + : m_data_id(data_id), + m_type(std::move(type)) {}; + + [[nodiscard]] auto get_task_output( + ) const -> std::optional> { return m_task_output; } - std::optional get_value() const { return m_value; } - std::optional get_data_id() const { return m_data_id; } - std::string get_type() const { return m_type; } + + [[nodiscard]] auto get_value() const -> std::optional { return m_value; } + + [[nodiscard]] auto get_data_id() const -> std::optional { + return m_data_id; + } + + [[nodiscard]] auto get_type() const -> std::string { return m_type; } }; class TaskOutput { @@ -35,36 +50,42 @@ class TaskOutput { std::string m_type; public: - TaskOutput(std::string value, std::string type) : m_value(value), m_type(type) {} + TaskOutput(std::string value, std::string type) : m_value(value), m_type(std::move(type)) {} - TaskOutput(boost::uuids::uuid data_id, std::string type) : m_data_id(data_id), m_type(type) {} + TaskOutput(boost::uuids::uuid data_id, std::string type) + : m_data_id(data_id), + m_type(std::move(type)) {} - std::optional get_value() const { return m_value; } - std::optional get_data_id() const { return m_data_id; } - std::string get_type() const { return m_type; } + [[nodiscard]] auto get_value() const -> std::optional { return m_value; } + + [[nodiscard]] auto get_data_id() const -> std::optional { + return m_data_id; + } + + [[nodiscard]] auto get_type() const -> std::string { return m_type; } }; class TaskInstance {}; -enum TaskState { - kPending = 0, - kReady, - kRunning, - kSucceed, - kFailed, - kCanceled, +enum class TaskState : std::uint8_t { + Pending, + Ready, + Running, + Succeed, + Failed, + Canceled, }; -enum TaskCreatorType { - kClient = 0, - kTask, +enum class TaskCreatorType : std::uint8_t { + Client = 0, + Task, }; class Task { private: boost::uuids::uuid m_id; std::string m_function_name; - TaskState m_state = kPending; + TaskState m_state = TaskState::Pending; TaskCreatorType m_creator_type; boost::uuids::uuid m_creator_id; float m_timeout = 0; @@ -84,17 +105,25 @@ class Task { void add_output(TaskOutput const& output) { m_outputs.emplace_back(output); } - boost::uuids::uuid get_id() const { return m_id; } - std::string get_function_name() const { return m_function_name; } - TaskState get_state() const { return m_state; } - TaskCreatorType get_creator_type() const { return m_creator_type; } - boost::uuids::uuid get_creator_id() const { return m_creator_id; } - float get_timeout() const { return m_timeout; } - - uint64_t get_num_inputs() const { return m_inputs.size(); } - uint64_t get_num_outputs() const { return m_outputs.size(); } - TaskInput get_input(uint64_t index) const { return m_inputs[index]; } - TaskInput get_output(uint64_t index) const { return m_outputs[index]; } + [[nodiscard]] auto get_id() const -> boost::uuids::uuid { return m_id; } + + [[nodiscard]] auto get_function_name() const -> std::string { return m_function_name; } + + [[nodiscard]] auto get_state() const -> TaskState { return m_state; } + + [[nodiscard]] auto get_creator_type() const -> TaskCreatorType { return m_creator_type; } + + [[nodiscard]] auto get_creator_id() const -> boost::uuids::uuid { return m_creator_id; } + + [[nodiscard]] auto get_timeout() const -> float { return m_timeout; } + + [[nodiscard]] auto get_num_inputs() const -> size_t { return m_inputs.size(); } + + [[nodiscard]] auto get_num_outputs() const -> size_t { return m_outputs.size(); } + + [[nodiscard]] auto get_input(uint64_t index) const -> TaskInput { return m_inputs[index]; } + + [[nodiscard]] auto get_output(uint64_t index) const -> TaskOutput { return m_outputs[index]; } }; } // namespace spider::core diff --git a/src/spider/core/TaskGraph.hpp b/src/spider/core/TaskGraph.hpp index 8ecc41fca..019b40edf 100644 --- a/src/spider/core/TaskGraph.hpp +++ b/src/spider/core/TaskGraph.hpp @@ -2,20 +2,25 @@ #define SPIDER_CORE_TASKGRAPH_HPP #include -#include -namespace spider::core { +#include +#include +#include +#include +#include "Task.hpp" +namespace spider::core { class TaskGraph { private: absl::flat_hash_map m_tasks; std::vector> m_dependencies; + public: - bool add_child_task(const Task& task, const std::vector& parents) { - boost::uuids::uuid task_id = task.get_id(); - for (boost::uuids::uuid const parent_id: parents) { + auto add_child_task(Task const& task, std::vector const& parents) -> bool { + boost::uuids::uuid const task_id = task.get_id(); + for (boost::uuids::uuid const parent_id : parents) { if (!m_tasks.contains(parent_id)) { return false; } @@ -25,23 +30,23 @@ class TaskGraph { } m_tasks.emplace(task_id, task); - for (boost::uuids::uuid const parent_id: parents) { + for (boost::uuids::uuid const parent_id : parents) { m_dependencies.emplace_back(parent_id, task_id); } return true; } - std::optional get_task(boost::uuids::uuid id) const { + [[nodiscard]] auto get_task(boost::uuids::uuid id) const -> std::optional { if (m_tasks.contains(id)) { return m_tasks.at(id); - } else { - return std::nullopt; } + return std::nullopt; } - std::vector get_child_tasks(boost::uuids::uuid id) const { + [[nodiscard]] auto get_child_tasks(boost::uuids::uuid id + ) const -> std::vector { std::vector children; - for (std::pair const dep: m_dependencies) { + for (std::pair const dep : m_dependencies) { if (dep.first == id) { children.emplace_back(dep.second); } @@ -49,9 +54,10 @@ class TaskGraph { return children; } - std::vector get_parent_tasks(boost::uuids::uuid id) const { + [[nodiscard]] auto get_parent_tasks(boost::uuids::uuid id + ) const -> std::vector { std::vector parents; - for (std::pair const dep: m_dependencies) { + for (std::pair const dep : m_dependencies) { if (dep.second == id) { parents.emplace_back(dep.first); } @@ -59,11 +65,12 @@ class TaskGraph { return parents; } - const absl::flat_hash_map& get_tasks() const { + [[nodiscard]] auto get_tasks() const -> absl::flat_hash_map const& { return m_tasks; } - const std::vector>& get_dependencies() const { + [[nodiscard]] auto get_dependencies( + ) const -> std::vector> const& { return m_dependencies; } }; diff --git a/src/spider/storage/DataStorage.hpp b/src/spider/storage/DataStorage.hpp index 5b77116e1..e03357109 100644 --- a/src/spider/storage/DataStorage.hpp +++ b/src/spider/storage/DataStorage.hpp @@ -10,18 +10,26 @@ namespace spider::core { class DataStorage { public: - virtual StorageErr connect(std::string url, boost::uuids::uuid id) = 0; + DataStorage(DataStorage const&) = default; + DataStorage(DataStorage&&) = default; + auto operator=(DataStorage const&) -> DataStorage& = default; + auto operator=(DataStorage&&) -> DataStorage& = default; + virtual ~DataStorage() = default; + + virtual auto connect(std::string url, boost::uuids::uuid id) -> StorageErr = 0; virtual void close() = 0; - virtual StorageErr initialize() = 0; + virtual auto initialize() -> StorageErr = 0; - virtual StorageErr add_data(Data const& data) = 0; - virtual StorageErr get_data(boost::uuids::uuid id, Data& data) = 0; - virtual StorageErr add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) = 0; - virtual StorageErr remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) = 0; - virtual StorageErr add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) - = 0; - virtual StorageErr remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) - = 0; + virtual auto add_data(Data const& data) -> StorageErr = 0; + virtual auto get_data(boost::uuids::uuid id, Data& data) -> StorageErr = 0; + virtual auto add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr + = 0; + virtual auto + remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr = 0; + virtual auto + add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) -> StorageErr = 0; + virtual auto + remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) -> StorageErr = 0; }; } // namespace spider::core diff --git a/src/spider/storage/MetadataStorage.hpp b/src/spider/storage/MetadataStorage.hpp index e027e1596..5c0b30a4c 100644 --- a/src/spider/storage/MetadataStorage.hpp +++ b/src/spider/storage/MetadataStorage.hpp @@ -3,6 +3,7 @@ #include #include +#include #include "../core/Error.hpp" #include "../core/Task.hpp" @@ -11,26 +12,33 @@ namespace spider::core { class MetadataStorage { public: - virtual StorageErr connect(std::string url, boost::uuids::uuid id) = 0; + MetadataStorage(MetadataStorage const&) = default; + MetadataStorage(MetadataStorage&&) = default; + auto operator=(MetadataStorage const&) -> MetadataStorage& = default; + auto operator=(MetadataStorage&&) -> MetadataStorage& = default; + virtual ~MetadataStorage() = default; + + virtual auto connect(std::string url, boost::uuids::uuid id) -> StorageErr = 0; virtual void close() = 0; - virtual StorageErr initialize() = 0; + virtual auto initialize() -> StorageErr = 0; - virtual StorageErr add_task_graph(TaskGraph const& task_graph) = 0; - virtual StorageErr get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) = 0; - virtual StorageErr get_task_graphs(std::vector& task_graphs) = 0; - virtual StorageErr remove_task_graph(boost::uuids::uuid id) = 0; - virtual StorageErr add_child(boost::uuids::uuid parent_id, Task const& child) = 0; - virtual StorageErr get_task(boost::uuids::uuid id, Task& task) = 0; - virtual StorageErr get_ready_tasks(std::vector& tasks) = 0; - virtual StorageErr set_task_state(boost::uuids::uuid id, TaskState state) = 0; - virtual StorageErr add_task_instance(TaskInstance const& instance) = 0; - virtual StorageErr task_finish(TaskInstance const& instance) = 0; - virtual StorageErr get_task_timeout(std::vector& tasks) = 0; - virtual StorageErr get_child_task(boost::uuids::uuid id, Task& child) = 0; - virtual StorageErr get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) = 0; + virtual auto add_task_graph(TaskGraph const& task_graph) -> StorageErr = 0; + virtual auto get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) -> StorageErr = 0; + virtual auto get_task_graphs(std::vector& task_graphs) -> StorageErr = 0; + virtual auto remove_task_graph(boost::uuids::uuid id) -> StorageErr = 0; + virtual auto add_child(boost::uuids::uuid parent_id, Task const& child) -> StorageErr = 0; + virtual auto get_task(boost::uuids::uuid id, Task& task) -> StorageErr = 0; + virtual auto get_ready_tasks(std::vector& tasks) -> StorageErr = 0; + virtual auto set_task_state(boost::uuids::uuid id, TaskState state) -> StorageErr = 0; + virtual auto add_task_instance(TaskInstance const& instance) -> StorageErr = 0; + virtual auto task_finish(TaskInstance const& instance) -> StorageErr = 0; + virtual auto get_task_timeout(std::vector& tasks) -> StorageErr = 0; + virtual auto get_child_task(boost::uuids::uuid id, Task& child) -> StorageErr = 0; + virtual auto get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) -> StorageErr + = 0; - virtual StorageErr update_heartbeat(boost::uuids::uuid id) = 0; - virtual StorageErr hearbeat_timeout(std::vector& ids) = 0; + virtual auto update_heartbeat(boost::uuids::uuid id) -> StorageErr = 0; + virtual auto heartbeat_timeout(std::vector& ids) -> StorageErr = 0; }; } // namespace spider::core diff --git a/src/spider/storage/MysqlStorage.cpp b/src/spider/storage/MysqlStorage.cpp index 40a7ea0b8..62aa48222 100644 --- a/src/spider/storage/MysqlStorage.cpp +++ b/src/spider/storage/MysqlStorage.cpp @@ -1,62 +1,131 @@ #include "MysqlStorage.hpp" +#include +#include +#include + +#include "../core/Data.hpp" +#include "../core/Error.hpp" +#include "../core/Task.hpp" +#include "../core/TaskGraph.hpp" + namespace spider::core { -StorageErr MySqlMetadataStorage::connect(std::string url, boost::uuids::uuid id) {} +auto MySqlMetadataStorage::connect(std::string /*url*/, boost::uuids::uuid /*id*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} void MySqlMetadataStorage::close() {} -StorageErr MySqlMetadataStorage::initialize() {} +auto MySqlMetadataStorage::initialize() -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::add_task_graph(TaskGraph const& task_graph) {} +auto MySqlMetadataStorage::add_task_graph(TaskGraph const& /*task_graph*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) {} +auto MySqlMetadataStorage::get_task_graph(boost::uuids::uuid /*id*/, TaskGraph& /*task_graph*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::get_task_graphs(std::vector& task_graphs) {} +auto MySqlMetadataStorage::get_task_graphs(std::vector& /*task_graphs*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::remove_task_graph(boost::uuids::uuid id) {} +auto MySqlMetadataStorage::remove_task_graph(boost::uuids::uuid /*id*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::add_child(boost::uuids::uuid parent_id, Task const& child) {} +auto MySqlMetadataStorage::add_child(boost::uuids::uuid /*parent_id*/, Task const& /*child*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::get_task(boost::uuids::uuid id, Task& task) {} +auto MySqlMetadataStorage::get_task(boost::uuids::uuid /*id*/, Task& /*task*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::get_ready_tasks(std::vector& tasks) {} +auto MySqlMetadataStorage::get_ready_tasks(std::vector& /*tasks*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::set_task_state(boost::uuids::uuid id, TaskState state) {} +auto MySqlMetadataStorage::set_task_state(boost::uuids::uuid /*id*/, TaskState /*state*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::add_task_instance(TaskInstance const& instance) {} +auto MySqlMetadataStorage::add_task_instance(TaskInstance const& /*instance*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::task_finish(TaskInstance const& instance) {} +auto MySqlMetadataStorage::task_finish(TaskInstance const& /*instance*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::get_task_timeout(std::vector& tasks) {} +auto MySqlMetadataStorage::get_task_timeout(std::vector& /*tasks*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr get_child_task(boost::uuids::uuid id, Task& child) {} +auto MySqlMetadataStorage::get_child_task(boost::uuids::uuid /*id*/, Task& /*child*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) {} +auto MySqlMetadataStorage::get_parent_tasks(boost::uuids::uuid /*id*/, std::vector& /*tasks*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::update_heartbeat(boost::uuids::uuid id) {} +auto MySqlMetadataStorage::update_heartbeat(boost::uuids::uuid /*id*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MySqlMetadataStorage::hearbeat_timeout(std::vector& ids) {} +auto MySqlMetadataStorage::heartbeat_timeout(std::vector& /*ids*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MysqlDataStorage::connect(std::string url, boost::uuids::uuid id) {} +auto MySqlDataStorage::connect(std::string /*url*/, boost::uuids::uuid /*id*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -void MysqlDataStorage::close() {} +void MySqlDataStorage::close() {} -StorageErr MysqlDataStorage::initialize() {} +auto MySqlDataStorage::initialize() -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MysqlDataStorage::add_data(Data const& data) {} +auto MySqlDataStorage::add_data(Data const& /*data*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MysqlDataStorage::get_data(boost::uuids::uuid id, Data& data) {} +auto MySqlDataStorage::get_data(boost::uuids::uuid /*id*/, Data& /*data*/) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr MysqlDataStorage::add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) { +auto MySqlDataStorage::add_task_reference(boost::uuids::uuid /*id*/, boost::uuids::uuid /*task_id*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; } -StorageErr -MysqlDataStorage::remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) {} +auto MySqlDataStorage:: + remove_task_reference(boost::uuids::uuid /*id*/, boost::uuids::uuid /*task_id*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr -MysqlDataStorage::add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) {} +auto MySqlDataStorage:: + add_driver_reference(boost::uuids::uuid /*id*/, boost::uuids::uuid /*driver_id*/) + -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} -StorageErr -MysqlDataStorage::remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) {} +auto MySqlDataStorage::remove_driver_reference( + boost::uuids::uuid /*id*/, + boost::uuids::uuid /*driver_id*/ +) -> StorageErr { + return StorageErr{StorageErrType::Success, ""}; +} } // namespace spider::core diff --git a/src/spider/storage/MysqlStorage.hpp b/src/spider/storage/MysqlStorage.hpp index 248cf22d0..a577b5e98 100644 --- a/src/spider/storage/MysqlStorage.hpp +++ b/src/spider/storage/MysqlStorage.hpp @@ -1,44 +1,65 @@ #ifndef SPIDER_STORAGE_MYSQLSTORAGE_HPP #define SPIDER_STORAGE_MYSQLSTORAGE_HPP +#include +#include +#include + +#include "../core/Data.hpp" +#include "../core/Error.hpp" +#include "../core/Task.hpp" +#include "../core/TaskGraph.hpp" #include "DataStorage.hpp" #include "MetadataStorage.hpp" namespace spider::core { class MySqlMetadataStorage : public MetadataStorage { public: - StorageErr connect(std::string url, boost::uuids::uuid id) override; + MySqlMetadataStorage(MySqlMetadataStorage const&) = delete; + MySqlMetadataStorage(MySqlMetadataStorage&&) = delete; + auto operator=(MySqlMetadataStorage const&) -> MySqlMetadataStorage& = delete; + auto operator=(MySqlMetadataStorage&&) -> MySqlMetadataStorage& = delete; + ~MySqlMetadataStorage() override = default; + auto connect(std::string url, boost::uuids::uuid id) -> StorageErr override; void close() override; - StorageErr initialize() override; - StorageErr add_task_graph(TaskGraph const& task_graph) override; - StorageErr get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) override; - StorageErr get_task_graphs(std::vector& task_graphs) override; - StorageErr remove_task_graph(boost::uuids::uuid id) override; - StorageErr add_child(boost::uuids::uuid parent_id, Task const& child) override; - StorageErr get_task(boost::uuids::uuid id, Task& task) override; - StorageErr get_ready_tasks(std::vector& tasks) override; - StorageErr set_task_state(boost::uuids::uuid id, TaskState state) override; - StorageErr add_task_instance(TaskInstance const& instance) override; - StorageErr task_finish(TaskInstance const& instance) override; - StorageErr get_task_timeout(std::vector& tasks) override; - StorageErr get_child_task(boost::uuids::uuid id, Task& child) override; - StorageErr get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) override; - StorageErr update_heartbeat(boost::uuids::uuid id) override; - StorageErr hearbeat_timeout(std::vector& ids) override; + auto initialize() -> StorageErr override; + auto add_task_graph(TaskGraph const& task_graph) -> StorageErr override; + auto get_task_graph(boost::uuids::uuid id, TaskGraph& task_graph) -> StorageErr override; + auto get_task_graphs(std::vector& task_graphs) -> StorageErr override; + auto remove_task_graph(boost::uuids::uuid id) -> StorageErr override; + auto add_child(boost::uuids::uuid parent_id, Task const& child) -> StorageErr override; + auto get_task(boost::uuids::uuid id, Task& task) -> StorageErr override; + auto get_ready_tasks(std::vector& tasks) -> StorageErr override; + auto set_task_state(boost::uuids::uuid id, TaskState state) -> StorageErr override; + auto add_task_instance(TaskInstance const& instance) -> StorageErr override; + auto task_finish(TaskInstance const& instance) -> StorageErr override; + auto get_task_timeout(std::vector& tasks) -> StorageErr override; + auto get_child_task(boost::uuids::uuid id, Task& child) -> StorageErr override; + auto get_parent_tasks(boost::uuids::uuid id, std::vector& tasks) -> StorageErr override; + auto update_heartbeat(boost::uuids::uuid id) -> StorageErr override; + auto heartbeat_timeout(std::vector& ids) -> StorageErr override; }; -class MysqlDataStorage : public DataStorage { +class MySqlDataStorage : public DataStorage { public: - StorageErr connect(std::string url, boost::uuids::uuid id) override; + MySqlDataStorage(MySqlDataStorage const&) = delete; + MySqlDataStorage(MySqlDataStorage&&) = delete; + auto operator=(MySqlDataStorage const&) -> MySqlDataStorage& = delete; + auto operator=(MySqlDataStorage&&) -> MySqlDataStorage& = delete; + ~MySqlDataStorage() override = default; + auto connect(std::string url, boost::uuids::uuid id) -> StorageErr override; void close() override; - StorageErr initialize() override; - StorageErr add_data(Data const& data) override; - StorageErr get_data(boost::uuids::uuid id, Data& data) override; - StorageErr add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) override; - StorageErr remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) override; - StorageErr add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) override; - StorageErr - remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) override; + auto initialize() -> StorageErr override; + auto add_data(Data const& data) -> StorageErr override; + auto get_data(boost::uuids::uuid id, Data& data) -> StorageErr override; + auto + add_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr override; + auto + remove_task_reference(boost::uuids::uuid id, boost::uuids::uuid task_id) -> StorageErr override; + auto add_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) + -> StorageErr override; + auto remove_driver_reference(boost::uuids::uuid id, boost::uuids::uuid driver_id) + -> StorageErr override; }; } // namespace spider::core From 29a2428e4f0d4243977f4b2fc1c26219e60b31f6 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Tue, 29 Oct 2024 17:29:40 +0800 Subject: [PATCH 4/6] fix: Add sudo for removing boost tmp directory on linux --- tools/scripts/lib_install/boost.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/scripts/lib_install/boost.sh b/tools/scripts/lib_install/boost.sh index e35005340..19a1e445a 100755 --- a/tools/scripts/lib_install/boost.sh +++ b/tools/scripts/lib_install/boost.sh @@ -12,8 +12,10 @@ version=$1 version_with_underscores=${version//./_} echo "Checking for elevated privileges..." +privileged_command_prefix="" if [ ${EUID:-$(id -u)} -ne 0 ] ; then sudo echo "Script can elevate privileges." + privileged_command_prefix="${privileged_command_prefix} sudo" fi # Get number of cpu cores @@ -38,11 +40,7 @@ cd boost_${version_with_underscores} ./b2 -j${num_cpus} # Install -if [ ${EUID:-$(id -u)} -ne 0 ] ; then - sudo ./b2 install -else - ./b2 install -fi +${privileged_command_prefix} ./b2 install # Clean up -rm -rf $temp_dir \ No newline at end of file +${privileged_command_prefix} rm -rf $temp_dir \ No newline at end of file From 4c0555c4f2a0e3599a52bfa2ea2cbc5d46ec4a8e Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Wed, 30 Oct 2024 16:47:25 +0800 Subject: [PATCH 5/6] style: Improve code quality based on coderabbit recommendation --- src/spider/CMakeLists.txt | 6 ------ src/spider/core/Data.hpp | 15 ++++++++------- src/spider/core/Error.hpp | 6 ++++-- src/spider/core/Task.hpp | 8 ++++++-- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/spider/CMakeLists.txt b/src/spider/CMakeLists.txt index 70463e264..33370968d 100644 --- a/src/spider/CMakeLists.txt +++ b/src/spider/CMakeLists.txt @@ -21,12 +21,6 @@ target_sources( ${SPIDER_CORE_SOURCES} ${SPIDER_WORKER_SOURCES} ) -target_sources( - spider_worker - PRIVATE - ${SPIDER_CORE_SOURCES} - ${SPIDER_WORKER_SOURCES} -) target_link_libraries( spider_worker Boost::boost diff --git a/src/spider/core/Data.hpp b/src/spider/core/Data.hpp index 04abe360b..37348bb7e 100644 --- a/src/spider/core/Data.hpp +++ b/src/spider/core/Data.hpp @@ -13,22 +13,23 @@ class Data { std::optional m_key; std::string m_value; -public: - explicit Data(std::string value) : m_value(std::move(value)) { + void init_id() { boost::uuids::random_generator gen; m_id = gen(); } +public: + explicit Data(std::string value) : m_value(std::move(value)) { init_id(); } + Data(std::string key, std::string value) : m_key(std::move(key)), m_value(std::move(value)) { - boost::uuids::random_generator gen; - m_id = gen(); + init_id(); } - auto get_id() -> boost::uuids::uuid { return m_id; } + [[nodiscard]] auto get_id() const -> boost::uuids::uuid { return m_id; } - auto get_key() -> std::optional { return m_key; } + [[nodiscard]] auto get_key() const -> std::optional { return m_key; } - auto get_value() -> std::string { return m_value; } + [[nodiscard]] auto get_value() const -> std::string { return m_value; } }; #endif // SPIDER_CORE_DATA_HPP diff --git a/src/spider/core/Error.hpp b/src/spider/core/Error.hpp index e9d78cb18..6a7c78680 100644 --- a/src/spider/core/Error.hpp +++ b/src/spider/core/Error.hpp @@ -7,18 +7,20 @@ namespace spider::core { enum class StorageErrType : std::uint8_t { + Success = 0, ConnectionErr, DbNotFound, KeyNotFoundErr, DuplicateKeyErr, - ConstraintViolationErr, - Success + ConstraintViolationErr }; struct StorageErr { StorageErrType type; std::string description; + StorageErr() : type(StorageErrType::Success) {} + StorageErr(StorageErrType type, std::string description) : type(type), description(std::move(description)) {} diff --git a/src/spider/core/Task.hpp b/src/spider/core/Task.hpp index c0cd0fd3c..71330de18 100644 --- a/src/spider/core/Task.hpp +++ b/src/spider/core/Task.hpp @@ -24,7 +24,9 @@ class TaskInput { TaskInput(boost::uuids::uuid output_task_id, std::uint8_t position, std::string type) : m_task_output({output_task_id, position}), m_type(std::move(type)) {}; - TaskInput(std::string value, std::string type) : m_value(value), m_type(std::move(type)) {}; + TaskInput(std::string value, std::string type) + : m_value(std::move(value)), + m_type(std::move(type)) {}; TaskInput(boost::uuids::uuid data_id, std::string type) : m_data_id(data_id), m_type(std::move(type)) {}; @@ -50,7 +52,9 @@ class TaskOutput { std::string m_type; public: - TaskOutput(std::string value, std::string type) : m_value(value), m_type(std::move(type)) {} + TaskOutput(std::string value, std::string type) + : m_value(std::move(value)), + m_type(std::move(type)) {} TaskOutput(boost::uuids::uuid data_id, std::string type) : m_data_id(data_id), From 6c5667bcffc50111728ea929eca49ad94dd20a87 Mon Sep 17 00:00:00 2001 From: Sitao Wang Date: Wed, 30 Oct 2024 20:24:06 +0800 Subject: [PATCH 6/6] style: Use default StorageErr constructor for suceess --- src/spider/storage/MysqlStorage.cpp | 50 ++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/spider/storage/MysqlStorage.cpp b/src/spider/storage/MysqlStorage.cpp index 62aa48222..04641ad9c 100644 --- a/src/spider/storage/MysqlStorage.cpp +++ b/src/spider/storage/MysqlStorage.cpp @@ -11,121 +11,121 @@ namespace spider::core { auto MySqlMetadataStorage::connect(std::string /*url*/, boost::uuids::uuid /*id*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } void MySqlMetadataStorage::close() {} auto MySqlMetadataStorage::initialize() -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::add_task_graph(TaskGraph const& /*task_graph*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::get_task_graph(boost::uuids::uuid /*id*/, TaskGraph& /*task_graph*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::get_task_graphs(std::vector& /*task_graphs*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::remove_task_graph(boost::uuids::uuid /*id*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::add_child(boost::uuids::uuid /*parent_id*/, Task const& /*child*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::get_task(boost::uuids::uuid /*id*/, Task& /*task*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::get_ready_tasks(std::vector& /*tasks*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::set_task_state(boost::uuids::uuid /*id*/, TaskState /*state*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::add_task_instance(TaskInstance const& /*instance*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::task_finish(TaskInstance const& /*instance*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::get_task_timeout(std::vector& /*tasks*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::get_child_task(boost::uuids::uuid /*id*/, Task& /*child*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::get_parent_tasks(boost::uuids::uuid /*id*/, std::vector& /*tasks*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::update_heartbeat(boost::uuids::uuid /*id*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlMetadataStorage::heartbeat_timeout(std::vector& /*ids*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlDataStorage::connect(std::string /*url*/, boost::uuids::uuid /*id*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } void MySqlDataStorage::close() {} auto MySqlDataStorage::initialize() -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlDataStorage::add_data(Data const& /*data*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlDataStorage::get_data(boost::uuids::uuid /*id*/, Data& /*data*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlDataStorage::add_task_reference(boost::uuids::uuid /*id*/, boost::uuids::uuid /*task_id*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlDataStorage:: remove_task_reference(boost::uuids::uuid /*id*/, boost::uuids::uuid /*task_id*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlDataStorage:: add_driver_reference(boost::uuids::uuid /*id*/, boost::uuids::uuid /*driver_id*/) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } auto MySqlDataStorage::remove_driver_reference( boost::uuids::uuid /*id*/, boost::uuids::uuid /*driver_id*/ ) -> StorageErr { - return StorageErr{StorageErrType::Success, ""}; + return StorageErr{}; } } // namespace spider::core