feat: Add storage interface

.gitignore

@@ -8,3 +8,4 @@ build
 
 # IDE-related directories and files
 .idea
+cmake-build-debug

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "submodules/Catch2"]
 	path = submodules/Catch2
 	url = https://github.com/catchorg/Catch2.git
+[submodule "submodules/abseil-cpp"]
+	path = submodules/abseil-cpp
+	url = https://github.com/abseil/abseil-cpp.git

CMakeLists.txt

@@ -77,6 +77,17 @@ else()
 endif()
 message(STATUS "Building using ${SPIDER_LIBS_STRING} libraries")
 
+# Find and setup Boost Library
+if(SPIDER_USE_STATIC_LIBS)
+    set(Boost_USE_STATIC_LIBS ON)
+endif()
+find_package(Boost 1.74 REQUIRED)
+if(Boost_FOUND)
+    message(STATUS "Found Boost ${Boost_VERSION}")
+else()
+    message(FATAL_ERROR "Could not find ${SPIDER_LIBS_STRING} libraries for Boost")
+endif()
+
 # Find and setup fmt
 find_package(fmt 8.0.1 REQUIRED)
 if(fmt_FOUND)
@@ -106,17 +117,21 @@ else()
 endif()
 
 # Find and setup MariaDBClient library
-if(CLP_USE_STATIC_LIBS)
+if(SPIDER_USE_STATIC_LIBS)
     # NOTE: We can't statically link to MariaDBClient since it's GPL
     message(AUTHOR_WARNING "MariaDBClient cannot be statically linked due to its license.")
 endif()
 find_package(MariaDBClient 3.1.0 REQUIRED)
 if(MariaDBClient_FOUND)
     message(STATUS "Found MariaDBClient ${MariaDBClient_VERSION}")
 else()
-    message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for MariaDBClient")
+    message(FATAL_ERROR "Could not find ${SPIDER_LIBS_STRING} libraries for MariaDBClient")
 endif()
 
+# Add abseil-cpp
+set(ABSL_PROPAGATE_CXX_STD ON)
+add_subdirectory(submodules/abseil-cpp EXCLUDE_FROM_ALL)
+
 find_package(Threads REQUIRED)
 
 add_subdirectory(src/spider)

src/spider/CMakeLists.txt

@@ -1,6 +1,16 @@
 # set variable as CACHE INTERNAL to access it from other scope
-set(SPIDER_CORE_SOURCES storage/MetadataStorage.hpp CACHE INTERNAL "spider core source files")
-
+set(SPIDER_CORE_SOURCES
+    core/Error.hpp
+    core/Data.hpp
+    core/Task.hpp
+    core/TaskGraph.hpp
+    storage/MetadataStorage.hpp
+    storage/DataStorage.hpp
+    storage/MysqlStorage.cpp
+    storage/MysqlStorage.hpp
+    CACHE INTERNAL
+    "spider core source files"
+)
 set(SPIDER_WORKER_SOURCES worker/worker.cpp CACHE INTERNAL "spider worker source files")
 
 add_executable(spider_worker)
@@ -11,4 +21,9 @@ target_sources(
         ${SPIDER_CORE_SOURCES}
         ${SPIDER_WORKER_SOURCES}
 )
+target_link_libraries(
+    spider_worker
+    Boost::boost
+    absl::flat_hash_map
+)
 add_executable(spider::worker ALIAS spider_worker)

src/spider/core/Data.hpp

@@ -0,0 +1,35 @@
+#ifndef SPIDER_CORE_DATA_HPP
+#define SPIDER_CORE_DATA_HPP
+
+#include <boost/uuid/random_generator.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <optional>
+#include <string>
+#include <utility>
+
+class Data {
+private:
+    boost::uuids::uuid m_id;
+    std::optional<std::string> m_key;
+    std::string m_value;
+
+    void init_id() {
+        boost::uuids::random_generator gen;
+        m_id = gen();
+    }
+
+public:
+    explicit Data(std::string value) : m_value(std::move(value)) { init_id(); }
+
+    Data(std::string key, std::string value) : m_key(std::move(key)), m_value(std::move(value)) {
+        init_id();
+    }
+
+    [[nodiscard]] auto get_id() const -> boost::uuids::uuid { return m_id; }
+
+    [[nodiscard]] auto get_key() const -> std::optional<std::string> { return m_key; }
+
+    [[nodiscard]] auto get_value() const -> std::string { return m_value; }
+};
+
+#endif  // SPIDER_CORE_DATA_HPP

src/spider/core/Error.hpp

@@ -0,0 +1,31 @@
+#ifndef SPIDER_CORE_ERROR_HPP
+#define SPIDER_CORE_ERROR_HPP
+
+#include <cstdint>
+#include <string>
+#include <utility>
+
+namespace spider::core {
+enum class StorageErrType : std::uint8_t {
+    Success = 0,
+    ConnectionErr,
+    DbNotFound,
+    KeyNotFoundErr,
+    DuplicateKeyErr,
+    ConstraintViolationErr
+};
+
+struct StorageErr {
+    StorageErrType type;
+    std::string description;
+
+    StorageErr() : type(StorageErrType::Success) {}
+
+    StorageErr(StorageErrType type, std::string description)
+            : type(type),
+              description(std::move(description)) {}
+};
+
+}  // namespace spider::core
+
+#endif  // SPIDER_CORE_ERROR_HPP

src/spider/core/Task.hpp

@@ -0,0 +1,135 @@
+#ifndef SPIDER_CORE_TASK_HPP
+#define SPIDER_CORE_TASK_HPP
+
+#include <boost/uuid/random_generator.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+namespace spider::core {
+
+class TaskInput {
+private:
+    std::optional<std::tuple<boost::uuids::uuid, std::uint8_t>> m_task_output;
+    std::optional<std::string> m_value;
+    std::optional<boost::uuids::uuid> m_data_id;
+    std::string m_type;
+
+public:
+    TaskInput(boost::uuids::uuid output_task_id, std::uint8_t position, std::string type)
+            : m_task_output({output_task_id, position}),
+              m_type(std::move(type)) {};
+    TaskInput(std::string value, std::string type)
+            : m_value(std::move(value)),
+              m_type(std::move(type)) {};
+    TaskInput(boost::uuids::uuid data_id, std::string type)
+            : m_data_id(data_id),
+              m_type(std::move(type)) {};
+
+    [[nodiscard]] auto get_task_output(
+    ) const -> std::optional<std::tuple<boost::uuids::uuid, std::uint8_t>> {
+        return m_task_output;
+    }
+
+    [[nodiscard]] auto get_value() const -> std::optional<std::string> { return m_value; }
+
+    [[nodiscard]] auto get_data_id() const -> std::optional<boost::uuids::uuid> {
+        return m_data_id;
+    }
+
+    [[nodiscard]] auto get_type() const -> std::string { return m_type; }
+};
+
+class TaskOutput {
+private:
+    std::optional<std::string> m_value;
+    std::optional<boost::uuids::uuid> m_data_id;
+    std::string m_type;
+
+public:
+    TaskOutput(std::string value, std::string type)
+            : m_value(std::move(value)),
+              m_type(std::move(type)) {}
+
+    TaskOutput(boost::uuids::uuid data_id, std::string type)
+            : m_data_id(data_id),
+              m_type(std::move(type)) {}
+
+    [[nodiscard]] auto get_value() const -> std::optional<std::string> { return m_value; }
+
+    [[nodiscard]] auto get_data_id() const -> std::optional<boost::uuids::uuid> {
+        return m_data_id;
+    }
+
+    [[nodiscard]] auto get_type() const -> std::string { return m_type; }
+};
+
+class TaskInstance {};
+
+enum class TaskState : std::uint8_t {
+    Pending,
+    Ready,
+    Running,
+    Succeed,
+    Failed,
+    Canceled,
+};
+
+enum class TaskCreatorType : std::uint8_t {
+    Client = 0,
+    Task,
+};
+
+class Task {
+private:
+    boost::uuids::uuid m_id;
+    std::string m_function_name;
+    TaskState m_state = TaskState::Pending;
+    TaskCreatorType m_creator_type;
+    boost::uuids::uuid m_creator_id;
+    float m_timeout = 0;
+    std::vector<TaskInput> m_inputs;
+    std::vector<TaskOutput> m_outputs;
+
+public:
+    Task(std::string function_name, TaskCreatorType creator_type, boost::uuids::uuid creator_id)
+            : m_function_name(std::move(function_name)),
+              m_creator_type(creator_type),
+              m_creator_id(creator_id) {
+        boost::uuids::random_generator gen;
+        m_id = gen();
+    }
+
+    void add_input(TaskInput const& input) { m_inputs.emplace_back(input); }
+
+    void add_output(TaskOutput const& output) { m_outputs.emplace_back(output); }
+
+    [[nodiscard]] auto get_id() const -> boost::uuids::uuid { return m_id; }
+
+    [[nodiscard]] auto get_function_name() const -> std::string { return m_function_name; }
+
+    [[nodiscard]] auto get_state() const -> TaskState { return m_state; }
+
+    [[nodiscard]] auto get_creator_type() const -> TaskCreatorType { return m_creator_type; }
+
+    [[nodiscard]] auto get_creator_id() const -> boost::uuids::uuid { return m_creator_id; }
+
+    [[nodiscard]] auto get_timeout() const -> float { return m_timeout; }
+
+    [[nodiscard]] auto get_num_inputs() const -> size_t { return m_inputs.size(); }
+
+    [[nodiscard]] auto get_num_outputs() const -> size_t { return m_outputs.size(); }
+
+    [[nodiscard]] auto get_input(uint64_t index) const -> TaskInput { return m_inputs[index]; }
+
+    [[nodiscard]] auto get_output(uint64_t index) const -> TaskOutput { return m_outputs[index]; }
+};
+
+}  // namespace spider::core
+
+#endif  // SPIDER_CORE_TASK_HPP

src/spider/core/TaskGraph.hpp

@@ -0,0 +1,79 @@
+#ifndef SPIDER_CORE_TASKGRAPH_HPP
+#define SPIDER_CORE_TASKGRAPH_HPP
+
+#include <absl/container/flat_hash_map.h>
+
+#include <boost/uuid/uuid.hpp>
+#include <optional>
+#include <utility>
+#include <vector>
+
+#include "Task.hpp"
+
+namespace spider::core {
+
+class TaskGraph {
+private:
+    absl::flat_hash_map<boost::uuids::uuid, Task> m_tasks;
+    std::vector<std::pair<boost::uuids::uuid, boost::uuids::uuid>> m_dependencies;
+
+public:
+    auto add_child_task(Task const& task, std::vector<boost::uuids::uuid> const& parents) -> bool {
+        boost::uuids::uuid const task_id = task.get_id();
+        for (boost::uuids::uuid const parent_id : parents) {
+            if (!m_tasks.contains(parent_id)) {
+                return false;
+            }
+        }
+        if (m_tasks.contains(task.get_id())) {
+            return false;
+        }
+
+        m_tasks.emplace(task_id, task);
+        for (boost::uuids::uuid const parent_id : parents) {
+            m_dependencies.emplace_back(parent_id, task_id);
+        }
+        return true;
+    }
+
+    [[nodiscard]] auto get_task(boost::uuids::uuid id) const -> std::optional<Task> {
+        if (m_tasks.contains(id)) {
+            return m_tasks.at(id);
+        }
+        return std::nullopt;
+    }
+
+    [[nodiscard]] auto get_child_tasks(boost::uuids::uuid id
+    ) const -> std::vector<boost::uuids::uuid> {
+        std::vector<boost::uuids::uuid> children;
+        for (std::pair<boost::uuids::uuid, boost::uuids::uuid> const dep : m_dependencies) {
+            if (dep.first == id) {
+                children.emplace_back(dep.second);
+            }
+        }
+        return children;
+    }
+
+    [[nodiscard]] auto get_parent_tasks(boost::uuids::uuid id
+    ) const -> std::vector<boost::uuids::uuid> {
+        std::vector<boost::uuids::uuid> parents;
+        for (std::pair<boost::uuids::uuid, boost::uuids::uuid> const dep : m_dependencies) {
+            if (dep.second == id) {
+                parents.emplace_back(dep.first);
+            }
+        }
+        return parents;
+    }
+
+    [[nodiscard]] auto get_tasks() const -> absl::flat_hash_map<boost::uuids::uuid, Task> const& {
+        return m_tasks;
+    }
+
+    [[nodiscard]] auto get_dependencies(
+    ) const -> std::vector<std::pair<boost::uuids::uuid, boost::uuids::uuid>> const& {
+        return m_dependencies;
+    }
+};
+}  // namespace spider::core
+
+#endif  // SPIDER_CORE_TASKGRAPH_HPP