feat(spider-py): Add job submission support for MariaDB storage backend.

.github/workflows/unit-tests.yaml

@@ -57,4 +57,4 @@ jobs:
 
       - run: "task test:cpp-non-storage-unit-tests"
 
-      - run: "task test:spider-py-unit-tests"
+      - run: "task test:spider-py-non-storage-unit-tests"

docs/src/dev-docs/testing.md

@@ -30,12 +30,14 @@ require this storage backend.
 
 You can use the following tasks to run the set of unit tests that's appropriate.
 
-| Task                              | Description                                                           |
-|-----------------------------------|-----------------------------------------------------------------------|
-| `test:cpp-unit-tests`             | Runs all C++ unit tests.                                              |
-| `test:cpp-non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. |
-| `test:cpp-storage-unit-tests`     | Runs all C++ unit tests which require a storage backend to run.       |
-| `test:spider-py-unit-tests`       | Runs all spider-py unit tests.                                        |
+| Task                                    | Description                                                                 |
+|-----------------------------------------|-----------------------------------------------------------------------------|
+| `test:cpp-unit-tests`                   | Runs all C++ unit tests.                                                    |
+| `test:cpp-non-storage-unit-tests`       | Runs all C++ unit tests which don't require a storage backend to run.       |
+| `test:cpp-storage-unit-tests`           | Runs all C++ unit tests which require a storage backend to run.             |
+| `test:spider-py-unit-tests`             | Runs all spider-py unit tests.                                              |
+| `test:spider-py-non-storage-unit-tests` | Runs all spider-py unit tests which don't require a storage backend to run. |
+| `test:spider-py-storage-unit-tests`     | Runs all spider-py unit tests which require a storage backend to run.       |
 
 If any tests show error messages for the connection function below, revisit the
 [setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly.

python/spider-py/README.md

@@ -25,12 +25,58 @@ directory at the Spider project root.
 
 ## Testing
 
-To run all unit tests:
+Unit tests are divided into two categories: storage and non-storage tests. Non-storage tests do not
+require any external services, while storage tests require a MariaDB instance to be available.
+
+### Non-Storage Unit Tests
+
+To run all non-storage unit tests:
+
+```shell
+task test:spider-py-non-storage-unit-tests
+```
+
+### Setup MariaDB for Storage Unit Tests
+
+To run storage unit tests, we need to create a MariaDB instance first.
+
+```shell
+docker run \
+        --detach \
+        --rm \
+        --name spider-storage \
+        --env MARIADB_USER=spider \
+        --env MARIADB_PASSWORD=password \
+        --env MARIADB_DATABASE=spider-storage \
+        --env MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=true \
+        --publish 3306:3306 mariadb:latest
+```
+
+The database tables will be automatically created by the spawned spider scheduler.
+
+You can choose to set up the database table manually by using the SQL script
+`tools/scripts/storage/init_db.sql` from the project root.
+
+### Storage Unit Tests
+
+To run all storage unit tests:
+
+```shell
+task test:spider-py-storage-unit-tests
+```
+
+This requires a running MariaDB instance as described above.
+
+### All Unit Tests
+
+To run all unit tests (both storage and non-storage):
 
 ```shell
 task test:spider-py-unit-tests
 ```
 
+This requires a running MariaDB instance as described above.
+
 ## Linting
 
 To run all linting checks:

python/spider-py/pyproject.toml

@@ -35,7 +35,15 @@ pretty = true
 show_error_code_links = true
 show_error_context = true
 
+[[tool.mypy.overrides]]
+module = ["mariadb.*"]
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
+markers = [
+  "integration: mark test as an integration test",
+  "storage: mark unit tests needing storage",
+]
 testpaths = ["tests"]
 
 [tool.ruff]

python/spider-py/src/spider_py/core/__init__.py

@@ -1,6 +1,7 @@
 """Spider core package."""
 
 from .data import Data, DataId
+from .driver import DriverId
 from .task import (
     Task,
     TaskId,
@@ -13,11 +14,13 @@
     TaskOutputValue,
     TaskState,
 )
-from .task_graph import TaskGraph
+from .task_graph import JobId, TaskGraph
 
 __all__ = [
     "Data",
     "DataId",
+    "DriverId",
+    "JobId",
     "Task",
     "TaskGraph",
     "TaskId",

python/spider-py/src/spider_py/core/driver.py

@@ -0,0 +1,5 @@
+"""Spider core Driver module."""
+
+from uuid import UUID
+
+DriverId = UUID

python/spider-py/src/spider_py/core/task.py

@@ -1,5 +1,7 @@
 """Task module for Spider."""
 
+from __future__ import annotations
+
 from dataclasses import dataclass, field
 from enum import IntEnum
 from uuid import UUID, uuid4
@@ -26,7 +28,7 @@ class TaskInput:
     """Represents a task input"""
 
     type: str
-    value: TaskInputData | TaskInputOutput | None
+    value: TaskInputData | TaskInputOutput | TaskInputValue | None
 
 
 TaskOutputValue = bytes
@@ -51,6 +53,39 @@ class TaskState(IntEnum):
     Failed = 4
     Cancelled = 5
 
+    def get_state_str(self) -> str:
+        """
+        Returns string representation of task state.
+        :return: The string representation of task state.
+        """
+        return _StateToStrMap[self]
+
+    @staticmethod
+    def get_state_from_str(state_str: str) -> TaskState:
+        """
+        Returns task state from string representation.
+        :param state_str: The string representation of task state.
+        :return: The task state from string representation.
+        :raises ValueError: If the state string is not recognized.
+        """
+        state = _StrToStateMap.get(state_str)
+        if state is not None:
+            return state
+        msg = f"Invalid task state string: {state_str}"
+        raise ValueError(msg)
+
+
+_StateToStrMap = {
+    TaskState.Pending: "pending",
+    TaskState.Ready: "ready",
+    TaskState.Running: "running",
+    TaskState.Succeeded: "success",
+    TaskState.Failed: "fail",
+    TaskState.Cancelled: "cancel",
+}
+
+_StrToStateMap = {value: key for key, value in _StateToStrMap.items()}
+
 
 @dataclass
 class Task:

python/spider-py/src/spider_py/core/task_graph.py

@@ -4,10 +4,13 @@
 
 from copy import deepcopy
 from typing import TYPE_CHECKING
+from uuid import UUID
 
 if TYPE_CHECKING:
     from spider_py.core.task import Task
 
+JobId = UUID
+
 
 class TaskGraph:
     """

python/spider-py/src/spider_py/storage/__init__.py

@@ -0,0 +1,13 @@
+"""Spider Storage package."""
+
+from .jdbc_url import JdbcParameters, parse_jdbc_url
+from .mariadb_storage import MariaDBStorage
+from .storage import Storage, StorageError
+
+__all__ = [
+    "JdbcParameters",
+    "MariaDBStorage",
+    "Storage",
+    "StorageError",
+    "parse_jdbc_url",
+]

python/spider-py/src/spider_py/storage/jdbc_url.py

@@ -0,0 +1,64 @@
+"""JDBC URL module."""
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class JdbcParameters:
+    """JDBC url parameters."""
+
+    protocol: str
+    host: str
+    database: str
+    port: int | None = None
+    user: str | None = None
+    password: str | None = None
+
+
+pattern = re.compile(
+    r"^(?P<protocol>[a-zA-Z][a-zA-Z0-9+.-]*(:[a-zA-Z0-9+.-]*)?)://"
+    r"(?P<host>([a-zA-Z0-9.-]+|\d{1,3}(?:\.\d{1,3}){3}))"
+    r"(?::(?P<port>\d+))?"
+    r"/(?P<database>[a-zA-Z0-9_\-]+)"
+    r"(?:\?(?P<query>[a-zA-Z0-9_\-=&]+))?"
+)
+
+
+def parse_jdbc_url(url: str) -> JdbcParameters:
+    """
+    Parses a JDBC URL.
+    :param url:
+    :return:
+    :raises ValueError: If the JDBC URL is invalid.
+    """
+    match = pattern.match(url)
+    if not match:
+        msg = f"Invalid JDBC URL: {url}"
+        raise ValueError(msg)
+
+    groups = match.groupdict()
+    query = groups.get("query") or ""
+    query_params = dict(param.split("=", 1) for param in query.split("&") if "=" in param)
+
+    protocol = groups.get("protocol")
+    if not protocol:
+        msg = f"Protocol is required in JDBC URL: {url}"
+        raise ValueError(msg)
+    host = groups.get("host")
+    if not host:
+        msg = f"Host is required in JDBC URL: {url}"
+        raise ValueError(msg)
+    database = groups.get("database")
+    if not database:
+        msg = f"Database is required in JDBC URL: {url}"
+        raise ValueError(msg)
+
+    return JdbcParameters(
+        protocol=protocol,
+        host=host,
+        port=int(groups["port"]) if groups.get("port") else None,
+        database=database,
+        user=query_params.get("user"),
+        password=query_params.get("password"),
+    )