feat(spider-py): Add job submission support for MariaDB storage backend.

.github/workflows/unit-tests.yaml

@@ -57,4 +57,4 @@ jobs:
 
       - run: "task test:cpp-non-storage-unit-tests"
 
-      - run: "task test:spider-py-unit-tests"
+      - run: "task test:spider-py-non-storage-unit-tests"

docs/src/dev-docs/testing.md

@@ -30,12 +30,14 @@ require this storage backend.
 
 You can use the following tasks to run the set of unit tests that's appropriate.
 
-| Task                              | Description                                                           |
-|-----------------------------------|-----------------------------------------------------------------------|
-| `test:cpp-unit-tests`             | Runs all C++ unit tests.                                              |
-| `test:cpp-non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. |
-| `test:cpp-storage-unit-tests`     | Runs all C++ unit tests which require a storage backend to run.       |
-| `test:spider-py-unit-tests`       | Runs all spider-py unit tests.                                        |
+| Task                                    | Description                                                                 |
+|-----------------------------------------|-----------------------------------------------------------------------------|
+| `test:cpp-unit-tests`                   | Runs all C++ unit tests.                                                    |
+| `test:cpp-non-storage-unit-tests`       | Runs all C++ unit tests which don't require a storage backend to run.       |
+| `test:cpp-storage-unit-tests`           | Runs all C++ unit tests which require a storage backend to run.             |
+| `test:spider-py-unit-tests`             | Runs all spider-py unit tests.                                              |
+| `test:spider-py-non-storage-unit-tests` | Runs all spider-py unit tests which don't require a storage backend to run. |
+| `test:spider-py-storage-unit-tests`     | Runs all spider-py unit tests which require a storage backend to run.       |
 
 If any tests show error messages for the connection function below, revisit the
 [setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly.

python/spider-py/README.md

@@ -25,12 +25,58 @@ directory at the Spider project root.
 
 ## Testing
 
-To run all unit tests:
+Unit tests are divided into two categories: storage and non-storage tests. Non-storage tests do not
+require any external services, while storage tests require a MariaDB instance to be available.
+
+### Non-Storage Unit Tests
+
+To run all non-storage unit tests:
+
+```shell
+task test:spider-py-non-storage-unit-tests
+```
+
+### Setup MariaDB for Storage Unit Tests
+
+To run storage unit tests, we need to create a MariaDB instance first.
+
+```shell
+docker run \
+        --detach \
+        --rm \
+        --name spider-storage \
+        --env MARIADB_USER=spider \
+        --env MARIADB_PASSWORD=password \
+        --env MARIADB_DATABASE=spider-storage \
+        --env MARIADB_ALLOW_EMPTY_ROOT_PASSWORD=true \
+        --publish 3306:3306 mariadb:latest
+```
+
+The database tables will be automatically created by the spawned spider scheduler.
+
+You can choose to set up the database table manually by using the SQL script
+`tools/scripts/storage/init_db.sql` from the project root.
+
+### Storage Unit Tests
+
+To run all storage unit tests:
+
+```shell
+task test:spider-py-storage-unit-tests
+```
+
+This requires a running MariaDB instance as described above.
+
+### All Unit Tests
+
+To run all unit tests (both storage and non-storage):
 
 ```shell
 task test:spider-py-unit-tests
 ```
 
+This requires a running MariaDB instance as described above.
+
 ## Linting
 
 To run all linting checks:

python/spider-py/pyproject.toml

@@ -35,7 +35,15 @@ pretty = true
 show_error_code_links = true
 show_error_context = true
 
+[[tool.mypy.overrides]]
+module = ["mariadb.*"]
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
+markers = [
+  "integration: Mark test as an integration test",
+  "storage: Mark unit tests as needing external storage service",
+]
 testpaths = ["tests"]
 
 [tool.ruff]

python/spider-py/src/spider_py/core/__init__.py

@@ -1,6 +1,7 @@
 """Spider core package."""
 
 from .data import Data, DataId
+from .driver import DriverId
 from .task import (
     Task,
     TaskId,
@@ -13,11 +14,13 @@
     TaskOutputValue,
     TaskState,
 )
-from .task_graph import TaskGraph
+from .task_graph import JobId, TaskGraph
 
 __all__ = [
     "Data",
     "DataId",
+    "DriverId",
+    "JobId",
     "Task",
     "TaskGraph",
     "TaskId",

python/spider-py/src/spider_py/core/driver.py

@@ -0,0 +1,5 @@
+"""Spider core Driver module."""
+
+from uuid import UUID
+
+DriverId = UUID

python/spider-py/src/spider_py/core/task.py

@@ -1,5 +1,7 @@
 """Task module for Spider."""
 
+from __future__ import annotations
+
 from dataclasses import dataclass, field
 from enum import IntEnum
 from uuid import UUID, uuid4
@@ -26,7 +28,7 @@ class TaskInput:
     """Represents a task input"""
 
     type: str
-    value: TaskInputData | TaskInputOutput | None
+    value: TaskInputData | TaskInputOutput | TaskInputValue | None
 
 
 TaskOutputValue = bytes
@@ -51,6 +53,39 @@ class TaskState(IntEnum):
     Failed = 4
     Cancelled = 5
 
+    def __str__(self) -> str:
+        """
+        Returns string representation of task state.
+        :return: The string representation of task state.
+        """
+        return _StateToStrMap[self]
+
+    @staticmethod
+    def from_str(state_str: str) -> TaskState:
+        """
+        Returns task state from string representation.
+        :param state_str: The string representation of task state.
+        :return: The task state from string representation.
+        :raises ValueError: If the state string is not recognized.
+        """
+        state = _StrToStateMap.get(state_str)
+        if state is not None:
+            return state
+        msg = f"Invalid task state string: {state_str}"
+        raise ValueError(msg)
+
+
+_StateToStrMap = {
+    TaskState.Pending: "pending",
+    TaskState.Ready: "ready",
+    TaskState.Running: "running",
+    TaskState.Succeeded: "success",
+    TaskState.Failed: "fail",
+    TaskState.Cancelled: "cancel",
+}
+
+_StrToStateMap = {value: key for key, value in _StateToStrMap.items()}
+
 
 @dataclass
 class Task:

python/spider-py/src/spider_py/core/task_graph.py

@@ -4,10 +4,13 @@
 
 from copy import deepcopy
 from typing import TYPE_CHECKING
+from uuid import UUID
 
 if TYPE_CHECKING:
     from spider_py.core.task import Task
 
+JobId = UUID
+
 
 class TaskGraph:
     """

python/spider-py/src/spider_py/storage/__init__.py

@@ -0,0 +1,13 @@
+"""Spider Storage package."""
+
+from .jdbc_url import JdbcParameters, parse_jdbc_url
+from .mariadb_storage import MariaDBStorage
+from .storage import Storage, StorageError
+
+__all__ = [
+    "JdbcParameters",
+    "MariaDBStorage",
+    "Storage",
+    "StorageError",
+    "parse_jdbc_url",
+]

python/spider-py/src/spider_py/storage/jdbc_url.py

@@ -0,0 +1,55 @@
+"""JDBC URL module."""
+
+import urllib.parse
+from dataclasses import dataclass
+
+
+@dataclass
+class JdbcParameters:
+    """JDBC url parameters."""
+
+    protocol: str
+    host: str
+    database: str
+    port: int | None = None
+    user: str | None = None
+    password: str | None = None
+
+
+_JdbcPrefix = "jdbc:"
+
+
+def parse_jdbc_url(url: str) -> JdbcParameters:
+    """
+    Parses a JDBC URL.
+    :param url:
+    :return: Parsed JDBC parameters.
+    :raises ValueError: If the JDBC URL is invalid.
+    """
+    protocol_prefix = ""
+    if url.startswith(_JdbcPrefix):
+        url = url.removeprefix(_JdbcPrefix)
+        protocol_prefix = _JdbcPrefix
+    parsed = urllib.parse.urlparse(url)
+
+    msg = "Invalid JDBC URL: {}. Missing {}."
+    if not parsed.scheme:
+        raise ValueError(msg.format(url, "protocol"))
+    if not parsed.hostname:
+        raise ValueError(msg.format(url, "host"))
+    if not parsed.path or not parsed.path.lstrip("/"):
+        raise ValueError(msg.format(url, "database"))
+
+    database = parsed.path.lstrip("/")
+    query_params = urllib.parse.parse_qs(parsed.query)
+    user = query_params.get("user", [None])[0]
+    password = query_params.get("password", [None])[0]
+
+    return JdbcParameters(
+        protocol=protocol_prefix + parsed.scheme,
+        host=parsed.hostname,
+        port=parsed.port,
+        database=database,
+        user=user,
+        password=password,
+    )