feat(spider-py): Add storage functions to get job status and results.

.github/workflows/code-linting-checks.yaml

@@ -44,6 +44,7 @@ jobs:
           python --version
           tar --version
           task --version
+          uv --version
 
       - name: "Install project dependencies "
         timeout-minutes: 10

.github/workflows/unit-tests.yaml

@@ -47,6 +47,7 @@ jobs:
           python --version
           tar --version
           task --version
+          uv --version
 
       - name: "Install project dependencies "
         timeout-minutes: 10
@@ -55,3 +56,5 @@ jobs:
         run: "task deps:lib_install"
 
       - run: "task test:non-storage-unit-tests"
+
+      - run: "task test:python-non-storage-tests"

README.md

@@ -8,6 +8,7 @@ Follow the steps below to develop and contribute to the project.
 ## Requirements
 * Python 3.10 or higher
 * [Task] 3.40.0 or higher
+* [uv] 0.7.0 or higher
 
 ## Set up
 Run dependency installation task:
@@ -67,3 +68,4 @@ in the table below.
 
 [spider-docs]: https://docs.yscope.com/spider/main/
 [Task]: https://taskfile.dev
+[uv]: https://docs.astral.sh/uv/

build-tasks.yaml

@@ -19,3 +19,7 @@ tasks:
     deps: [":config-cmake-project"]
     cmds:
       - "cmake --build {{.G_BUILD_SPIDER_DIR}} --target clean --parallel {{numCPU}}"
+
+  py-build:
+    cmds:
+      - "uv build --directory {{.G_SRC_PYTHON_DIR}} -o {{.G_BUILD_PYTHON_DIR}}"

docs/src/dev-docs/testing.md

@@ -30,11 +30,14 @@ require this storage backend.
 
 You can use the following tasks to run the set of unit tests that's appropriate.
 
-| Task                          | Description                                                       |
-|-------------------------------|-------------------------------------------------------------------|
-| `test:all`                    | Runs all unit tests.                                              |
-| `test:non-storage-unit-tests` | Runs all unit tests which don't require a storage backend to run. |
-| `test:storage-unit-tests`     | Runs all unit tests which require a storage backend to run.       |
+| Task                            | Description                                                           |
+|---------------------------------|-----------------------------------------------------------------------|
+| `test:cpp-all`                  | Runs all C++ unit tests.                                              |
+| `test:non-storage-unit-tests`   | Runs all C++ unit tests which don't require a storage backend to run. |
+| `test:storage-unit-tests`       | Runs all C++ unit tests which require a storage backend to run.       |
+| `test:python-tests`             | Runs all Python tests.                                                |
+| `test:python-non-storage-tests` | Runs all Python tests which don't require a storage backend to run.   |
+| `test:python-storage-tests`     | Runs all Python tests which require a storage backend to run.         |
 
 If any tests show error messages for the connection function below, revisit the
 [setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly.
@@ -52,9 +55,9 @@ pull requests, and daily. Currently, it only runs unit tests that don't require
 
 You can use the following tasks to run integration tests.
 
-| Task                          | Description                                                       |
-|-------------------------------|-------------------------------------------------------------------|
-| `test:integration`            | Runs all integration tests.                                       |
+| Task                   | Description                     |
+|------------------------|---------------------------------|
+| `test:cpp-integration` | Runs all C++ integration tests. |
 
 
 [gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml

lint-tasks.yaml

@@ -136,6 +136,11 @@ tasks:
           mypy "{{.ITEM}}"
           ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}"
           ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}"
+      - cmd: |-
+          cd "{{.G_SRC_PYTHON_DIR}}"
+          uv run mypy .
+          uv run ruff check {{.RUFF_CHECK_FLAGS}}
+          uv run ruff format {{.RUFF_FORMAT_FLAGS}}
 
   yml:
     aliases:

python/.python-version

@@ -0,0 +1 @@
+3.13

python/pyproject.toml

@@ -0,0 +1,89 @@
+[project]
+name = "spider"
+version = "0.0.1"
+description = "Spider is a distributed task execution framework"
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+  "lark>=1.2.2",
+  "mariadb>=1.1.13",
+  "msgpack-types>=0.5.0",
+]
+
+[project.scripts]
+spider_task_executor = "spider.task_executor.task_executor:main"
+
+[dependency-groups]
+dev = [
+  "mypy>=1.12.0",
+  "pytest>=8.4.1",
+  "ruff>=0.4.4",
+]
+
+[build-system]
+requires = ["hatchling>=1.18.0"]
+build-backend = "hatchling.build"
+
+[tool.mypy]
+strict = true
+# Additional output
+pretty = true
+show_error_code_links = true
+show_error_context = true
+
+[[tool.mypy.overrides]]
+module = ["mariadb.*"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+markers = [
+  "integration: mark test as an integration test",
+  "storage: mark unit tests needing storage",
+]
+testpaths = ["tests"]
+
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = 100
+
+[tool.ruff.lint]
+select = ["ALL"]
+extend-select = ["PT"]
+ignore = [
+  "COM812",  # Redundant and conflicts with ruff format
+  "D203",  # No blank line before docstrings (D211)
+  "D205",  # Breaks if summary is larger than one line due to wrapping or if no summary exists
+  "D212",  # Enforce docstring summary line on the next line after quotes (D213)
+  "D400",  # First line of docstrings may not end in period
+  "D401",  # Docstrings should be written in present tense (not imperative)
+  "D415",  # First line of docstrings may not end in a period, question mark, or exclamation point
+  "FA102",  # Allow use of PEP 604 union in type annotations
+  "FBT",  # Allow bool positional parameters since other value positions are allowed
+  "FIX002",  # Allow todo statements
+  "PERF401",  # Allow for loops when creating lists
+  "PERF403",  # Allow for loops when creating dicts
+  "S311",  # Allow usage of `random` package
+  "SIM102",  # Allow collapsible if statements for readability
+  "TD002",  # Author unnecessary for todo statement
+  "TD003",  # Issue link unnecessary for todo statement
+  "UP015",  # Explicit open modes are helpful
+]
+isort.order-by-type = false
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = [
+  "INP001",  # Allow implicit namespace package for tests
+  "PLR2004",  # Allow use of magic value
+  "S101",  # Allow use of `assert` (security warning)
+  "S603",  # Allow use of `subprocess.Popen` (security warning)
+  "T201",  # Allow use of `print` (testing)
+]
+
+[tool.ruff.lint.flake8-self]
+ignore-names = ["_impl"]
+
+[tool.ruff.lint.pydocstyle]
+ignore-decorators = ["typing.override"]

python/src/spider/__init__.py

@@ -0,0 +1,19 @@
+"""Spider package root."""
+
+from spider.client import chain, Data, Driver, group, TaskContext, TaskGraph
+from spider.type import Double, Float, Int8, Int16, Int32, Int64
+
+__all__ = [
+    "Data",
+    "Double",
+    "Driver",
+    "Float",
+    "Int8",
+    "Int16",
+    "Int32",
+    "Int64",
+    "TaskContext",
+    "TaskGraph",
+    "chain",
+    "group",
+]

python/src/spider/client/__init__.py

@@ -0,0 +1,15 @@
+"""Spider python client."""
+
+from .data import Data
+from .driver import Driver
+from .task import TaskContext
+from .taskgraph import chain, group, TaskGraph
+
+__all__ = [
+    "Data",
+    "Driver",
+    "TaskContext",
+    "TaskGraph",
+    "chain",
+    "group",
+]

python/src/spider/client/data.py

@@ -0,0 +1,12 @@
+"""Spider client Data module."""
+
+from spider import core
+
+
+class Data:
+    """Represents a spider client data."""
+
+    def __init__(self, value: bytes) -> None:
+        """Initialize the Data object with the given value."""
+        self.data_id = core.DataId()
+        self.value = value

python/src/spider/client/driver.py

@@ -0,0 +1,65 @@
+"""Spider client driver module."""
+
+from collections.abc import Sequence
+from uuid import uuid4
+
+import msgpack
+
+from spider import core
+from spider.client.data import Data
+from spider.client.job import Job
+from spider.client.taskgraph import TaskGraph
+from spider.storage import MariaDBStorage, parse_jdbc_url
+from spider.type import to_tdl_type_str
+
+
+class Driver:
+    """Spider client driver class."""
+
+    def __init__(self, storage_url: str) -> None:
+        """
+        Creates a new Spider client driver and connects to the storage.
+        :param storage_url: The URL of the storage to connect to.
+        :raises StorageError: If the storage cannot be connected to.
+        """
+        self.driver_id = uuid4()
+        self.storage = MariaDBStorage(parse_jdbc_url(storage_url))
+
+    def submit_jobs(
+        self, graphs: Sequence[TaskGraph], args: Sequence[Sequence[object]]
+    ) -> Sequence[Job]:
+        """
+        Submits a list of jobs to the storage.
+        :param graphs: The list of task graphs to submit.
+        :param args: The arguments for each job.
+        :return: A sequence of Job objects representing the submitted jobs.
+        :raises StorageError: If the jobs cannot be submitted to the storage.
+        :raises ValueError: If the number of job inputs does not match the number of arguments.
+        :raises TypeError: If the arguments are not of the expected type.
+        """
+        msg = "Number of job inputs does not match number of arguments"
+        if len(graphs) != len(args):
+            raise ValueError(msg)
+
+        if not graphs:
+            return []
+        for task_graph, task_args in zip(graphs, args, strict=True):
+            arg_index = 0
+            for task_id in task_graph._impl.input_tasks:
+                task = task_graph._impl.tasks[task_id]
+                for task_input in task.task_inputs:
+                    if arg_index >= len(task_args):
+                        raise ValueError(msg)
+                    arg = task_args[arg_index]
+                    if isinstance(arg, Data):
+                        task_input.type = to_tdl_type_str(Data)
+                        task_input.value = arg.data_id
+                    else:
+                        task_input.type = to_tdl_type_str(type(arg))
+                        task_input.value = core.TaskInputValue(msgpack.packb(arg))
+                    arg_index += 1
+            if arg_index != len(task_args):
+                raise ValueError(msg)
+
+        jobs = self.storage.submit_jobs(self.driver_id, [graph._impl for graph in graphs])
+        return [Job(job, self.storage) for job in jobs]

python/src/spider/client/job.py

@@ -0,0 +1,17 @@
+"""Spider job module."""
+
+from spider import core
+from spider.storage import Storage
+
+
+class Job:
+    """Represents Spider job."""
+
+    def __init__(self, job: core.Job, storage: Storage) -> None:
+        """
+        Creates a new Spider job.
+        :param job: Core job object.
+        :param storage: The storage backend.
+        """
+        self._impl = job
+        self.storage = storage