diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 48f8eacd7..f8dd495ae 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -44,6 +44,7 @@ jobs: python --version tar --version task --version + uv --version - name: "Install project dependencies " timeout-minutes: 10 diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 1ced6a889..2b8b0754c 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -47,6 +47,7 @@ jobs: python --version tar --version task --version + uv --version - name: "Install project dependencies " timeout-minutes: 10 @@ -55,3 +56,5 @@ jobs: run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" + + - run: "task test:python-non-storage-tests" diff --git a/README.md b/README.md index 8b96a86a7..11aec22fd 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Follow the steps below to develop and contribute to the project. ## Requirements * Python 3.10 or higher * [Task] 3.40.0 or higher +* [uv] 0.7.0 or higher ## Set up Run dependency installation task: @@ -67,3 +68,4 @@ in the table below. [spider-docs]: https://docs.yscope.com/spider/main/ [Task]: https://taskfile.dev +[uv]: https://docs.astral.sh/uv/ \ No newline at end of file diff --git a/build-tasks.yaml b/build-tasks.yaml index 53828027c..1ce1efb9b 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -19,3 +19,7 @@ tasks: deps: [":config-cmake-project"] cmds: - "cmake --build {{.G_BUILD_SPIDER_DIR}} --target clean --parallel {{numCPU}}" + + py-build: + cmds: + - "uv build --directory {{.G_SRC_PYTHON_DIR}} -o {{.G_BUILD_PYTHON_DIR}}" diff --git a/docs/src/dev-docs/testing.md b/docs/src/dev-docs/testing.md index 873ef7b67..e2bd98501 100644 --- a/docs/src/dev-docs/testing.md +++ b/docs/src/dev-docs/testing.md @@ -30,11 +30,14 @@ require this storage backend. You can use the following tasks to run the set of unit tests that's appropriate. -| Task | Description | -|-------------------------------|-------------------------------------------------------------------| -| `test:all` | Runs all unit tests. | -| `test:non-storage-unit-tests` | Runs all unit tests which don't require a storage backend to run. | -| `test:storage-unit-tests` | Runs all unit tests which require a storage backend to run. | +| Task | Description | +|---------------------------------|-----------------------------------------------------------------------| +| `test:cpp-all` | Runs all C++ unit tests. | +| `test:non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. | +| `test:storage-unit-tests` | Runs all C++ unit tests which require a storage backend to run. | +| `test:python-tests` | Runs all Python tests. | +| `test:python-non-storage-tests` | Runs all Python tests which don't require a storage backend to run. | +| `test:python-storage-tests` | Runs all Python tests which require a storage backend to run. | If any tests show error messages for the connection function below, revisit the [setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly. @@ -52,9 +55,9 @@ pull requests, and daily. Currently, it only runs unit tests that don't require You can use the following tasks to run integration tests. -| Task | Description | -|-------------------------------|-------------------------------------------------------------------| -| `test:integration` | Runs all integration tests. | +| Task | Description | +|------------------------|---------------------------------| +| `test:cpp-integration` | Runs all C++ integration tests. | [gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 881101f80..205317b4c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -136,6 +136,11 @@ tasks: mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" + - cmd: |- + cd "{{.G_SRC_PYTHON_DIR}}" + uv run mypy . + uv run ruff check {{.RUFF_CHECK_FLAGS}} + uv run ruff format {{.RUFF_FORMAT_FLAGS}} yml: aliases: diff --git a/python/.python-version b/python/.python-version new file mode 100644 index 000000000..24ee5b1be --- /dev/null +++ b/python/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 000000000..8c846c944 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,89 @@ +[project] +name = "spider" +version = "0.0.1" +description = "Spider is a distributed task execution framework" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "lark>=1.2.2", + "mariadb>=1.1.13", + "msgpack-types>=0.5.0", +] + +[project.scripts] +spider_task_executor = "spider.task_executor.task_executor:main" + +[dependency-groups] +dev = [ + "mypy>=1.12.0", + "pytest>=8.4.1", + "ruff>=0.4.4", +] + +[build-system] +requires = ["hatchling>=1.18.0"] +build-backend = "hatchling.build" + +[tool.mypy] +strict = true +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true + +[[tool.mypy.overrides]] +module = ["mariadb.*"] +ignore_missing_imports = true + +[tool.pytest.ini_options] +markers = [ + "integration: mark test as an integration test", + "storage: mark unit tests needing storage", +] +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 + +[tool.ruff.lint] +select = ["ALL"] +extend-select = ["PT"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "INP001", # Allow implicit namespace package for tests + "PLR2004", # Allow use of magic value + "S101", # Allow use of `assert` (security warning) + "S603", # Allow use of `subprocess.Popen` (security warning) + "T201", # Allow use of `print` (testing) +] + +[tool.ruff.lint.flake8-self] +ignore-names = ["_from_impl", "_impl"] + +[tool.ruff.lint.pydocstyle] +ignore-decorators = ["typing.override"] diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py new file mode 100644 index 000000000..d5264f86c --- /dev/null +++ b/python/src/spider/__init__.py @@ -0,0 +1,22 @@ +"""Spider package root.""" + +from spider.client import chain, Data, Driver, group, Job, TaskContext, TaskGraph +from spider.core import JobStatus +from spider.type import Double, Float, Int8, Int16, Int32, Int64 + +__all__ = [ + "Data", + "Double", + "Driver", + "Float", + "Int8", + "Int16", + "Int32", + "Int64", + "Job", + "JobStatus", + "TaskContext", + "TaskGraph", + "chain", + "group", +] diff --git a/python/src/spider/client/__init__.py b/python/src/spider/client/__init__.py new file mode 100644 index 000000000..b0405dcdc --- /dev/null +++ b/python/src/spider/client/__init__.py @@ -0,0 +1,17 @@ +"""Spider python client.""" + +from .data import Data +from .driver import Driver +from .job import Job +from .task_context import TaskContext +from .taskgraph import chain, group, TaskGraph + +__all__ = [ + "Data", + "Driver", + "Job", + "TaskContext", + "TaskGraph", + "chain", + "group", +] diff --git a/python/src/spider/client/data.py b/python/src/spider/client/data.py new file mode 100644 index 000000000..bf274c661 --- /dev/null +++ b/python/src/spider/client/data.py @@ -0,0 +1,43 @@ +"""Spider client Data module.""" + +from uuid import uuid4 + +from spider import core + + +class Data: + """Represents a spider client data.""" + + def __init__(self, value: bytes) -> None: + """Initialize the Data object with the given value.""" + self._impl = core.Data(uuid4(), value) + + @staticmethod + def _from_impl(impl: core.Data) -> "Data": + """Creates a Data instance from an existing core.Data implementation.""" + data = Data(impl.value) + data._impl = impl + return data + + @property + def value(self) -> bytes: + """Property to get the value of the data.""" + return self._impl.value + + @property + def hard_locality(self) -> bool: + """Property to check if the data has hard locality.""" + return self._impl.hard_locality + + @hard_locality.setter + def hard_locality(self, value: bool) -> None: + """Sets the hard locality for the data.""" + self._impl.hard_locality = value + + def get_localities(self) -> list[str]: + """Gets the list of localities where the data is stored.""" + return [locality.address for locality in self._impl.localities] + + def add_locality(self, address: str) -> None: + """Adds a new locality to the data.""" + self._impl.localities.append(core.DataLocality(address)) diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py new file mode 100644 index 000000000..95731f185 --- /dev/null +++ b/python/src/spider/client/driver.py @@ -0,0 +1,74 @@ +"""Spider client driver module.""" + +from collections.abc import Sequence +from uuid import uuid4 + +import msgpack + +from spider import core +from spider.client.data import Data +from spider.client.job import Job +from spider.client.taskgraph import TaskGraph +from spider.storage import MariaDBStorage, parse_jdbc_url +from spider.type import to_tdl_type_str + + +class Driver: + """Spider client driver class.""" + + def __init__(self, storage_url: str) -> None: + """ + Creates a new Spider client driver and connects to the storage. + :param storage_url: The URL of the storage to connect to. + :raises StorageError: If the storage cannot be connected to. + """ + self._driver_id = uuid4() + self._storage = MariaDBStorage(parse_jdbc_url(storage_url)) + self._storage.create_driver(self._driver_id) + + def submit_jobs( + self, graphs: Sequence[TaskGraph], args: Sequence[Sequence[object]] + ) -> Sequence[Job]: + """ + Submits a list of jobs to the storage. + :param graphs: The list of task graphs to submit. + :param args: The arguments for each job. + :return: A sequence of Job objects representing the submitted jobs. + :raises StorageError: If the jobs cannot be submitted to the storage. + :raises ValueError: If the number of job inputs does not match the number of arguments. + :raises TypeError: If the arguments are not of the expected type. + """ + msg = "Number of job inputs does not match number of arguments" + if len(graphs) != len(args): + raise ValueError(msg) + + if not graphs: + return [] + for task_graph, task_args in zip(graphs, args, strict=True): + arg_index = 0 + for task_id in task_graph._impl.input_tasks: + task = task_graph._impl.tasks[task_id] + for task_input in task.task_inputs: + if arg_index >= len(task_args): + raise ValueError(msg) + arg = task_args[arg_index] + if isinstance(arg, Data): + task_input.type = to_tdl_type_str(Data) + task_input.value = arg._impl.id + else: + task_input.type = to_tdl_type_str(type(arg)) + task_input.value = core.TaskInputValue(msgpack.packb(arg)) + arg_index += 1 + if arg_index != len(task_args): + raise ValueError(msg) + + jobs = self._storage.submit_jobs(self._driver_id, [graph._impl for graph in graphs]) + return [Job(job, self._storage) for job in jobs] + + def create_data(self, data: Data) -> None: + """ + Registers a Data object in the storage. + :param data: The Data object to register. + :raises StorageError: If the Data object cannot be registered in the storage. + """ + self._storage.create_driver_data(self._driver_id, data._impl) diff --git a/python/src/spider/client/job.py b/python/src/spider/client/job.py new file mode 100644 index 000000000..2c430ff48 --- /dev/null +++ b/python/src/spider/client/job.py @@ -0,0 +1,72 @@ +"""Spider job module.""" + +import msgpack + +from spider import core +from spider.client.data import Data +from spider.storage import Storage, StorageError +from spider.type import parse_tdl_type +from spider.utils import msgpack_decoder + + +def _convert_outputs(outputs: list[core.TaskOutput]) -> tuple[object, ...]: + """ + Converts a list of TaskOutput objects to a tuple of their values. + :param outputs: The list of TaskOutput objects. + :return: A tuple containing the values of the TaskOutput objects. + :raises msgpack.exceptions.UnpackException: If there was an error deserializing the TaskOutput + values. + :raises StorageError: If there was an error in the TaskOutput values. + """ + results = [] + for output in outputs: + if isinstance(output.value, core.TaskOutputValue): + cls = parse_tdl_type(output.type).native_type() + unpacked = msgpack.unpackb(output.value, raw=False, strict_map_key=False) + results.append(msgpack_decoder(cls, unpacked)) + elif isinstance(output.value, core.Data): + results.append(Data._from_impl(output.value)) + else: + msg = "Fail to get data from storage." + raise StorageError(msg) + return tuple(results) + + +class Job: + """Represents Spider job.""" + + def __init__(self, job: core.Job, storage: Storage) -> None: + """ + Creates a new Spider job. + :param job: Core job object. + :param storage: The storage backend. + """ + self._impl = job + self._storage = storage + + def get_status(self) -> core.JobStatus: + """ + :return: The current job status. + :raises StorageError: If there was an error retrieving the job status from storage. + """ + if self._impl.status != core.JobStatus.Running: + return self._impl.status + + status = self._storage.get_job_status(self._impl) + self._impl.status = status + return status + + def get_results(self) -> object | None: + """ + :return: The job results or None if the status is not Running. + :raises StorageError: If there was an error retrieving the job results from storage. + :raises msgpack.exceptions.UnpackException: If there was an error deserializing the job + results. + """ + if self._impl.results is not None: + return _convert_outputs(self._impl.results) + + results = self._storage.get_job_results(self._impl) + if results is None: + return None + return _convert_outputs(results) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py new file mode 100644 index 000000000..ecc450648 --- /dev/null +++ b/python/src/spider/client/task.py @@ -0,0 +1,95 @@ +"""Spider client task module.""" + +import inspect +from collections.abc import Callable +from types import FunctionType, GenericAlias +from typing import get_args, get_origin + +from spider import core +from spider.client.data import Data +from spider.client.task_context import TaskContext +from spider.core import TaskInput, TaskOutput, TaskOutputValue +from spider.type import to_tdl_type_str + +# Check the TaskFunction signature at runtime. +# Enforcing static check for first argument requires the use of Protocol. However, functions, which +# are Callable, are not considered a Protocol without explicit cast. +TaskFunction = Callable[..., object] + + +def is_tuple(t: type | GenericAlias) -> bool: + """ + :param t: + :return: Whether t is a tuple. + """ + return get_origin(t) is tuple + + +def _process_parameters(task: core.Task, signature: inspect.Signature) -> None: + """ + Checks the parameters validity and add them to the task. + :param task: + :param signature: + :raises TypeError: If the parameters are invalid. + """ + params = list(signature.parameters.values()) + if not params or params[0].annotation is not TaskContext: + msg = "First argument is not a TaskContext." + raise TypeError(msg) + for param in params[1:]: + if param.kind in {inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD}: + msg = "Variadic parameters are not supported." + raise TypeError(msg) + if param.annotation is inspect.Parameter.empty: + msg = "Argument must have type annotation" + raise TypeError(msg) + tdl_type_str = to_tdl_type_str(param.annotation) + task.task_inputs.append(TaskInput(tdl_type_str, None)) + + +def _process_return(task: core.Task, signature: inspect.Signature) -> None: + """ + Checks the return type validity and add them to the task. + :param task: + :param signature: + :raises TypeError: If the return type is invalid. + """ + returns = signature.return_annotation + if returns is inspect.Parameter.empty: + msg = "Return type must have type annotation" + raise TypeError(msg) + if is_tuple(returns): + args = get_args(returns) + if Ellipsis in args: + msg = "Variable-length tuple return types are not supported." + raise TypeError(msg) + for r in args: + tdl_type_str = to_tdl_type_str(r) + if r is Data: + task.task_outputs.append(TaskOutput(tdl_type_str, core.DataId())) + else: + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + else: + tdl_type_str = to_tdl_type_str(returns) + if returns is Data: + task.task_outputs.append(TaskOutput(tdl_type_str, core.DataId())) + else: + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + + +def _create_task(func: TaskFunction) -> core.Task: + """ + Creates a core Task object from the task function. + :param func: + :return: + :raise TypeError: If the function signature contains unsupported types. + """ + task = core.Task() + if not isinstance(func, FunctionType): + msg = "`func` is not a function." + raise TypeError(msg) + task.function_name = func.__qualname__ + signature = inspect.signature(func) + _process_parameters(task, signature) + _process_return(task, signature) + return task diff --git a/python/src/spider/client/task_context.py b/python/src/spider/client/task_context.py new file mode 100644 index 000000000..cf1aaf58f --- /dev/null +++ b/python/src/spider/client/task_context.py @@ -0,0 +1,26 @@ +"""Spider client task context module.""" + +from spider import core, storage +from spider.client.data import Data + + +class TaskContext: + """Spider task context provides access to the task id and data creation for tasks.""" + + def __init__(self, task_id: core.TaskId, storage: storage.Storage) -> None: + """Initializes the task context.""" + self._task_id = task_id + self._storage = storage + + @property + def task_id(self) -> core.TaskId: + """:return: The task id.""" + return self._task_id + + def create_data(self, data: Data) -> None: + """ + Creates a new data object in the storage associated with the task. + :param data: The data object to be created. + :raises StorageError: If there is an error during storage operation. + """ + self._storage.create_task_data(self._task_id, data._impl) diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py new file mode 100644 index 000000000..bd3c43c24 --- /dev/null +++ b/python/src/spider/client/taskgraph.py @@ -0,0 +1,63 @@ +"""Spider client TaskGraph module.""" + +from collections.abc import Sequence + +from spider import core +from spider.client.task import _create_task, TaskFunction + + +class TaskGraph: + """ + Spider client TaskGraph class. + Wraps around the core TaskGraph class. + """ + + def __init__(self) -> None: + """Initialize TaskGraph.""" + self._impl = core.TaskGraph() + + def chain_graph(self, child: "TaskGraph") -> "TaskGraph": + """ + Chains another task graph with this task graph. + :param child: The task graph to be chained as child. + :return: The chained task graph. + :raise TypeError: If the outputs and the inputs of `graph` do not match. + """ + graph = TaskGraph() + graph._impl = self._impl.chain_graph(child._impl) + return graph + + +def group(tasks: Sequence[TaskFunction | TaskGraph]) -> TaskGraph: + """ + Groups task functions and task graph into a single task graph. + :param tasks: List of task functions or task graphs. + :return: The new task graph. + """ + graph = TaskGraph() + for task in tasks: + if callable(task): + graph._impl.add_task(_create_task(task)) + else: + graph._impl.merge_graph(task._impl) + + return graph + + +def chain(parent: TaskFunction | TaskGraph, child: TaskFunction | TaskGraph) -> TaskGraph: + """ + Chains two task functions or task graphs into a single task graph. + :param parent: + :param child: + :return: + :raises TypeError: If the parent outputs and child inputs do not match. + """ + if callable(parent): + task = _create_task(parent) + parent = TaskGraph() + parent._impl.add_task(task) + if callable(child): + task = _create_task(child) + child = TaskGraph() + child._impl.add_task(task) + return parent.chain_graph(child) diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py new file mode 100644 index 000000000..f4fbd3a6b --- /dev/null +++ b/python/src/spider/core/__init__.py @@ -0,0 +1,43 @@ +"""Spider core package.""" + +from .data import Data, DataId, DataLocality +from .driver import DriverId +from .job import Job, JobId, JobStatus +from .task import ( + get_state_from_str, + get_state_str, + Task, + TaskId, + TaskInput, + TaskInputData, + TaskInputOutput, + TaskInputValue, + TaskOutput, + TaskOutputData, + TaskOutputValue, + TaskState, +) +from .taskgraph import TaskGraph + +__all__ = [ + "Data", + "DataId", + "DataLocality", + "DriverId", + "Job", + "JobId", + "JobStatus", + "Task", + "TaskGraph", + "TaskId", + "TaskInput", + "TaskInputData", + "TaskInputOutput", + "TaskInputValue", + "TaskOutput", + "TaskOutputData", + "TaskOutputValue", + "TaskState", + "get_state_from_str", + "get_state_str", +] diff --git a/python/src/spider/core/data.py b/python/src/spider/core/data.py new file mode 100644 index 000000000..306afb2d5 --- /dev/null +++ b/python/src/spider/core/data.py @@ -0,0 +1,24 @@ +"""Data module for Spider.""" + +from dataclasses import dataclass, field +from uuid import UUID + +DataId = UUID + + +@dataclass +class DataLocality: + """Represents the locality of a data object.""" + + address: str + + +@dataclass +class Data: + """Represents a data object.""" + + id: DataId + value: bytes + localities: list[DataLocality] = field(default_factory=list) + hard_locality: bool = False + persisted: bool = False diff --git a/python/src/spider/core/driver.py b/python/src/spider/core/driver.py new file mode 100644 index 000000000..876c9b00b --- /dev/null +++ b/python/src/spider/core/driver.py @@ -0,0 +1,5 @@ +"""Spider core Driver module.""" + +from uuid import UUID + +DriverId = UUID diff --git a/python/src/spider/core/job.py b/python/src/spider/core/job.py new file mode 100644 index 000000000..851149e7f --- /dev/null +++ b/python/src/spider/core/job.py @@ -0,0 +1,34 @@ +"""Job module for Spider.""" + +from enum import IntEnum +from typing import TYPE_CHECKING +from uuid import UUID + +if TYPE_CHECKING: + from spider.core.task import TaskOutput + +JobId = UUID + + +class JobStatus(IntEnum): + """Job status.""" + + Running = 0 + Succeeded = 1 + Failed = 2 + Cancelled = 3 + + +class Job: + """Represents a submitted job.""" + + def __init__(self, job_id: JobId) -> None: + """ + Initializes a running job. + Status and results are cache for the data in storage. + Once a job completes, the status and results won't change. + :param job_id: + """ + self.job_id = job_id + self.status = JobStatus.Running + self.results: list[TaskOutput] | None = None diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py new file mode 100644 index 000000000..0cd31bb9c --- /dev/null +++ b/python/src/spider/core/task.py @@ -0,0 +1,100 @@ +"""Task module for Spider.""" + +from dataclasses import dataclass, field +from enum import IntEnum +from uuid import UUID, uuid4 + +from spider.core.data import Data, DataId + +TaskId = UUID + + +@dataclass +class TaskInputOutput: + """Represents a task input that points to output of another task""" + + task_id: TaskId + position: int + + +TaskInputValue = bytes +TaskInputData = Data | DataId + + +@dataclass +class TaskInput: + """Represents a task input""" + + type: str + value: TaskInputData | TaskInputOutput | TaskInputValue | None + + +TaskOutputValue = bytes +TaskOutputData = Data | DataId + + +@dataclass +class TaskOutput: + """Represents a task output""" + + type: str + value: TaskOutputData | TaskOutputValue + + +class TaskState(IntEnum): + """Represents state of a task""" + + Pending = 0 + Ready = 1 + Running = 2 + Succeeded = 3 + Failed = 4 + Cancelled = 5 + + +_StateStrMap = { + TaskState.Pending: "pending", + TaskState.Ready: "ready", + TaskState.Running: "running", + TaskState.Succeeded: "success", + TaskState.Failed: "fail", + TaskState.Cancelled: "cancel", +} + +_StrStateMap = {v: k for k, v in _StateStrMap.items()} + + +def get_state_str(state: TaskState) -> str: + """ + Returns string representation of task state. + :param state: The task state. + :return: The string representation of task state. + """ + return _StateStrMap[state] + + +def get_state_from_str(state_str: str) -> TaskState: + """ + Returns task state from string representation. + :param state_str: The string representation of task state. + :return: The task state from string representation. + :raises ValueError: If the state string is not recognized. + """ + state = _StrStateMap.get(state_str) + if state is not None: + return state + msg = f"Invalid task state string: {state_str}" + raise ValueError(msg) + + +@dataclass +class Task: + """Represents a task in Spider.""" + + task_id: TaskId = field(default_factory=uuid4) + function_name: str = "" + state: TaskState = TaskState.Pending + timeout: float = 0 + max_retries: int = 0 + task_inputs: list[TaskInput] = field(default_factory=list) + task_outputs: list[TaskOutput] = field(default_factory=list) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py new file mode 100644 index 000000000..e4d03ffe6 --- /dev/null +++ b/python/src/spider/core/taskgraph.py @@ -0,0 +1,148 @@ +"""TaskGraph module for Spider.""" + +from copy import deepcopy +from uuid import uuid4 + +from spider.core.task import Task, TaskId, TaskInputOutput + + +class TaskGraph: + """Represents a task graph in Spider.""" + + def __init__(self) -> None: + """Initializes an empty task graph.""" + self.tasks: dict[TaskId, Task] = {} + # Dependency list consists of a list of tuples of + # - parent task id + # - child task id + self.dependencies: list[tuple[TaskId, TaskId]] = [] + self.input_tasks: list[TaskId] = [] + self.output_tasks: list[TaskId] = [] + + def add_task( + self, task: Task, parents: list[TaskId] | None = None, children: list[TaskId] | None = None + ) -> None: + """ + Adds a task to the graph. + :param task: The task to add. + :param parents: The parent ids of the task. Must be already in the task graph. + :param children: The children ids of the task. Must be already in the task graph. + """ + self.tasks[task.task_id] = deepcopy(task) + if parents is not None and len(parents) > 0: + for parent in parents: + self.dependencies.append((parent, task.task_id)) + if parent in self.output_tasks: + self.output_tasks.remove(parent) + else: + self.input_tasks.append(task.task_id) + if children is not None and len(children) > 0: + for child in children: + self.dependencies.append((task.task_id, child)) + if child in self.input_tasks: + self.input_tasks.remove(child) + else: + self.output_tasks.append(task.task_id) + + def get_parents(self, task_id: TaskId) -> list[Task]: + """ + Gets parent tasks of task. + :param task_id: ID of the task. + :return: List of parent tasks. + """ + return [self.tasks[parent] for (parent, child) in self.dependencies if child == task_id] + + def get_children(self, task_id: TaskId) -> list[Task]: + """ + Gets child tasks of task. + :param task_id: ID of the task. + :return: List of children tasks. + """ + return [self.tasks[child] for (parent, child) in self.dependencies if parent == task_id] + + def reset_ids(self) -> None: + """Resets task ids.""" + id_map = {} + for task_id in self.tasks: + id_map[task_id] = uuid4() + + new_tasks = {} + for task_id in self.tasks: + new_task_id = id_map[task_id] + new_tasks[new_task_id] = deepcopy(self.tasks[task_id]) + new_tasks[new_task_id].task_id = new_task_id + for task_input in new_tasks[new_task_id].task_inputs: + if isinstance(task_input.value, TaskInputOutput): + task_input.value.task_id = id_map[task_input.value.task_id] + self.tasks = new_tasks + + new_dependencies = [] + for parent, child in self.dependencies: + new_dependencies.append((id_map[parent], id_map[child])) + self.dependencies = new_dependencies + + new_input_tasks = [] + for task_id in self.input_tasks: + new_input_tasks.append(id_map[task_id]) + self.input_tasks = new_input_tasks + + new_output_tasks = [] + for task_id in self.output_tasks: + new_output_tasks.append(id_map[task_id]) + self.output_tasks = new_output_tasks + + def merge_graph(self, graph: "TaskGraph") -> None: + """ + Merges another task graph into this task graph. + :param graph: The task graph to merge. + """ + new_graph = deepcopy(graph) + new_graph.reset_ids() + self.tasks.update(new_graph.tasks) + self.dependencies.extend(new_graph.dependencies) + self.input_tasks.extend(new_graph.input_tasks) + self.output_tasks.extend(new_graph.output_tasks) + + def chain_graph(self, child: "TaskGraph") -> "TaskGraph": + """ + Chains another task graph with this task graph. + :param child: The task graph to be chained as child. + :return: The chained task graph. + :raise TypeError: If the outputs and the inputs of `graph` do not match. + """ + graph = deepcopy(self) + graph.reset_ids() + parent_output_tasks = graph.output_tasks + graph.tasks.update(child.tasks) + graph.dependencies.extend(child.dependencies) + graph.output_tasks = deepcopy(child.output_tasks) + + size_mismatch_msg = "Parent outputs size and child inputs size do not match." + + task_index, output_position = 0, 0 + for task_id in child.input_tasks: + input_task = graph.tasks[task_id] + for i in range(len(input_task.task_inputs)): + if task_index >= len(parent_output_tasks): + raise TypeError(size_mismatch_msg) + output_task_id = parent_output_tasks[task_index] + + if (output_task_id, task_id) not in graph.dependencies: + graph.dependencies.append((output_task_id, task_id)) + + input_type = input_task.task_inputs[i].type + output_type = graph.tasks[output_task_id].task_outputs[output_position].type + if input_type != output_type: + msg = f"Output type {output_type} does not match input type {input_type}" + raise TypeError(msg) + input_task.task_inputs[i].value = TaskInputOutput(output_task_id, output_position) + output_position += 1 + if output_position >= len(graph.tasks[output_task_id].task_outputs): + output_position = 0 + task_index += 1 + + if task_index != len(parent_output_tasks) or output_position != 0: + raise TypeError(size_mismatch_msg) + + graph.reset_ids() + return graph diff --git a/python/src/spider/storage/__init__.py b/python/src/spider/storage/__init__.py new file mode 100644 index 000000000..b96d8dd12 --- /dev/null +++ b/python/src/spider/storage/__init__.py @@ -0,0 +1,13 @@ +"""Spider Storage package.""" + +from .jdbc_url import JdbcParameters, parse_jdbc_url +from .mariadb_storage import MariaDBStorage +from .storage import Storage, StorageError + +__all__ = [ + "JdbcParameters", + "MariaDBStorage", + "Storage", + "StorageError", + "parse_jdbc_url", +] diff --git a/python/src/spider/storage/jdbc_url.py b/python/src/spider/storage/jdbc_url.py new file mode 100644 index 000000000..c180077e4 --- /dev/null +++ b/python/src/spider/storage/jdbc_url.py @@ -0,0 +1,64 @@ +"""JDBC URL module.""" + +import re +from dataclasses import dataclass + + +@dataclass +class JdbcParameters: + """JDBC url parameters.""" + + protocol: str + host: str + database: str + port: int | None = None + user: str | None = None + password: str | None = None + + +pattern = re.compile( + r"^(?P[a-zA-Z][a-zA-Z0-9+.-]*(:[a-zA-Z0-9+.-]*)?)://" + r"(?P([a-zA-Z0-9.-]+|\d{1,3}(?:\.\d{1,3}){3}))" + r"(?::(?P\d+))?" + r"/(?P[a-zA-Z0-9_\-]+)" + r"(?:\?(?P[a-zA-Z0-9_\-=&]+))?" +) + + +def parse_jdbc_url(url: str) -> JdbcParameters: + """ + Parses a JDBC URL. + :param url: + :return: + :raises ValueError: If the JDBC URL is invalid. + """ + match = pattern.match(url) + if not match: + msg = f"Invalid JDBC URL: {url}" + raise ValueError(msg) + + groups = match.groupdict() + query = groups.get("query") or "" + query_params = dict(param.split("=", 1) for param in query.split("&") if "=" in param) + + protocol = groups.get("protocol") + if not protocol: + msg = f"Protocol is required in JDBC URL: {url}" + raise ValueError(msg) + host = groups.get("host") + if not host: + msg = f"Host is required in JDBC URL: {url}" + raise ValueError(msg) + database = groups.get("database") + if not database: + msg = f"Database is required in JDBC URL: {url}" + raise ValueError(msg) + + return JdbcParameters( + protocol=protocol, + host=host, + port=int(groups["port"]) if groups.get("port") else None, + database=database, + user=query_params.get("user"), + password=query_params.get("password"), + ) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py new file mode 100644 index 000000000..b5ea27c92 --- /dev/null +++ b/python/src/spider/storage/mariadb_storage.py @@ -0,0 +1,411 @@ +"""MariaDB Storage module.""" + +from collections.abc import Sequence +from uuid import UUID, uuid4 + +import mariadb +from typing_extensions import override + +from spider import core +from spider.core import get_state_str +from spider.storage.jdbc_url import JdbcParameters +from spider.storage.storage import Storage, StorageError + +InsertJob = """ +INSERT INTO + `jobs` (`id`, `client_id`) +VALUES + (?, ?)""" + +InsertTask = """ +INSERT INTO + `tasks` (`id`, `job_id`, `func_name`, `state`, `timeout`, `max_retry`) +VALUES + (?, ?, ?, ?, ?, ?)""" + +InsertTaskInputOutput = """ +INSERT INTO + `task_inputs` (`task_id`, `position`, `type`, `output_task_id`, `output_task_position`) +VALUES + (?, ?, ?, ?, ?)""" + +InsertTaskInputData = """ +INSERT INTO + `task_inputs` (`task_id`, `position`, `type`, `data_id`) +VALUES + (?, ?, ?, ?)""" + +InsertTaskInputValue = """ +INSERT INTO + `task_inputs` (`task_id`, `position`, `type`, `value`) +VALUES + (?, ?, ?, ?)""" + +InsertTaskOutput = """ +INSERT INTO + `task_outputs` (`task_id`, `position`, `type`) +VALUES + (?, ?, ?)""" + +InsertTaskDependency = """ +INSERT INTO + `task_dependencies` (parent, child) +VALUES + (?, ?)""" + +InsertInputTask = """ +INSERT INTO + `input_tasks` (`job_id`, `task_id`, `position`) +VALUES + (?, ?, ?)""" + +InsertOutputTask = """ +INSERT INTO + `output_tasks` (`job_id`, `task_id`, `position`) +VALUES + (?, ?, ?)""" + + +GetJobStatus = """ +SELECT + `state` +FROM + `jobs` +WHERE + `id` = ?""" + +GetOutputTasks = """ +SELECT + `task_id` +FROM + `output_tasks` +WHERE + `job_id` = ? +ORDER BY + `position`""" + +GetTaskOutputs = """ +SELECT + `type`, + `value`, + `data_id` +FROM + `task_outputs` +WHERE + `task_id` = ? +ORDER BY + `position`""" + +InsertData = """ +INSERT INTO + `data` (`id`, `value`, `hard_locality`) +VALUES + (?, ?, ?)""" + +InsertDataLocality = """ +INSERT INTO + `data_locality` (`id`, `address`) +VALUES + (?, ?)""" + +InsertDataRefDriver = """ +INSERT INTO + `data_ref_driver` (`id`, `driver_id`) +VALUES + (?, ?)""" + +InsertDataRefTask = """ +INSERT INTO + `data_ref_task` (`id`, `task_id`) +VALUES + (?, ?)""" + +GetData = """ +SELECT + `value`, + `hard_locality` +FROM + `data` +WHERE + `id` = ?""" + +GetDataLocality = """ +SELECT + `address` +FROM + `data_locality` +WHERE + `id` = ?""" + +InsertDriver = """ +INSERT INTO + `drivers` (`id`) +VALUES + (?)""" + + +class MariaDBStorage(Storage): + """MariaDB Storage class.""" + + def __init__(self, params: JdbcParameters) -> None: + """ + Connects to the MariaDB database. + :param params: The JDBC parameters for connecting to the database. + :raises StorageError: If the connection to the database fails. + """ + try: + self._conn = mariadb.connect( + host=params.host, + port=params.port, + user=params.user, + password=params.password, + database=params.database, + ) + except mariadb.Error as e: + raise StorageError(str(e)) from e + + @override + def submit_jobs( + self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] + ) -> Sequence[core.Job]: + if not task_graphs: + return [] + try: + job_ids = [uuid4() for _ in task_graphs] + with self._conn.cursor() as cursor: + cursor.executemany( + InsertJob, [(job_id.bytes, driver_id.bytes) for job_id in job_ids] + ) + cursor.executemany( + InsertTask, + [ + ( + task.task_id.bytes, + job_id.bytes, + task.function_name, + get_state_str(task.state), + task.timeout, + task.max_retries, + ) + for job_id, task_graph in zip(job_ids, task_graphs, strict=True) + for task in task_graph.tasks.values() + ], + ) + dep_params = [ + (parent.bytes, child.bytes) + for task_graph in task_graphs + for parent, child in task_graph.dependencies + ] + if dep_params: + cursor.executemany( + InsertTaskDependency, + dep_params, + ) + cursor.executemany( + InsertInputTask, + [ + (job_id.bytes, task_id.bytes, position) + for job_id, task_graph in zip(job_ids, task_graphs, strict=True) + for position, task_id in enumerate(task_graph.input_tasks) + ], + ) + cursor.executemany( + InsertOutputTask, + [ + (job_id.bytes, task_id.bytes, position) + for job_id, task_graph in zip(job_ids, task_graphs, strict=True) + for position, task_id in enumerate(task_graph.output_tasks) + ], + ) + cursor.executemany( + InsertTaskOutput, + [ + (task.task_id.bytes, position, task_output.type) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_output in enumerate(task.task_outputs) + ], + ) + input_data_params = [ + ( + task.task_id.bytes, + position, + task_input.type, + task_input.value.id.bytes + if isinstance(task_input.value, core.Data) + else task_input.value.bytes, + ) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputData) + ] + if input_data_params: + cursor.executemany( + InsertTaskInputData, + input_data_params, + ) + input_value_params = [ + (task.task_id.bytes, position, task_input.type, task_input.value) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputValue) + ] + if input_value_params: + cursor.executemany( + InsertTaskInputValue, + input_value_params, + ) + input_output_params = [ + ( + task.task_id.bytes, + position, + task_input.type, + task_input.value.task_id.bytes, + task_input.value.position, + ) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputOutput) + ] + if input_output_params: + cursor.executemany( + InsertTaskInputOutput, + input_output_params, + ) + self._conn.commit() + return [core.Job(job_id) for job_id in job_ids] + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e + + @override + def get_job_status(self, job: core.Job) -> core.JobStatus: + try: + with self._conn.cursor() as cursor: + cursor.execute(GetJobStatus, (job.job_id.bytes,)) + row = cursor.fetchone() + if row is None: + msg = f"No job found with id {job.job_id}" + raise StorageError(msg) + status_str = row[0] + match status_str: + case "running": + status = core.JobStatus.Running + case "success": + status = core.JobStatus.Succeeded + case "fail": + status = core.JobStatus.Failed + case "cancel": + status = core.JobStatus.Cancelled + case _: + msg = "Unknown job status" + raise StorageError(msg) + self._conn.commit() + return status + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e + + @override + def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: + try: + with self._conn.cursor() as cursor: + cursor.execute(GetOutputTasks, (job.job_id.bytes,)) + task_ids = [task_id for (task_id,) in cursor.fetchall()] + + results = [] + for task_id in task_ids: + cursor.execute(GetTaskOutputs, (task_id,)) + for output_type, value, data_id in cursor.fetchall(): + if value is not None: + results.append( + core.TaskOutput( + type=output_type, + value=core.TaskOutputValue(value), + ) + ) + elif data_id is not None: + data = self.get_data(core.DataId(data_id)) + results.append( + core.TaskOutput( + type=output_type, + value=data, + ) + ) + else: + self._conn.commit() + return None + self._conn.commit() + return results + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e + + def _create_data_with_ref(self, data: core.Data, insert_ref: str, ref_id: UUID) -> None: + """ + Inserts data object into the database with reference. + :param data: The data object to insert. + :param insert_ref: The SQL statement to insert the reference. + :param ref_id: The reference ID (DriverId or TaskId). + :raises StorageError: If the insertion fails. + """ + try: + with self._conn.cursor() as cursor: + cursor.execute( + InsertData, + (data.id.bytes, data.value, data.hard_locality), + ) + if data.localities: + cursor.executemany( + InsertDataLocality, + [(data.id.bytes, locality.address) for locality in data.localities], + ) + cursor.execute( + insert_ref, + (data.id.bytes, ref_id.bytes), + ) + self._conn.commit() + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e + + @override + def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: + self._create_data_with_ref(data, InsertDataRefDriver, driver_id) + + @override + def create_task_data(self, task_id: core.TaskId, data: core.Data) -> None: + self._create_data_with_ref(data, InsertDataRefTask, task_id) + + @override + def get_data(self, data_id: core.DataId) -> core.Data: + try: + with self._conn.cursor() as cursor: + cursor.execute(GetData, (data_id.bytes,)) + row = cursor.fetchone() + if row is None: + msg = f"No data found with id {data_id}" + raise StorageError(msg) + value, hard_locality = row + data = core.Data(id=data_id, value=value, hard_locality=hard_locality) + cursor.execute(GetDataLocality, (data_id.bytes,)) + for (address,) in cursor.fetchall(): + data.localities.append(core.DataLocality(address)) + self._conn.commit() + return data + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e + + @override + def create_driver(self, driver_id: core.DriverId) -> None: + try: + with self._conn.cursor() as cursor: + cursor.execute(InsertDriver, (driver_id.bytes,)) + self._conn.commit() + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py new file mode 100644 index 000000000..6cf852e12 --- /dev/null +++ b/python/src/spider/storage/storage.py @@ -0,0 +1,84 @@ +"""Storage backend interface module.""" + +from abc import ABC, abstractmethod +from collections.abc import Sequence + +from spider import core +from spider.core import JobStatus + + +class StorageError(Exception): + """Storage error.""" + + def __init__(self, message: str) -> None: + """Initializes storage error.""" + super().__init__(message) + + +class Storage(ABC): + """Storage backend interface.""" + + @abstractmethod + def submit_jobs( + self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] + ) -> Sequence[core.Job]: + """ + Submits jobs to the storage. + :param driver_id: Driver id. + :param task_graphs: Task graphs to submit. + :return: List of jobs representing the submitted jobs. + :raises StorageError: If the storage operations fail. + """ + + @abstractmethod + def get_job_status(self, job: core.Job) -> JobStatus: + """ + Gets the job status. This function does not set the `status` field in jobs. + :param job: + :return: + :raises StorageError: If the storage operations fail. + """ + + @abstractmethod + def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: + """ + Gets the job's results. This function does not set the `results` field in the job. + :param job: + :return: List of task outputs or None if the job has no results. + :raises StorageError: If the storage operations fail. + """ + + @abstractmethod + def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: + """ + Creates data from a driver in the storage. + :param driver_id: The driver id. + :param data: Data to create. + :raises StorageError: If the storage operations fail. + """ + + @abstractmethod + def create_task_data(self, task_id: core.TaskId, data: core.Data) -> None: + """ + Creates data from a task in the storage. + :param task_id: The task id. + :param data: Data to create. + :raises StorageError: If the storage operations fail. + """ + + @abstractmethod + def get_data(self, data_id: core.DataId) -> core.Data: + """ + Gets data from the storage. + :param data_id: + :return: The Data object associated with `data_id`. + :raises StorageError: If the storage operations fail. + """ + + @abstractmethod + def create_driver(self, driver_id: core.DriverId) -> None: + """ + Creates a driver in the storage. + :param driver_id: + :raises StorageError: If the storage operations fail. + """ diff --git a/python/src/spider/task_executor/__init__.py b/python/src/spider/task_executor/__init__.py new file mode 100644 index 000000000..03195404d --- /dev/null +++ b/python/src/spider/task_executor/__init__.py @@ -0,0 +1 @@ +"""Python task executor for running Python task function.""" diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py new file mode 100644 index 000000000..e4afc0db1 --- /dev/null +++ b/python/src/spider/task_executor/task_executor.py @@ -0,0 +1,154 @@ +"""Executes a Spider Python task.""" + +import argparse +import importlib +import inspect +import logging +from io import BufferedReader +from os import fdopen +from uuid import UUID + +import msgpack + +from spider import client, storage +from spider.task_executor.task_executor_message import get_request_body, TaskExecutorResponseType +from spider.utils import msgpack_decoder, msgpack_encoder + +# Set up logger +logger = logging.getLogger(__name__) + + +def parse_args() -> argparse.Namespace: + """ + Parses task executor arguments. + :return: Parsed arguments. + """ + parser = argparse.ArgumentParser() + parser.add_argument("--func", type=str, required=True) + parser.add_argument("--libs", nargs="+", type=str, required=True) + parser.add_argument("--storage_url", type=str, required=True) + parser.add_argument("--task_id", type=str, required=True) + parser.add_argument("--input-pipe", type=int, required=True) + parser.add_argument("--output-pipe", type=int, required=True) + return parser.parse_args() + + +HeaderSize = 16 + + +def receive_message(pipe: BufferedReader) -> bytes: + """ + Receives message from the pipe with a size header. + :param pipe: Pipe to receive message from. + :return: Received message body. + :raises IOError: If read from pipe fails. + :raises UnicodeDecodeError: If parsing header fails. + :raises EOFError: If the message body size does not match header size. + """ + body_size_str = pipe.read(HeaderSize).decode() + body_size = int(body_size_str, base=10) + body = pipe.read(body_size) + if len(body) != body_size: + msg = "Received message body size does not match header size." + raise EOFError(msg) + return body + + +def parse_arguments( + store: storage.Storage, params: list[inspect.Parameter], arguments: list[object] +) -> list[object]: + """ + Parses arguments for the function to be executed. + :param store: Storage instance to use to get Data. + :param params: List of parameters in the function signature. + :param arguments: List of arguments to parse. + :return: Parsed arguments. + :raises TypeError: If a parameter has no type annotation or if an argument cannot be parsed. + """ + parsed_args: list[object] = [] + for i, param in enumerate(params): + arg = arguments[i] + cls = param.annotation + if param.annotation is inspect.Parameter.empty: + msg = f"Parameter {param.name} has no type annotation." + raise TypeError(msg) + if cls is client.Data: + if not isinstance(arg, bytes): + msg = f"Argument {i} for spider.Data is not bytes." + raise TypeError(msg) + core_data = store.get_data(UUID(bytes=arg)) + parsed_args.append(client.Data._from_impl(core_data)) + else: + parsed_args.append(msgpack_decoder(cls, arg)) + return parsed_args + + +def parse_results(results: object) -> list[object]: + """ + Parses results from the function execution. + :param results: Results to parse. + :return: Parsed results. + :raises TypeError: If a result cannot be parsed. + """ + response_messages: list[object] = [TaskExecutorResponseType.Result] + if isinstance(results, tuple): + for result in results: + if isinstance(result, client.Data): + response_messages.append(result.value) + else: + response_messages.append(msgpack_encoder(result)) + return response_messages + + +def main() -> None: + """ + Main function to execute the task. + :raises StorageError: If a storage operation fails. + """ + # Parses arguments + args = parse_args() + func = args.func + task_id = args.task_id + task_id = UUID(task_id) + storage_url = args.storage_url + input_pipe = args.input_pipe + output_pipe = args.output_pipe + + logger.debug("Function to run: %s", func) + + # Sets up storage + store = storage.MariaDBStorage(storage_url) + + input_pipe = fdopen(input_pipe, "rb") + output_pipe = fdopen(output_pipe, "wb") + input_message = receive_message(input_pipe) + arguments = get_request_body(input_message) + logger.debug("Args buffer parsed") + + # Get the function to run + module_name, function_name = func.rsplit(".", 1) + module = importlib.import_module(module_name) + function = getattr(module, function_name) + logger.debug("Function %s imported from module %s", function_name, module_name) + + signature = inspect.signature(function) + if len(signature.parameters) != len(arguments) + 1: + msg = ( + f"Function {function_name} expects {len(signature.parameters) - 1} " + f"arguments, but {len(arguments)} were provided." + ) + raise ValueError(msg) + task_context = client.TaskContext(task_id, store) + arguments = [ + task_context, + *parse_arguments(store, list(signature.parameters.values()), arguments), + ] + results = function(*arguments) + logger.debug("Function %s executed", function_name) + + responses = parse_results(results) + output_pipe.write(msgpack.packb(responses)) + + +if __name__ == "__main__": + main() diff --git a/python/src/spider/task_executor/task_executor_message.py b/python/src/spider/task_executor/task_executor_message.py new file mode 100644 index 000000000..7838e1b47 --- /dev/null +++ b/python/src/spider/task_executor/task_executor_message.py @@ -0,0 +1,58 @@ +"""Task executor message module.""" + +from enum import IntEnum + +import msgpack + + +class TaskExecutorResponseType(IntEnum): + """Task executor response type.""" + + Unknown = 0 + Result = 1 + Error = 2 + Block = 3 + Ready = 4 + Cancel = 5 + + +class TaskExecutorRequestType(IntEnum): + """Task executor request type.""" + + Unknown = 0 + Arguments = 1 + Resume = 2 + + +class InvalidRequestTypeError(Exception): + """Exception raised for invalid request types.""" + + def __init__(self, message: str) -> None: + """Initializes the InvalidRequestTypeError with a message.""" + super().__init__(message) + + +MinArgRequestLength = 2 + + +def get_request_body(message: bytes) -> list[object]: + """ + Gets the request body from the request message. + :param message: The msgpack serialized request message. + :return: The request body as a list of objects. + :raises msgpack.exceptions.UnpackException: If the data is not a valid msgpack serialized list. + :raises TypeError: If the data is not a msgpack list or the list is too short. + :raises InvalidRequestTypeError: If the message header is not an `Arguments`. + """ + data = msgpack.unpackb(message) + if not isinstance(data, list): + msg = "Message is not a list." + raise TypeError(msg) + if len(data) < MinArgRequestLength: + msg = "Message is too short." + raise TypeError(msg) + message_header = int(data[0]) + if TaskExecutorRequestType.Arguments != message_header: + msg = f"Message header is not an `Arguments`: {message_header}" + raise InvalidRequestTypeError(msg) + return data[1:] diff --git a/python/src/spider/type/__init__.py b/python/src/spider/type/__init__.py new file mode 100644 index 000000000..fda34ee9b --- /dev/null +++ b/python/src/spider/type/__init__.py @@ -0,0 +1,16 @@ +"""Spider type package.""" + +from spider.type.tdl_convert import to_tdl_type_str +from spider.type.tdl_parse import parse_tdl_type +from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 + +__all__ = [ + "Double", + "Float", + "Int8", + "Int16", + "Int32", + "Int64", + "parse_tdl_type", + "to_tdl_type_str", +] diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py new file mode 100644 index 000000000..09a2f416d --- /dev/null +++ b/python/src/spider/type/tdl_convert.py @@ -0,0 +1,101 @@ +"""Converts native types to TDL types.""" + +from collections.abc import Collection +from types import GenericAlias +from typing import get_args, get_origin + +from spider.type.tdl_type import ( + BoolType, + ClassType, + DoubleType, + FloatType, + Int8Type, + Int16Type, + Int32Type, + Int64Type, + ListType, + MapType, + TdlType, +) +from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 +from spider.type.utils import get_class_name + + +def to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: + """ + Converts a native type to primitive TDL type. + :param native_type: + :return: Converted TDL primitive. None if `native_type` is not a supported primitive type. + """ + tdl_type: TdlType | None = None + if native_type is Int8: + tdl_type = Int8Type() + elif native_type is Int16: + tdl_type = Int16Type() + elif native_type is Int32: + tdl_type = Int32Type() + elif native_type is Int64: + tdl_type = Int64Type() + elif native_type is Float: + tdl_type = FloatType() + elif native_type is Double: + tdl_type = DoubleType() + elif native_type is bool: + tdl_type = BoolType() + return tdl_type + + +def to_tdl_type(native_type: type | GenericAlias) -> TdlType: + """ + Converts a Python type to TDL type. + :param native_type: + :return: + :raise: TypeError if `native_type` is not a valid TDL type. + """ + primitive_tdl_type = to_primitive_tdl_type(native_type) + if primitive_tdl_type is not None: + return primitive_tdl_type + + if native_type in (int, float, str, complex, bytes): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) + + if isinstance(native_type, GenericAlias): + origin = get_origin(native_type) + if origin is list: + arg = get_args(native_type) + if arg is None: + msg = "List does not have a key type." + raise TypeError(msg) + arg = arg[0] + return ListType(to_tdl_type(arg)) + + if origin is dict: + arg = get_args(native_type) + msg = "Dict does not have a key/value type." + if arg is None: + raise TypeError(msg) + if len(arg) != 2: # noqa: PLR2004 + raise TypeError(msg) + key = arg[0] + value = arg[1] + return MapType(to_tdl_type(key), to_tdl_type(value)) + + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) + + if issubclass(native_type, Collection): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) + + return ClassType(get_class_name(native_type)) + + +def to_tdl_type_str(native_type: type | GenericAlias) -> str: + """ + Converts a Python type to TDL type string. + :param native_type: + :return: + :raise: TypeError if `native_type` is not a valid TDL type. + """ + return to_tdl_type(native_type).type_str() diff --git a/python/src/spider/type/tdl_parse.py b/python/src/spider/type/tdl_parse.py new file mode 100644 index 000000000..eebbe55ce --- /dev/null +++ b/python/src/spider/type/tdl_parse.py @@ -0,0 +1,84 @@ +"""Parse TDL type string.""" + +from lark import Lark, LarkError, Token, Transformer, v_args + +from spider.type.tdl_type import ( + BoolType, + ClassType, + DoubleType, + FloatType, + Int8Type, + Int16Type, + Int32Type, + Int64Type, + ListType, + MapType, + TdlType, +) + +grammar = r""" +type: map_type | list_type | base_type + +map_type: "Map" "<" type "," type ">" +list_type: "List" "<" type ">" +base_type: ID + +ID: /[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*/ + +%import common.WS +%ignore WS +""" + +primitive_type_map = { + "bool": BoolType, + "double": DoubleType, + "float": FloatType, + "int8": Int8Type, + "int16": Int16Type, + "int32": Int32Type, + "int64": Int64Type, +} + + +class TypeTransformer(Transformer[Token, TdlType]): + """Transform Lark parse tree into TDL type.""" + + @v_args(inline=True) + def type(self, value: TdlType) -> TdlType: + """Unwraps the type node to return the TdlType.""" + return value + + @v_args(inline=True) + def map_type(self, key: TdlType, value: TdlType) -> TdlType: + """Transforms map node into Map type.""" + return MapType(key, value) + + @v_args(inline=True) + def list_type(self, key: TdlType) -> TdlType: + """Transforms list node into List type.""" + return ListType(key) + + def base_type(self, children: list[Token]) -> TdlType: + """Transforms primitive node into primitive type.""" + name = str(children[0]) + if name in primitive_type_map: + return primitive_type_map[name]() # type: ignore[abstract] + return ClassType(name) + + +parser = Lark(grammar, start="type", parser="lalr") + + +def parse_tdl_type(string: str) -> TdlType: + """ + Parses TDL type string into TDL type. + :param string: TDL type string. + :return: Parsed TDL type. + :raise: TypeError if string is not a valid TDL type. + """ + try: + tree = parser.parse(string) + return TypeTransformer(visit_tokens=False).transform(tree) + except LarkError as ecx: + msg = f"Cannot parse TDL type '{string}'" + raise TypeError(msg) from ecx diff --git a/python/src/spider/type/tdl_type.py b/python/src/spider/type/tdl_type.py new file mode 100644 index 000000000..cd0256b49 --- /dev/null +++ b/python/src/spider/type/tdl_type.py @@ -0,0 +1,182 @@ +"""Spider TDL types.""" + +from abc import ABC, abstractmethod +from types import GenericAlias + +from typing_extensions import override + +from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 +from spider.type.utils import get_class_by_name + + +class TdlType(ABC): + """Abstract base class for all TDL types.""" + + @abstractmethod + def type_str(self) -> str: + """:return: String representation of the TDL type.""" + + @abstractmethod + def native_type(self) -> type | GenericAlias: + """:return: Native Python type of the TDL type.""" + + +class DoubleType(TdlType): + """TDL double type.""" + + @override + def type_str(self) -> str: + return "double" + + @override + def native_type(self) -> type | GenericAlias: + return Double + + +class FloatType(TdlType): + """TDL float type.""" + + @override + def type_str(self) -> str: + return "float" + + @override + def native_type(self) -> type | GenericAlias: + return Float + + +class Int8Type(TdlType): + """TDL int8 type.""" + + @override + def type_str(self) -> str: + return "int8" + + @override + def native_type(self) -> type | GenericAlias: + return Int8 + + +class Int16Type(TdlType): + """TDL int16 type.""" + + @override + def type_str(self) -> str: + return "int16" + + @override + def native_type(self) -> type | GenericAlias: + return Int16 + + +class Int32Type(TdlType): + """TDL int32 type.""" + + @override + def type_str(self) -> str: + return "int32" + + @override + def native_type(self) -> type | GenericAlias: + return Int32 + + +class Int64Type(TdlType): + """TDL int64 type.""" + + @override + def type_str(self) -> str: + return "int64" + + @override + def native_type(self) -> type | GenericAlias: + return Int64 + + +class BoolType(TdlType): + """TDL bool type.""" + + @override + def type_str(self) -> str: + return "bool" + + @override + def native_type(self) -> type | GenericAlias: + return bool + + +class ClassType(TdlType): + """TDL Custom class type.""" + + def __init__(self, name: str) -> None: + """ + Creates a TDL custom class type. + :param name: The name of the class. + """ + self.name = name + + @override + def type_str(self) -> str: + return self.name + + @override + def native_type(self) -> type | GenericAlias: + """ + :return: Native Python type of the class. + :raise: TypeError if `class_name` is not a valid class. + """ + return get_class_by_name(self.name) + + +class ListType(TdlType): + """TDL List type.""" + + def __init__(self, key: TdlType) -> None: + """Creates a TDL list type.""" + self.key = key + + @override + def type_str(self) -> str: + return f"List<{self.key.type_str()}>" + + @override + def native_type(self) -> type | GenericAlias: + return list[self.key.native_type()] # type: ignore[misc] + + +def is_integral(tdl_type: TdlType) -> bool: + """:return: If TDL type is an integral type.""" + return isinstance(tdl_type, (Int8Type, Int16Type, Int32Type, Int64Type)) + + +def is_string(tdl_type: TdlType) -> bool: + """:return: If TDL type is a string type, i.e. List.""" + return isinstance(tdl_type, ListType) and isinstance(tdl_type.key, Int8Type) + + +def is_map_key(tdl_type: TdlType) -> bool: + """:return: If TDL type is a valid type for map key.""" + return is_integral(tdl_type) or is_string(tdl_type) + + +class MapType(TdlType): + """TDL Map type.""" + + def __init__(self, key: TdlType, value: TdlType) -> None: + """ + Creates a TDL map type. + :raises TypeError: If key is not a supported type. + """ + if not is_map_key(key): + msg = f"{key} is not a supported type for map key." + raise TypeError(msg) + self.key = key + self.value = value + + @override + def type_str(self) -> str: + return f"Map<{self.key.type_str()},{self.value.type_str()}>" + + @override + def native_type(self) -> type | GenericAlias: + return dict[self.key.native_type(), self.value.native_type()] # type: ignore[misc] diff --git a/python/src/spider/type/type.py b/python/src/spider/type/type.py new file mode 100644 index 000000000..903a9972c --- /dev/null +++ b/python/src/spider/type/type.py @@ -0,0 +1,66 @@ +"""Custom type module for Spider.""" + +from typing import cast + + +class BoundedInt(int): + """Bounded integer type.""" + + def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": + """Creates a bounded integer.""" + min_val = -(1 << (bits - 1)) + max_val = (1 << (bits - 1)) - 1 + + if not (min_val <= value <= max_val): + msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" + raise ValueError(msg) + + return super().__new__(cls, value) + + +class Int8(BoundedInt): + """8 bits integer type.""" + + def __new__(cls, value: int) -> "Int8": + """Creates an int8 integer.""" + return cast("Int8", super().__new__(cls, value, bits=8)) + + +class Int16(BoundedInt): + """16 bits integer type.""" + + def __new__(cls, value: int) -> "Int16": + """Creates an int16 integer.""" + return cast("Int16", super().__new__(cls, value, bits=16)) + + +class Int32(BoundedInt): + """32 bits integer type.""" + + def __new__(cls, value: int) -> "Int32": + """Creates an int32 integer.""" + return cast("Int32", super().__new__(cls, value, bits=32)) + + +class Int64(BoundedInt): + """64 bits integer type.""" + + def __new__(cls, value: int) -> "Int64": + """Creates an int64 integer.""" + return cast("Int64", super().__new__(cls, value, bits=64)) + + +class Float(float): + """Float type.""" + + def __new__(cls, value: float) -> "Float": + """Creates a float number.""" + return super().__new__(cls, value) + + +class Double(float): + """Double type.""" + + def __new__(cls, value: float) -> "Double": + """Creates a double number.""" + return super().__new__(cls, value) diff --git a/python/src/spider/type/utils.py b/python/src/spider/type/utils.py new file mode 100644 index 000000000..023ceed74 --- /dev/null +++ b/python/src/spider/type/utils.py @@ -0,0 +1,30 @@ +"""Utility for TDL types.""" + +from importlib import import_module +from typing import cast + + +def get_class_name(cls: type) -> str: + """ + :param cls: + :return: Full name of `cls`. + """ + return f"{cls.__module__}.{cls.__qualname__}" + + +def get_class_by_name(name: str) -> type: + """ + Gets class by name. + :param name: + :return: + :raise: TypeError if `class_name` is not a valid class. + """ + parts = name.split(".") + module_name = ".".join(parts[:-1]) + class_name = parts[-1] + try: + module = import_module(module_name) + return cast("type", getattr(module, class_name)) + except (ValueError, ModuleNotFoundError, AttributeError) as exc: + msg = f"{name} is not a valid TDL type." + raise TypeError(msg) from exc diff --git a/python/src/spider/utils/__init__.py b/python/src/spider/utils/__init__.py new file mode 100644 index 000000000..8d99c82a7 --- /dev/null +++ b/python/src/spider/utils/__init__.py @@ -0,0 +1,8 @@ +"""Spider utils package.""" + +from .msgpack_serde import msgpack_decoder, msgpack_encoder + +__all__ = [ + "msgpack_decoder", + "msgpack_encoder", +] diff --git a/python/src/spider/utils/msgpack_serde.py b/python/src/spider/utils/msgpack_serde.py new file mode 100644 index 000000000..b2b95f1d1 --- /dev/null +++ b/python/src/spider/utils/msgpack_serde.py @@ -0,0 +1,80 @@ +"""Serialization and deserialization using msgpack.""" + +from dataclasses import fields, is_dataclass +from types import GenericAlias +from typing import cast, get_args, get_origin + + +def msgpack_encoder(obj: object) -> list[object] | object: + """ + Encodes an object into a list of its field values. + This function recursively encodes nested dataclasses, lists, and dictionaries. + :param obj: Dataclass instance to serialize. + :return: List of field values. + """ + if is_dataclass(obj): + return [msgpack_encoder(getattr(obj, f.name)) for f in fields(obj)] + if isinstance(obj, list): + return [msgpack_encoder(item) for item in obj] + if isinstance(obj, dict): + return {msgpack_encoder(k): msgpack_encoder(v) for k, v in obj.items()} + return obj + + +def _decode_class(cls: type, data: object) -> object: + """ + Decodes data into an instance of a `cls`. + This function only works for non-container classes (not lists or dicts). + :param cls: Class to deserialize into. + :param data: Data to decode. + :return: Instance of `cls`. + :raise: TypeError if `data` is not compatible with `cls`. + """ + msg = f"Cannot create instance of {cls} with {data!r}" + if is_dataclass(cls): + if not isinstance(data, list): + raise TypeError(msg) + parameters = fields(cls) + if len(data) != len(parameters): + raise TypeError(msg) + args = [] + for param, value in zip(parameters, data, strict=True): + if not isinstance(param.type, type): + raise TypeError(msg) + args.append(msgpack_decoder(param.type, value)) + return cls(*args) + + return cls(data) + + +def msgpack_decoder(cls: type | GenericAlias, data: object) -> object: + """ + Decodes data into an instance of `cls`. + This function recursively decodes nested dataclasses, lists, and dictionaries. + :param cls: Class to deserialize into. + :param data: Data to decode. + :return: Instance of `cls`. + :raise: TypeError if `data` is not compatible with `cls`. + """ + msg = f"Cannot create instance of {cls} with {data!r}" + + origin = get_origin(cls) + if origin is None: + # If `cls` does not have an origin, it is a concrete type. + return _decode_class(cast("type", cls), data) + + if origin is list: + (key_type,) = get_args(cls) + if not isinstance(data, list): + raise TypeError(msg) + return [msgpack_decoder(key_type, item) for item in data] + + if origin is dict: + key_type, value_type = get_args(cls) + if not isinstance(data, dict): + raise TypeError(msg) + return { + msgpack_decoder(key_type, k): msgpack_decoder(value_type, v) for k, v in data.items() + } + + raise TypeError(msg) diff --git a/python/tests/client/test_data.py b/python/tests/client/test_data.py new file mode 100644 index 000000000..5ea7d9b43 --- /dev/null +++ b/python/tests/client/test_data.py @@ -0,0 +1,23 @@ +"""Tests for client data module.""" + +import pytest +from test_driver import driver # noqa: F401 + +from spider import Data, Driver + + +class TestData: + """Test data class for client data module.""" + + @pytest.mark.storage + def test_data(self, driver: Driver) -> None: # noqa: F811 + """Tests data creation.""" + data = Data(b"test_data") + data.hard_locality = True + data.add_locality("localhost") + + assert data.value == b"test_data" + assert data.hard_locality is True + assert data.get_localities() == ["localhost"] + + driver.create_data(data) diff --git a/python/tests/client/test_driver.py b/python/tests/client/test_driver.py new file mode 100644 index 000000000..99a42783f --- /dev/null +++ b/python/tests/client/test_driver.py @@ -0,0 +1,67 @@ +"""Tests for the driver module.""" + +import pytest + +from spider import chain, Driver, group, Int8, TaskContext + +MariaDBTestUrl = "jdbc:mariadb://127.0.0.1:3306/spider-storage?user=spider&password=password" + + +@pytest.fixture(scope="session") +def driver() -> Driver: + """Fixture for the driver.""" + return Driver(MariaDBTestUrl) + + +def double(_: TaskContext, x: Int8) -> Int8: + """Double a number.""" + return Int8(x * 2) + + +def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: + """Swaps two numbers.""" + return y, x + + +@pytest.mark.storage +class TestDriver: + """Test class for the driver module.""" + + def test_job_submission(self, driver: Driver) -> None: + """Tests successful job submission.""" + jobs = driver.submit_jobs( + [ + group([double]), + group([double, double]), + chain(group([double, double]), swap), + ], + [ + (Int8(1),), + (Int8(1), Int8(2)), + (Int8(1), Int8(2)), + ], + ) + assert len(jobs) == 3 + + def test_job_submission_fail(self, driver: Driver) -> None: + """Tests job submission failure.""" + with pytest.raises( + ValueError, match="Number of job inputs does not match number of arguments" + ): + driver.submit_jobs( + [ + group([double]), + ], + [ + (Int8(1), Int8(2)), + ], + ) + with pytest.raises(TypeError): + driver.submit_jobs( + [ + group([double]), + ], + [ + (1,), + ], + ) diff --git a/python/tests/client/test_job.py b/python/tests/client/test_job.py new file mode 100644 index 000000000..013e533e4 --- /dev/null +++ b/python/tests/client/test_job.py @@ -0,0 +1,30 @@ +"""Tests for client job module.""" + +import pytest +from test_driver import driver # noqa: F401 +from test_task_graph import double, swap + +from spider import chain, Driver, group, Int8, JobStatus + + +class TestJob: + """Tests for client job module.""" + + @pytest.mark.storage + def test_job(self, driver: Driver) -> None: # noqa: F811 + """Test getting running job status and results.""" + jobs = driver.submit_jobs( + [ + group([double, double]), + chain(group([double, double]), swap), + ], + [ + (Int8(1), Int8(2)), + (Int8(1), Int8(2)), + ], + ) + + assert jobs[0].get_status() == JobStatus.Running + assert jobs[1].get_status() == JobStatus.Running + assert jobs[0].get_results() is None + assert jobs[1].get_results() is None diff --git a/python/tests/client/test_task_graph.py b/python/tests/client/test_task_graph.py new file mode 100644 index 000000000..6477bdee3 --- /dev/null +++ b/python/tests/client/test_task_graph.py @@ -0,0 +1,74 @@ +"""Unit tests for Spider client TaskGraph""" + +import pytest + +from spider import chain, group, Int8, TaskContext + + +def no_context(x: Int8, y: Int8) -> Int8: + """Invalid task function with no context.""" + return Int8(x + y) + + +def invalid_type(_: TaskContext, x: int) -> int: + """Invalid task function with unsupported type.""" + return Int8(x + x) + + +def add(_: TaskContext, x: Int8, y: Int8) -> Int8: + """Adds two numbers.""" + return Int8(x + y) + + +def double(_: TaskContext, x: Int8) -> Int8: + """Double a number.""" + return Int8(x * 2) + + +def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: + """Swaps two numbers.""" + return y, x + + +class TestTaskGraph: + """Test task graph composition.""" + + def test_group(self) -> None: + """Tests task grouping.""" + graph = group([add, add]) + graph = group([graph, graph]) + assert len(graph._impl.tasks) == 4 + assert len(graph._impl.dependencies) == 0 + assert len(graph._impl.input_tasks) == 4 + assert len(graph._impl.output_tasks) == 4 + + def test_task_fail(self) -> None: + """Tests task failing.""" + with pytest.raises(TypeError): + group([no_context]) + with pytest.raises(TypeError): + group([invalid_type]) + + def test_chain(self) -> None: + """Tests task chaining.""" + graph = group([add, add]) + graph = chain(graph, swap) + assert len(graph._impl.tasks) == 3 + assert len(graph._impl.dependencies) == 2 + assert len(graph._impl.input_tasks) == 2 + assert len(graph._impl.output_tasks) == 1 + graph = chain(graph, add) + assert len(graph._impl.tasks) == 4 + assert len(graph._impl.dependencies) == 3 + assert len(graph._impl.input_tasks) == 2 + assert len(graph._impl.output_tasks) == 1 + graph = chain(swap, group([double, double])) + assert len(graph._impl.tasks) == 3 + assert len(graph._impl.dependencies) == 2 + assert len(graph._impl.input_tasks) == 1 + assert len(graph._impl.output_tasks) == 2 + + def test_chain_fail(self) -> None: + """Tests task chaining failure.""" + with pytest.raises(TypeError): + chain(add, add) diff --git a/python/tests/storage/test_jdbc_url.py b/python/tests/storage/test_jdbc_url.py new file mode 100644 index 000000000..680c02185 --- /dev/null +++ b/python/tests/storage/test_jdbc_url.py @@ -0,0 +1,37 @@ +"""Tests for the jdbc url module.""" + +import pytest + +from spider.storage import parse_jdbc_url + + +class TestJdbcUrl: + """Tests for the jdbc url module.""" + + def test_full_jdbc_url(self) -> None: + """Tests parsing a full JDBC URL with all fields.""" + url = "jdbc:mariadb://localhost:3306/dbname?user=root&password=secret" + params = parse_jdbc_url(url) + assert params.protocol == "jdbc:mariadb" + assert params.host == "localhost" + assert params.port == 3306 + assert params.database == "dbname" + assert params.user == "root" + assert params.password == "secret" # noqa: S105 + + def test_jdbc_url_simple(self) -> None: + """Tests parsing a simple JDBC URL without port, user and password.""" + url = "jdbc:postgresql://127.0.0.1/dbname" + params = parse_jdbc_url(url) + assert params.protocol == "jdbc:postgresql" + assert params.host == "127.0.0.1" + assert params.port is None + assert params.database == "dbname" + assert params.user is None + assert params.password is None + + def test_invalid_jdbc_url(self) -> None: + """Tests parsing an invalid JDBC URL.""" + url = "invalid_jdbc_url" + with pytest.raises(ValueError, match=f"Invalid JDBC URL: {url}"): + parse_jdbc_url(url) diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py new file mode 100644 index 000000000..7ec909a10 --- /dev/null +++ b/python/tests/storage/test_mariadb.py @@ -0,0 +1,101 @@ +"""Tests for the MariaDB storage backend.""" + +from uuid import uuid4 + +import msgpack +import pytest + +from spider import chain, group, Int8, TaskContext +from spider.core import Data, DataLocality, DriverId, Job, JobStatus, TaskInputValue +from spider.storage import MariaDBStorage, parse_jdbc_url, StorageError + +MariaDBTestUrl = "jdbc:mariadb://127.0.0.1:3306/spider-storage?user=spider&password=password" + + +@pytest.fixture(scope="session") +def mariadb_storage() -> MariaDBStorage: + """Fixture to create a MariaDB storage instance.""" + params = parse_jdbc_url(MariaDBTestUrl) + return MariaDBStorage(params) + + +def double(_: TaskContext, x: Int8) -> Int8: + """Double a number.""" + return Int8(x * 2) + + +def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: + """Swaps two numbers.""" + return y, x + + +@pytest.fixture +def submit_job(mariadb_storage: MariaDBStorage) -> Job: + """Submits a simple job.""" + graph = chain(group([double, double]), group([swap]))._impl + # Fill input data + for i, task_id in enumerate(graph.input_tasks): + task = graph.tasks[task_id] + task.task_inputs[0].value = TaskInputValue(msgpack.packb(i)) + + driver_id = uuid4() + jobs = mariadb_storage.submit_jobs(driver_id, [graph]) + return jobs[0] + + +@pytest.fixture +def driver(mariadb_storage: MariaDBStorage) -> DriverId: + """Fixture to create a driver.""" + driver_id = uuid4() + mariadb_storage.create_driver(driver_id) + return driver_id + + +class TestMariaDBStorage: + """Test class for the MariaDB storage backend.""" + + @pytest.mark.storage + def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: + """Tests job submission to the MariaDB storage backend.""" + graph = chain(group([double, double, double, double]), group([swap, swap]))._impl + # Fill input data + for i, task_id in enumerate(graph.input_tasks): + task = graph.tasks[task_id] + task.task_inputs[0].value = TaskInputValue(msgpack.packb(i)) + + driver_id = uuid4() + jobs = mariadb_storage.submit_jobs(driver_id, [graph]) + assert len(jobs) == 1 + + @pytest.mark.storage + def test_running_job_status(self, mariadb_storage: MariaDBStorage, submit_job: Job) -> None: + """Tests getting status of a running job.""" + status = mariadb_storage.get_job_status(submit_job) + assert status == JobStatus.Running + + @pytest.mark.storage + def test_running_job_result(self, mariadb_storage: MariaDBStorage, submit_job: Job) -> None: + """Tests getting results of a running job.""" + results = mariadb_storage.get_job_results(submit_job) + assert results is None + + @pytest.mark.storage + def test_data(self, mariadb_storage: MariaDBStorage, driver: DriverId) -> None: + """Tests data storage and retrieval.""" + value = b"test data" + data = Data(id=uuid4(), value=value, localities=[DataLocality("localhost")]) + mariadb_storage.create_driver_data(driver, data) + retrieved_data = mariadb_storage.get_data(data.id) + assert retrieved_data is not None + assert retrieved_data.id == data.id + assert retrieved_data.value == value + assert retrieved_data.hard_locality == data.hard_locality + assert retrieved_data.localities == data.localities + + @pytest.mark.storage + def test_create_data_fail(self, mariadb_storage: MariaDBStorage) -> None: + """Tests creating data without a driver fails.""" + value = b"test data" + data = Data(id=uuid4(), value=value, localities=[DataLocality("localhost")]) + with pytest.raises(StorageError): + mariadb_storage.create_driver_data(uuid4(), data) diff --git a/python/tests/type/test_to_native.py b/python/tests/type/test_to_native.py new file mode 100644 index 000000000..0f5a9c5ae --- /dev/null +++ b/python/tests/type/test_to_native.py @@ -0,0 +1,54 @@ +"""Test converting TDL type to native type.""" + +from types import GenericAlias + +import pytest + +import spider +from spider.type.tdl_parse import parse_tdl_type +from spider.type.utils import get_class_name + + +def string_to_native(s: str) -> type | GenericAlias: + """ + Converts TDL string to native type. + :param s: TDL string to convert. + :return: The converted native type. + :raises: TypeError forwards from `parse_tdl_type` and `native_type`. + """ + return parse_tdl_type(s).native_type() + + +class TestToNativeType: + """Test converting TDL type to native type.""" + + def test_to_primitive_native_type(self) -> None: + """Test converting primitive TDL type to native type.""" + assert string_to_native("bool") is bool + assert string_to_native("double") is spider.Double + assert string_to_native("float") is spider.Float + assert string_to_native("int8") is spider.Int8 + assert string_to_native("int16") is spider.Int16 + assert string_to_native("int32") is spider.Int32 + assert string_to_native("int64") is spider.Int64 + + def test_to_class_type(self) -> None: + """Test converting class TDL type to native type.""" + assert string_to_native(get_class_name(TestToNativeType)) == TestToNativeType + with pytest.raises(TypeError): + string_to_native("NonExistType") + with pytest.raises(TypeError): + string_to_native("NonExistModule.NonExistType") + + def test_to_list_type(self) -> None: + """Test converting list TDL type to native type.""" + assert string_to_native("List") == list[spider.Int8] + assert string_to_native("List>") == list[list[spider.Int8]] + + def test_to_map_type(self) -> None: + """Test converting map TDL type to native type.""" + assert string_to_native("Map") == dict[spider.Int8, spider.Int8] + assert ( + string_to_native("Map,Map>") + == dict[list[spider.Int8], dict[spider.Int8, spider.Double]] + ) diff --git a/python/tests/type/test_to_tdl.py b/python/tests/type/test_to_tdl.py new file mode 100644 index 000000000..c0fa734d9 --- /dev/null +++ b/python/tests/type/test_to_tdl.py @@ -0,0 +1,69 @@ +"""Unit tests for converting to TDL.""" + +import pytest + +from spider import Double, Float, Int8, Int16, Int32, Int64 +from spider.type import to_tdl_type_str + + +class TestToTDL: + """Unit tests for converting to TDL.""" + + def test_to_tdl_primitive(self) -> None: + """Test converting primitive types to TDL Types.""" + assert to_tdl_type_str(bool) == "bool" + assert to_tdl_type_str(Double) == "double" + assert to_tdl_type_str(Float) == "float" + assert to_tdl_type_str(Int8) == "int8" + assert to_tdl_type_str(Int16) == "int16" + assert to_tdl_type_str(Int32) == "int32" + assert to_tdl_type_str(Int64) == "int64" + + def test_to_tdl_list(self) -> None: + """Test converting lists to TDL Types.""" + assert to_tdl_type_str(list[Int8]) == "List" + assert to_tdl_type_str(list[list[Int8]]) == "List>" + + def test_to_tdl_map(self) -> None: + """Test converting maps to TDL Types.""" + assert to_tdl_type_str(dict[Int8, dict[Int16, Float]]) == "Map>" + assert to_tdl_type_str(dict[list[Int8], Double]) == "Map,double>" + + def test_to_tdl_class(self) -> None: + """Test converting class to TDL Types.""" + assert to_tdl_type_str(TestToTDL) == "test_to_tdl.TestToTDL" + + def test_to_tdl_primitive_exception(self) -> None: + """Test converting unsupported primitive types to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(int) + with pytest.raises(TypeError): + to_tdl_type_str(float) + with pytest.raises(TypeError): + to_tdl_type_str(str) + with pytest.raises(TypeError): + to_tdl_type_str(bytes) + with pytest.raises(TypeError): + to_tdl_type_str(list) + with pytest.raises(TypeError): + to_tdl_type_str(dict) + with pytest.raises(TypeError): + to_tdl_type_str(tuple) + + def test_to_tdl_list_exception(self) -> None: + """Test converting unsupported lists to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(list[int]) + + def test_to_tdl_map_exception(self) -> None: + """Test converting unsupported maps to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(dict[Int8, int]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[int, Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[list[Int16], Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[Float, Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[dict[Int8, Float], Int8]) diff --git a/python/tests/utils/test_msgpack_serde.py b/python/tests/utils/test_msgpack_serde.py new file mode 100644 index 000000000..92c267bf4 --- /dev/null +++ b/python/tests/utils/test_msgpack_serde.py @@ -0,0 +1,56 @@ +"""Tests for msgpack serialization and deserialization.""" + +from dataclasses import dataclass + +import msgpack + +import spider +from spider.utils.msgpack_serde import msgpack_decoder, msgpack_encoder + + +def compare_serde(obj: object) -> None: + """ + Serializes and then deserializes an object, and checks if the result matches the original + object. + """ + serialized = msgpack.packb(obj, default=msgpack_encoder) + unpacked_data = msgpack.unpackb(serialized, raw=False, strict_map_key=False) + deserialized = msgpack_decoder(type(obj), unpacked_data) + assert obj == deserialized + + +@dataclass +class Address: + """A simple address class for testing""" + + city: str + zipcode: str + + +@dataclass +class User: + """A simple user class for testing""" + + id: int + name: str + address: Address + + +class TestMsgpackSerde: + """Test class for msgpack serialization and deserialization.""" + + def test_primitives(self) -> None: + """Tests serialization and deserialization of primitive types.""" + compare_serde(True) + compare_serde([1, 2, 3]) + compare_serde({"key": "value"}) + compare_serde(spider.Int8(1)) + compare_serde(spider.Float(0.0)) + compare_serde([spider.Int8(1), spider.Int8(2)]) + compare_serde({spider.Int8(1): spider.Int8(3)}) + compare_serde([[spider.Int8(1), spider.Int8(2)], [spider.Int8(3), spider.Int8(4)]]) + + def test_class(self) -> None: + """Tests serialization and deserialization of a custom class.""" + user = User(id=1, name="Alice", address=Address(city="Wonderland", zipcode="12345")) + compare_serde(user) diff --git a/python/uv.lock b/python/uv.lock new file mode 100644 index 000000000..f3a9e6630 --- /dev/null +++ b/python/uv.lock @@ -0,0 +1,334 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "lark" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, +] + +[[package]] +name = "mariadb" +version = "1.1.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/a0/0498c90f46940b6fd7f5c76d22df9f011849964e94e0b22fde26c42191c7/mariadb-1.1.13.tar.gz", hash = "sha256:3a0fd24fae2b9990dc13a0a427a43d4a5434cc2a8c3b260a27f40b7824719037", size = 111126, upload-time = "2025-07-11T09:55:11.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e2/5e4068b225ccbd6fb7ea2b8f5ba52060e48bf2e63448206670cdfe73a1c5/mariadb-1.1.13-cp310-cp310-win32.whl", hash = "sha256:f49467e7076819ccf21a7e5ef030761af2160b23ce4484d88d260459809d90ff", size = 185138, upload-time = "2025-07-11T09:54:53.334Z" }, + { url = "https://files.pythonhosted.org/packages/33/8d/65b28ce61c422aba68c052a3cdb23772003267f47438a27a6e2cbf58c1c9/mariadb-1.1.13-cp310-cp310-win_amd64.whl", hash = "sha256:1c9aa9a31f943737c3a58bb4dc9deb9cf2b50b3ec807f90c5c99c7b35f8442bd", size = 202048, upload-time = "2025-07-11T09:54:54.978Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/c23f6ffb06ba2c0ad2e90f4236b1541f9005e24eaeb166800b19c4f584f6/mariadb-1.1.13-cp311-cp311-win32.whl", hash = "sha256:c15ab872f1e951dd1bd4006d2ee2b942156f66c1cd724a1b2992dc02c47dff6a", size = 185136, upload-time = "2025-07-11T09:54:56.791Z" }, + { url = "https://files.pythonhosted.org/packages/c8/08/10308cb9bbccf83ff72caa1146014445a79dfeffb4eba9a85f73575c5959/mariadb-1.1.13-cp311-cp311-win_amd64.whl", hash = "sha256:d6fa7f2cdfae762d2d69b71e51f61415d0251118a622a6d06c7f05261579100a", size = 202046, upload-time = "2025-07-11T09:54:58.365Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b4/d961a5dfa8a7a04ede70f76f85b951b8d0ec28d79b86ced829a67af78bdd/mariadb-1.1.13-cp312-cp312-win32.whl", hash = "sha256:358ecdc8775f4a97357fc233bb84a1c8d8d52866ee5fe98f2d13e9e58a04b6fb", size = 185334, upload-time = "2025-07-11T09:54:59.901Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3b/16cc69a9c90f91b74c096c0cb9634e14692486beb28b82ec5efd97866de7/mariadb-1.1.13-cp312-cp312-win_amd64.whl", hash = "sha256:0604c57e6f90ff7ef190a56894d2b32ca5f0182924b5159be9319ba243ffbf98", size = 202109, upload-time = "2025-07-11T09:55:01.846Z" }, + { url = "https://files.pythonhosted.org/packages/02/16/583719e127774294d4af085a063b38f20407801cd9194a676334fb121edd/mariadb-1.1.13-cp313-cp313-win32.whl", hash = "sha256:36c644c13ecf3e38cb89f37f9bd054703760b94c881476492fe789801a063ce3", size = 185301, upload-time = "2025-07-11T09:55:03.486Z" }, + { url = "https://files.pythonhosted.org/packages/1b/da/4170f5532813ffba2641f821ed23ed1f22fa3ff8d603e47c91b0776ff9b7/mariadb-1.1.13-cp313-cp313-win_amd64.whl", hash = "sha256:6ec51ad2e33312c397cfcc5e2bca59861f572d5e8880e025d43db9a5d472a3a9", size = 202105, upload-time = "2025-07-11T09:55:05.681Z" }, +] + +[[package]] +name = "msgpack" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/52/f30da112c1dc92cf64f57d08a273ac771e7b29dea10b4b30369b2d7e8546/msgpack-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:353b6fc0c36fde68b661a12949d7d49f8f51ff5fa019c1e47c87c4ff34b080ed", size = 81799, upload-time = "2025-06-13T06:51:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/7bfc0def2f04ab4145f7f108e3563f9b4abae4ab0ed78a61f350518cc4d2/msgpack-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:79c408fcf76a958491b4e3b103d1c417044544b68e96d06432a189b43d1215c8", size = 78278, upload-time = "2025-06-13T06:51:38.534Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c5/df5d6c1c39856bc55f800bf82778fd4c11370667f9b9e9d51b2f5da88f20/msgpack-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78426096939c2c7482bf31ef15ca219a9e24460289c00dd0b94411040bb73ad2", size = 402805, upload-time = "2025-06-13T06:51:39.538Z" }, + { url = "https://files.pythonhosted.org/packages/20/8e/0bb8c977efecfe6ea7116e2ed73a78a8d32a947f94d272586cf02a9757db/msgpack-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b17ba27727a36cb73aabacaa44b13090feb88a01d012c0f4be70c00f75048b4", size = 408642, upload-time = "2025-06-13T06:51:41.092Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/731d52c1aeec52006be6d1f8027c49fdc2cfc3ab7cbe7c28335b2910d7b6/msgpack-1.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a17ac1ea6ec3c7687d70201cfda3b1e8061466f28f686c24f627cae4ea8efd0", size = 395143, upload-time = "2025-06-13T06:51:42.575Z" }, + { url = "https://files.pythonhosted.org/packages/2b/92/b42911c52cda2ba67a6418ffa7d08969edf2e760b09015593c8a8a27a97d/msgpack-1.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88d1e966c9235c1d4e2afac21ca83933ba59537e2e2727a999bf3f515ca2af26", size = 395986, upload-time = "2025-06-13T06:51:43.807Z" }, + { url = "https://files.pythonhosted.org/packages/61/dc/8ae165337e70118d4dab651b8b562dd5066dd1e6dd57b038f32ebc3e2f07/msgpack-1.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f6d58656842e1b2ddbe07f43f56b10a60f2ba5826164910968f5933e5178af75", size = 402682, upload-time = "2025-06-13T06:51:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/58/27/555851cb98dcbd6ce041df1eacb25ac30646575e9cd125681aa2f4b1b6f1/msgpack-1.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96decdfc4adcbc087f5ea7ebdcfd3dee9a13358cae6e81d54be962efc38f6338", size = 406368, upload-time = "2025-06-13T06:51:46.97Z" }, + { url = "https://files.pythonhosted.org/packages/d4/64/39a26add4ce16f24e99eabb9005e44c663db00e3fce17d4ae1ae9d61df99/msgpack-1.1.1-cp310-cp310-win32.whl", hash = "sha256:6640fd979ca9a212e4bcdf6eb74051ade2c690b862b679bfcb60ae46e6dc4bfd", size = 65004, upload-time = "2025-06-13T06:51:48.582Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/73dfa3e9d5d7450d39debde5b0d848139f7de23bd637a4506e36c9800fd6/msgpack-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b65b53204fe1bd037c40c4148d00ef918eb2108d24c9aaa20bc31f9810ce0a8", size = 71548, upload-time = "2025-06-13T06:51:49.558Z" }, + { url = "https://files.pythonhosted.org/packages/7f/83/97f24bf9848af23fe2ba04380388216defc49a8af6da0c28cc636d722502/msgpack-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71ef05c1726884e44f8b1d1773604ab5d4d17729d8491403a705e649116c9558", size = 82728, upload-time = "2025-06-13T06:51:50.68Z" }, + { url = "https://files.pythonhosted.org/packages/aa/7f/2eaa388267a78401f6e182662b08a588ef4f3de6f0eab1ec09736a7aaa2b/msgpack-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36043272c6aede309d29d56851f8841ba907a1a3d04435e43e8a19928e243c1d", size = 79279, upload-time = "2025-06-13T06:51:51.72Z" }, + { url = "https://files.pythonhosted.org/packages/f8/46/31eb60f4452c96161e4dfd26dbca562b4ec68c72e4ad07d9566d7ea35e8a/msgpack-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a32747b1b39c3ac27d0670122b57e6e57f28eefb725e0b625618d1b59bf9d1e0", size = 423859, upload-time = "2025-06-13T06:51:52.749Z" }, + { url = "https://files.pythonhosted.org/packages/45/16/a20fa8c32825cc7ae8457fab45670c7a8996d7746ce80ce41cc51e3b2bd7/msgpack-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a8b10fdb84a43e50d38057b06901ec9da52baac6983d3f709d8507f3889d43f", size = 429975, upload-time = "2025-06-13T06:51:53.97Z" }, + { url = "https://files.pythonhosted.org/packages/86/ea/6c958e07692367feeb1a1594d35e22b62f7f476f3c568b002a5ea09d443d/msgpack-1.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0c325c3f485dc54ec298d8b024e134acf07c10d494ffa24373bea729acf704", size = 413528, upload-time = "2025-06-13T06:51:55.507Z" }, + { url = "https://files.pythonhosted.org/packages/75/05/ac84063c5dae79722bda9f68b878dc31fc3059adb8633c79f1e82c2cd946/msgpack-1.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:88daaf7d146e48ec71212ce21109b66e06a98e5e44dca47d853cbfe171d6c8d2", size = 413338, upload-time = "2025-06-13T06:51:57.023Z" }, + { url = "https://files.pythonhosted.org/packages/69/e8/fe86b082c781d3e1c09ca0f4dacd457ede60a13119b6ce939efe2ea77b76/msgpack-1.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8b55ea20dc59b181d3f47103f113e6f28a5e1c89fd5b67b9140edb442ab67f2", size = 422658, upload-time = "2025-06-13T06:51:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2b/bafc9924df52d8f3bb7c00d24e57be477f4d0f967c0a31ef5e2225e035c7/msgpack-1.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a28e8072ae9779f20427af07f53bbb8b4aa81151054e882aee333b158da8752", size = 427124, upload-time = "2025-06-13T06:51:59.969Z" }, + { url = "https://files.pythonhosted.org/packages/a2/3b/1f717e17e53e0ed0b68fa59e9188f3f610c79d7151f0e52ff3cd8eb6b2dc/msgpack-1.1.1-cp311-cp311-win32.whl", hash = "sha256:7da8831f9a0fdb526621ba09a281fadc58ea12701bc709e7b8cbc362feabc295", size = 65016, upload-time = "2025-06-13T06:52:01.294Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/9d1780768d3b249accecc5a38c725eb1e203d44a191f7b7ff1941f7df60c/msgpack-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fd1b58e1431008a57247d6e7cc4faa41c3607e8e7d4aaf81f7c29ea013cb458", size = 72267, upload-time = "2025-06-13T06:52:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/389b9c593eda2b8551b2e7126ad3a06af6f9b44274eb3a4f054d48ff7e47/msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238", size = 82359, upload-time = "2025-06-13T06:52:03.909Z" }, + { url = "https://files.pythonhosted.org/packages/ab/65/7d1de38c8a22cf8b1551469159d4b6cf49be2126adc2482de50976084d78/msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157", size = 79172, upload-time = "2025-06-13T06:52:05.246Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bd/cacf208b64d9577a62c74b677e1ada005caa9b69a05a599889d6fc2ab20a/msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce", size = 425013, upload-time = "2025-06-13T06:52:06.341Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ec/fd869e2567cc9c01278a736cfd1697941ba0d4b81a43e0aa2e8d71dab208/msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a", size = 426905, upload-time = "2025-06-13T06:52:07.501Z" }, + { url = "https://files.pythonhosted.org/packages/55/2a/35860f33229075bce803a5593d046d8b489d7ba2fc85701e714fc1aaf898/msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c", size = 407336, upload-time = "2025-06-13T06:52:09.047Z" }, + { url = "https://files.pythonhosted.org/packages/8c/16/69ed8f3ada150bf92745fb4921bd621fd2cdf5a42e25eb50bcc57a5328f0/msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b", size = 409485, upload-time = "2025-06-13T06:52:10.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b6/0c398039e4c6d0b2e37c61d7e0e9d13439f91f780686deb8ee64ecf1ae71/msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef", size = 412182, upload-time = "2025-06-13T06:52:11.644Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/0cf4a6ecb9bc960d624c93effaeaae75cbf00b3bc4a54f35c8507273cda1/msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a", size = 419883, upload-time = "2025-06-13T06:52:12.806Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/9697c211720fa71a2dfb632cad6196a8af3abea56eece220fde4674dc44b/msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c", size = 65406, upload-time = "2025-06-13T06:52:14.271Z" }, + { url = "https://files.pythonhosted.org/packages/c0/23/0abb886e80eab08f5e8c485d6f13924028602829f63b8f5fa25a06636628/msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4", size = 72558, upload-time = "2025-06-13T06:52:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/a1/38/561f01cf3577430b59b340b51329803d3a5bf6a45864a55f4ef308ac11e3/msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0", size = 81677, upload-time = "2025-06-13T06:52:16.64Z" }, + { url = "https://files.pythonhosted.org/packages/09/48/54a89579ea36b6ae0ee001cba8c61f776451fad3c9306cd80f5b5c55be87/msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9", size = 78603, upload-time = "2025-06-13T06:52:17.843Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/daba2699b308e95ae792cdc2ef092a38eb5ee422f9d2fbd4101526d8a210/msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8", size = 420504, upload-time = "2025-06-13T06:52:18.982Z" }, + { url = "https://files.pythonhosted.org/packages/20/22/2ebae7ae43cd8f2debc35c631172ddf14e2a87ffcc04cf43ff9df9fff0d3/msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a", size = 423749, upload-time = "2025-06-13T06:52:20.211Z" }, + { url = "https://files.pythonhosted.org/packages/40/1b/54c08dd5452427e1179a40b4b607e37e2664bca1c790c60c442c8e972e47/msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac", size = 404458, upload-time = "2025-06-13T06:52:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/2e/60/6bb17e9ffb080616a51f09928fdd5cac1353c9becc6c4a8abd4e57269a16/msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b", size = 405976, upload-time = "2025-06-13T06:52:22.995Z" }, + { url = "https://files.pythonhosted.org/packages/ee/97/88983e266572e8707c1f4b99c8fd04f9eb97b43f2db40e3172d87d8642db/msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7", size = 408607, upload-time = "2025-06-13T06:52:24.152Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/36c78af2efaffcc15a5a61ae0df53a1d025f2680122e2a9eb8442fed3ae4/msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5", size = 424172, upload-time = "2025-06-13T06:52:25.704Z" }, + { url = "https://files.pythonhosted.org/packages/8c/87/a75eb622b555708fe0427fab96056d39d4c9892b0c784b3a721088c7ee37/msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323", size = 65347, upload-time = "2025-06-13T06:52:26.846Z" }, + { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, +] + +[[package]] +name = "msgpack-types" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/26/a15707f2af5681333cd598724bedd1948844ac2af45eafc4175af0671a8d/msgpack_types-0.5.0.tar.gz", hash = "sha256:aebd1b8da23f8f9966d66ebb1a43bd261b95751c6a267bd21a124d2ccac84201", size = 6702, upload-time = "2024-09-21T13:55:05.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/dd/cd9d2b0ef506f6164cd81d4e92e408095041f28523d751b9f7dabdc244eb/msgpack_types-0.5.0-py3-none-any.whl", hash = "sha256:8b633ed75e495a555fa0615843de559a74b1d176828d59bb393d266e51f6bda7", size = 8182, upload-time = "2024-09-21T13:55:04.232Z" }, +] + +[[package]] +name = "mypy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/81/0bd3594fa0f690466e41bd033bdcdf86cba8288345ac77ad4afbe5ec743a/ruff-0.12.7.tar.gz", hash = "sha256:1fc3193f238bc2d7968772c82831a4ff69252f673be371fb49663f0068b7ec71", size = 5197814, upload-time = "2025-07-29T22:32:35.877Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d2/6cb35e9c85e7a91e8d22ab32ae07ac39cc34a71f1009a6f9e4a2a019e602/ruff-0.12.7-py3-none-linux_armv6l.whl", hash = "sha256:76e4f31529899b8c434c3c1dede98c4483b89590e15fb49f2d46183801565303", size = 11852189, upload-time = "2025-07-29T22:31:41.281Z" }, + { url = "https://files.pythonhosted.org/packages/63/5b/a4136b9921aa84638f1a6be7fb086f8cad0fde538ba76bda3682f2599a2f/ruff-0.12.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:789b7a03e72507c54fb3ba6209e4bb36517b90f1a3569ea17084e3fd295500fb", size = 12519389, upload-time = "2025-07-29T22:31:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c9/3e24a8472484269b6b1821794141f879c54645a111ded4b6f58f9ab0705f/ruff-0.12.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2e1c2a3b8626339bb6369116e7030a4cf194ea48f49b64bb505732a7fce4f4e3", size = 11743384, upload-time = "2025-07-29T22:31:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/26/7c/458dd25deeb3452c43eaee853c0b17a1e84169f8021a26d500ead77964fd/ruff-0.12.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32dec41817623d388e645612ec70d5757a6d9c035f3744a52c7b195a57e03860", size = 11943759, upload-time = "2025-07-29T22:32:01.95Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8b/658798472ef260ca050e400ab96ef7e85c366c39cf3dfbef4d0a46a528b6/ruff-0.12.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47ef751f722053a5df5fa48d412dbb54d41ab9b17875c6840a58ec63ff0c247c", size = 11654028, upload-time = "2025-07-29T22:32:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/a8/86/9c2336f13b2a3326d06d39178fd3448dcc7025f82514d1b15816fe42bfe8/ruff-0.12.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a828a5fc25a3efd3e1ff7b241fd392686c9386f20e5ac90aa9234a5faa12c423", size = 13225209, upload-time = "2025-07-29T22:32:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/76/69/df73f65f53d6c463b19b6b312fd2391dc36425d926ec237a7ed028a90fc1/ruff-0.12.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5726f59b171111fa6a69d82aef48f00b56598b03a22f0f4170664ff4d8298efb", size = 14182353, upload-time = "2025-07-29T22:32:10.053Z" }, + { url = "https://files.pythonhosted.org/packages/58/1e/de6cda406d99fea84b66811c189b5ea139814b98125b052424b55d28a41c/ruff-0.12.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74e6f5c04c4dd4aba223f4fe6e7104f79e0eebf7d307e4f9b18c18362124bccd", size = 13631555, upload-time = "2025-07-29T22:32:12.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ae/625d46d5164a6cc9261945a5e89df24457dc8262539ace3ac36c40f0b51e/ruff-0.12.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0bfe4e77fba61bf2ccadf8cf005d6133e3ce08793bbe870dd1c734f2699a3e", size = 12667556, upload-time = "2025-07-29T22:32:15.312Z" }, + { url = "https://files.pythonhosted.org/packages/55/bf/9cb1ea5e3066779e42ade8d0cd3d3b0582a5720a814ae1586f85014656b6/ruff-0.12.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06bfb01e1623bf7f59ea749a841da56f8f653d641bfd046edee32ede7ff6c606", size = 12939784, upload-time = "2025-07-29T22:32:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/7ead2663be5627c04be83754c4f3096603bf5e99ed856c7cd29618c691bd/ruff-0.12.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e41df94a957d50083fd09b916d6e89e497246698c3f3d5c681c8b3e7b9bb4ac8", size = 11771356, upload-time = "2025-07-29T22:32:20.134Z" }, + { url = "https://files.pythonhosted.org/packages/17/40/a95352ea16edf78cd3a938085dccc55df692a4d8ba1b3af7accbe2c806b0/ruff-0.12.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4000623300563c709458d0ce170c3d0d788c23a058912f28bbadc6f905d67afa", size = 11612124, upload-time = "2025-07-29T22:32:22.645Z" }, + { url = "https://files.pythonhosted.org/packages/4d/74/633b04871c669e23b8917877e812376827c06df866e1677f15abfadc95cb/ruff-0.12.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:69ffe0e5f9b2cf2b8e289a3f8945b402a1b19eff24ec389f45f23c42a3dd6fb5", size = 12479945, upload-time = "2025-07-29T22:32:24.765Z" }, + { url = "https://files.pythonhosted.org/packages/be/34/c3ef2d7799c9778b835a76189c6f53c179d3bdebc8c65288c29032e03613/ruff-0.12.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a07a5c8ffa2611a52732bdc67bf88e243abd84fe2d7f6daef3826b59abbfeda4", size = 12998677, upload-time = "2025-07-29T22:32:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/77/ab/aca2e756ad7b09b3d662a41773f3edcbd262872a4fc81f920dc1ffa44541/ruff-0.12.7-py3-none-win32.whl", hash = "sha256:c928f1b2ec59fb77dfdf70e0419408898b63998789cc98197e15f560b9e77f77", size = 11756687, upload-time = "2025-07-29T22:32:29.381Z" }, + { url = "https://files.pythonhosted.org/packages/b4/71/26d45a5042bc71db22ddd8252ca9d01e9ca454f230e2996bb04f16d72799/ruff-0.12.7-py3-none-win_amd64.whl", hash = "sha256:9c18f3d707ee9edf89da76131956aba1270c6348bfee8f6c647de841eac7194f", size = 12912365, upload-time = "2025-07-29T22:32:31.517Z" }, + { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" }, +] + +[[package]] +name = "spider" +version = "0.0.1" +source = { editable = "." } +dependencies = [ + { name = "lark" }, + { name = "mariadb" }, + { name = "msgpack-types" }, +] + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "lark", specifier = ">=1.2.2" }, + { name = "mariadb", specifier = ">=1.1.13" }, + { name = "msgpack-types", specifier = ">=0.5.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=1.12.0" }, + { name = "pytest", specifier = ">=8.4.1" }, + { name = "ruff", specifier = ">=0.4.4" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, +] diff --git a/taskfile.yaml b/taskfile.yaml index 470edd582..1a94fdc38 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,9 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_SRC_PYTHON_DIR: "{{.ROOT_DIR}}/python" + G_BUILD_PYTHON_DIR: "{{.G_BUILD_DIR}}/spider-py" + # Build parameters # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- diff --git a/test-tasks.yaml b/test-tasks.yaml index 936211e5b..60b0724ef 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -18,7 +18,7 @@ tasks: cmds: - "{{.G_UNIT_TEST_BINARY}} \"[storage]\"" - all: + cpp-all: deps: - "build-unit-test" cmds: @@ -31,7 +31,7 @@ tasks: vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] - integration: + cpp-integration: dir: "{{.G_BUILD_SPIDER_DIR}}" deps: - "venv" @@ -76,3 +76,27 @@ tasks: vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + + python-tests: + dir: "{{.G_SRC_PYTHON_DIR}}" + env: + # Don't create __pycache__ directories in source tree. + PYTHONDONTWRITEBYTECODE: "1" + cmds: + - "uv run pytest" + + python-non-storage-tests: + dir: "{{.G_SRC_PYTHON_DIR}}" + env: + # Don't create __pycache__ directories in source tree. + PYTHONDONTWRITEBYTECODE: "1" + cmds: + - "uv run pytest -m \"not storage\"" + + python-storage-tests: + dir: "{{.G_SRC_PYTHON_DIR}}" + env: + # Don't create __pycache__ directories in source tree. + PYTHONDONTWRITEBYTECODE: "1" + cmds: + - "uv run pytest -m \"storage\""