Skip to content
Merged
Show file tree
Hide file tree
Changes from 133 commits
Commits
Show all changes
148 commits
Select commit Hold shift + click to select a range
d2ac198
Update yscope-dev-utils
sitaowang1998 Jul 15, 2025
cc8a097
Use boost install task
sitaowang1998 Jul 15, 2025
7f68bdc
Update install task variable names
sitaowang1998 Jul 15, 2025
2394649
Set CMP0074 to NEW to find boost
sitaowang1998 Jul 15, 2025
6ff3e33
Add uv to install script
sitaowang1998 Jul 15, 2025
7ae8a78
Fix cpp-lint root paths
sitaowang1998 Jul 15, 2025
56502f9
Fix clang-tidy file pattern
sitaowang1998 Jul 15, 2025
6f6becf
Limit build parallelism
sitaowang1998 Jul 17, 2025
a74523d
Bug fix
sitaowang1998 Jul 17, 2025
2345d22
Merge branch 'dep-concurrency' into yscope-dev-utils
sitaowang1998 Jul 17, 2025
77ee494
Bug fix
sitaowang1998 Jul 17, 2025
38b86c8
Merge branch 'dep-concurrency' into yscope-dev-utils
sitaowang1998 Jul 17, 2025
3698339
Bug fix
sitaowang1998 Jul 17, 2025
369e9f1
Rename variables to mirror CLP core
sitaowang1998 Aug 4, 2025
90aa5a2
Rename variables to mirror clp core
sitaowang1998 Aug 4, 2025
1769c95
Merge branch 'dep-concurrency' of github.com:sitaowang1998/spider int…
sitaowang1998 Aug 4, 2025
edaa834
Merge branch 'dep-concurrency' into yscope-dev-utils
sitaowang1998 Aug 4, 2025
7faac8f
Revert "Merge branch 'dep-concurrency' of github.com:sitaowang1998/sp…
sitaowang1998 Aug 4, 2025
f494a90
Add comment for deps parallelism default value
sitaowang1998 Aug 4, 2025
eb01bb2
Merge branch 'dep-concurrency' into yscope-dev-utils
sitaowang1998 Aug 4, 2025
ff2fe1c
Update yscope-dev-utils
sitaowang1998 Aug 4, 2025
d476e42
Merge branch 'main' into yscope-dev-utils
sitaowang1998 Aug 4, 2025
850126d
Merge branch 'yscope-dev-utils' into python_lint
sitaowang1998 Aug 4, 2025
65841a0
Add latest python lint config files
sitaowang1998 Aug 4, 2025
8564fc2
Update ruff lint tasks
sitaowang1998 Aug 4, 2025
a6e7d29
Fix ruff lint
sitaowang1998 Aug 4, 2025
573b448
Fix ruff lint
sitaowang1998 Aug 4, 2025
6ad72c3
Bug fix
sitaowang1998 Aug 4, 2025
ba7c6e5
Fix ruff
sitaowang1998 Aug 4, 2025
209acb1
Fix ruff
sitaowang1998 Aug 4, 2025
406b514
Reformat files
sitaowang1998 Aug 4, 2025
66892f5
Remove .inc from cpp linting
sitaowang1998 Aug 4, 2025
80f0a10
Merge branch 'yscope-dev-utils' into python_lint
sitaowang1998 Aug 4, 2025
3186a42
Add mypy and merge lint and test requirements.txt
sitaowang1998 Aug 4, 2025
0925938
Fix mysql connection type
sitaowang1998 Aug 4, 2025
6936a9b
Fix socket name type
sitaowang1998 Aug 4, 2025
7407090
Fix return type from db cursor
sitaowang1998 Aug 4, 2025
6a95b24
Fix db cursor return type
sitaowang1998 Aug 4, 2025
f29dcba
Fix mypy import untyped
sitaowang1998 Aug 4, 2025
453f209
Fix mypy and Popen
sitaowang1998 Aug 5, 2025
d1011c7
Fix generator type hint
sitaowang1998 Aug 5, 2025
a7dc642
Fix mypy
sitaowang1998 Aug 5, 2025
a7d92c2
Fix ruff
sitaowang1998 Aug 5, 2025
1c6213b
Simply socket return types.
sitaowang1998 Aug 5, 2025
0f13d07
Merge branch 'main' into python_lint
sitaowang1998 Aug 5, 2025
14d6326
Merge branch 'python_lint' into mypy_lint
sitaowang1998 Aug 5, 2025
55a727a
Add tombi lint tasks
sitaowang1998 Aug 5, 2025
c98fea1
Rename cpp build tasks
sitaowang1998 Aug 5, 2025
ef64454
Merge branch 'main' into mypy_lint
sitaowang1998 Aug 5, 2025
09ba8ad
Merge branch 'main' into tombi
sitaowang1998 Aug 5, 2025
4e7237e
Add basic python structure
sitaowang1998 Aug 5, 2025
1f944d0
Fix code structure
sitaowang1998 Aug 5, 2025
68992fc
Fix ruff lint
sitaowang1998 Aug 5, 2025
1127b22
Extend lint tasks to python directory
sitaowang1998 Aug 5, 2025
4c51c91
Add python build tasks
sitaowang1998 Aug 5, 2025
56878b2
Merge branch 'main' into mypy_lint
sitaowang1998 Aug 5, 2025
bb428c9
Merge branch 'mypy_lint' into tombi
sitaowang1998 Aug 5, 2025
944bd5e
Merge branch 'tombi' into build-task
sitaowang1998 Aug 5, 2025
d75bc15
Merge branch 'build-task' into python_structure
sitaowang1998 Aug 5, 2025
4b7eca2
Merge branch 'main' into tombi
sitaowang1998 Aug 5, 2025
8ddbda7
Merge branch 'tombi' into build-task
sitaowang1998 Aug 5, 2025
0d998ec
Merge branch 'build-task' into python_structure
sitaowang1998 Aug 5, 2025
6004d64
Fix yaml lint error
sitaowang1998 Aug 5, 2025
bcab5db
Fix tombi lint
sitaowang1998 Aug 5, 2025
a50d25a
Remove wrong mypy config
sitaowang1998 Aug 5, 2025
fbebb96
Fix typo and format file
sitaowang1998 Aug 5, 2025
0322bd5
Use typed msgpack and remove mypy config for msgpack from pyproject
sitaowang1998 Aug 6, 2025
2a08155
Update uv lock
sitaowang1998 Aug 6, 2025
0b62f73
Merge branch 'main' into tombi
sitaowang1998 Aug 6, 2025
7384158
Merge branch 'main' into build-task
sitaowang1998 Aug 6, 2025
70ef90a
Merge branch 'main' into python_structure
sitaowang1998 Aug 6, 2025
b09515c
Increase min version of tombi
sitaowang1998 Aug 6, 2025
bad9675
Merge branch 'tombi' into build-task
sitaowang1998 Aug 6, 2025
e009ff4
Merge branch 'tombi' into python_structure
sitaowang1998 Aug 6, 2025
a0407a7
Merge branch 'main' into tombi
sitaowang1998 Aug 6, 2025
aa9a8de
Merge branch 'tombi' into build-task
sitaowang1998 Aug 6, 2025
3c7e794
Merge branch 'tombi' into python_structure
sitaowang1998 Aug 6, 2025
b4d6576
Add uv in README
sitaowang1998 Aug 6, 2025
23f31cd
Merge branch 'main' into build-task
sitaowang1998 Aug 6, 2025
0167ac0
Merge branch 'build-task' into python_structure
sitaowang1998 Aug 6, 2025
f447614
Fix redenduncy caused by merge
sitaowang1998 Aug 6, 2025
a84de83
Merge branch 'build-task' into python_structure
sitaowang1998 Aug 6, 2025
2d1b6fc
Merge branch 'main' into python_structure
sitaowang1998 Aug 7, 2025
061d101
Merge branch 'main' into python_structure
sitaowang1998 Aug 7, 2025
de149ee
Add integral types
sitaowang1998 Aug 7, 2025
d332dbe
Revert "Add integral types"
sitaowang1998 Aug 7, 2025
80b6772
Add package export control for type
sitaowang1998 Aug 7, 2025
f2ebaa6
Add floating point type
sitaowang1998 Aug 8, 2025
8c27b16
Restructure under src/spider
sitaowang1998 Aug 8, 2025
b7f1884
Merge branch 'python_structure' into python_type
sitaowang1998 Aug 8, 2025
a7aa6b0
Re-export spider.type under spider
sitaowang1998 Aug 8, 2025
95b98c8
Merge branch 'main' into python_structure
sitaowang1998 Aug 8, 2025
5765acf
Add tdl types; Allow no docstring for override function
sitaowang1998 Aug 8, 2025
427dd6b
Merge branch 'main' into python_structure
sitaowang1998 Aug 8, 2025
213d2fc
Add type conversion to tdl type
sitaowang1998 Aug 8, 2025
a1a86f3
Add to_tdl_str
sitaowang1998 Aug 8, 2025
97af71b
Add to_tdl_type_str to type package export
sitaowang1998 Aug 8, 2025
74f31e2
Add pytest and basic test structure
sitaowang1998 Aug 8, 2025
7d95706
Add python test tasks
sitaowang1998 Aug 8, 2025
96b5324
Rename some cpp tests and add python tests to GH workflow and doc
sitaowang1998 Aug 8, 2025
14ea6fb
Don't create __pycache__ when running pytest
sitaowang1998 Aug 8, 2025
854cd11
Fix missing link
sitaowang1998 Aug 8, 2025
5565c9e
Merge branch 'python_structure' of github.com:sitaowang1998/spider in…
sitaowang1998 Aug 8, 2025
2504429
Merge branch 'python_structure' into python_test_setup
sitaowang1998 Aug 8, 2025
7ea4928
Use task env
sitaowang1998 Aug 8, 2025
01dd0c6
Fix task executor path
sitaowang1998 Aug 8, 2025
4cec09b
Merge branch 'python_structure' into python_test_setup
sitaowang1998 Aug 8, 2025
e022404
Merge branch 'main' into python_structure
sitaowang1998 Aug 8, 2025
8894b90
Merge branch 'main' into python_test_setup
sitaowang1998 Aug 8, 2025
7753153
Merge branch 'main' into python_structure
sitaowang1998 Aug 9, 2025
dd59915
Merge branch 'main' into python_test_setup
sitaowang1998 Aug 9, 2025
ac1d4ee
Merge branch 'python_structure' into python_type
sitaowang1998 Aug 9, 2025
0d38c43
Merge branch 'python_test_setup' into python_type
sitaowang1998 Aug 9, 2025
438f6e7
Remove unnecessary __init__.py files
sitaowang1998 Aug 9, 2025
2caad5d
Merge branch 'python_structure' into python_test_setup
sitaowang1998 Aug 9, 2025
c6815ce
Merge branch 'python_test_setup' into python_type
sitaowang1998 Aug 9, 2025
61e0651
Fix bugs
sitaowang1998 Aug 9, 2025
8b13abd
Add some type convertion tests
sitaowang1998 Aug 9, 2025
50b2b25
Bug fix
sitaowang1998 Aug 9, 2025
519dcd8
Merge branch 'python_test_setup' into python_type
sitaowang1998 Aug 9, 2025
8e2786b
Add more tests
sitaowang1998 Aug 9, 2025
41d5de2
Add parsing of tdl type string
sitaowang1998 Aug 9, 2025
0a377f5
Add tests and fix parser
sitaowang1998 Aug 9, 2025
a8c8641
Fix pytest
sitaowang1998 Aug 9, 2025
9fabc44
Merge branch 'python_test_setup' into python_type
sitaowang1998 Aug 9, 2025
ade589b
Merge branch 'python_type' into python_type_parse
sitaowang1998 Aug 9, 2025
0b18170
Satisfy linter
sitaowang1998 Aug 9, 2025
0bfbdb5
Fix comment
sitaowang1998 Aug 9, 2025
e558f67
Improve code according to coderabbit
sitaowang1998 Aug 9, 2025
73e8f0c
Merge branch 'python_type' into python_type_parse
sitaowang1998 Aug 9, 2025
6b58ff0
Move functions to util
sitaowang1998 Aug 9, 2025
90ab334
Unwarp the type node to get TdlType
sitaowang1998 Aug 10, 2025
49acaee
Catch and rethrow lark error
sitaowang1998 Aug 10, 2025
d3f3eb1
Export tdl parse function
sitaowang1998 Aug 14, 2025
43343aa
Refactor spider-py structure to perpare for merge
sitaowang1998 Aug 23, 2025
2f81cd9
Merge branch 'main' into python_type_parse
sitaowang1998 Aug 23, 2025
ff60b0f
Fix merge
sitaowang1998 Aug 23, 2025
8223792
Fix the docstring
sitaowang1998 Aug 23, 2025
311e041
Fix merge
sitaowang1998 Aug 23, 2025
3e75f87
Fix ruff
sitaowang1998 Aug 23, 2025
de28844
Improve style based on code review
sitaowang1998 Aug 23, 2025
346a45a
Use v_args for base_type
sitaowang1998 Aug 23, 2025
d348ed0
Use static lark transformer
sitaowang1998 Aug 23, 2025
84aea87
Apply suggestions from code review
sitaowang1998 Aug 23, 2025
1f02daf
Improve docstring
sitaowang1998 Aug 23, 2025
138ef8d
Fix style
sitaowang1998 Aug 23, 2025
9d2d70e
Rename private functions
sitaowang1998 Aug 23, 2025
af5dd2c
Avoid using Yoda condition
sitaowang1998 Aug 23, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/code-linting-checks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ jobs:
python --version
tar --version
task --version
uv --version
Comment thread
sitaowang1998 marked this conversation as resolved.
Resolved

- name: "Install project dependencies "
timeout-minutes: 10
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/unit-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ jobs:
python --version
tar --version
task --version
uv --version

- name: "Install project dependencies "
timeout-minutes: 10
Expand All @@ -55,3 +56,5 @@ jobs:
run: "task deps:lib_install"

- run: "task test:non-storage-unit-tests"

- run: "task test:python-tests"
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Follow the steps below to develop and contribute to the project.
## Requirements
* Python 3.10 or higher
* [Task] 3.40.0 or higher
* [uv] 0.7.0 or higher

## Set up
Run dependency installation task:
Expand Down Expand Up @@ -67,3 +68,4 @@ in the table below.

[spider-docs]: https://docs.yscope.com/spider/main/
[Task]: https://taskfile.dev
[uv]: https://docs.astral.sh/uv/
4 changes: 4 additions & 0 deletions build-tasks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,7 @@ tasks:
deps: [":config-cmake-project"]
cmds:
- "cmake --build {{.G_BUILD_SPIDER_DIR}} --target clean --parallel {{numCPU}}"

py-build:
cmds:
- "uv build --directory {{.G_SRC_PYTHON_DIR}} -o {{.G_BUILD_PYTHON_DIR}}"
17 changes: 9 additions & 8 deletions docs/src/dev-docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,12 @@ require this storage backend.

You can use the following tasks to run the set of unit tests that's appropriate.

| Task | Description |
|-------------------------------|-------------------------------------------------------------------|
| `test:all` | Runs all unit tests. |
| `test:non-storage-unit-tests` | Runs all unit tests which don't require a storage backend to run. |
| `test:storage-unit-tests` | Runs all unit tests which require a storage backend to run. |
| Task | Description |
|-------------------------------|-----------------------------------------------------------------------|
| `test:cpp-all` | Runs all C++ unit tests. |
| `test:non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. |
| `test:storage-unit-tests` | Runs all C++ unit tests which require a storage backend to run. |
| `test:python-tests` | Runs all Python tests. |

If any tests show error messages for the connection function below, revisit the
[setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly.
Expand All @@ -52,9 +53,9 @@ pull requests, and daily. Currently, it only runs unit tests that don't require

You can use the following tasks to run integration tests.

| Task | Description |
|-------------------------------|-------------------------------------------------------------------|
| `test:integration` | Runs all integration tests. |
| Task | Description |
|------------------------|---------------------------------|
| `test:cpp-integration` | Runs all C++ integration tests. |


[gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml
5 changes: 5 additions & 0 deletions lint-tasks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,11 @@ tasks:
mypy "{{.ITEM}}"
ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}"
ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}"
- cmd: |-
cd "{{.G_SRC_PYTHON_DIR}}"
uv run mypy .
uv run ruff check {{.RUFF_CHECK_FLAGS}}
uv run ruff format {{.RUFF_FORMAT_FLAGS}}

Comment thread
sitaowang1998 marked this conversation as resolved.
Resolved
yml:
aliases:
Expand Down
1 change: 1 addition & 0 deletions python/.python-version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3.13
Comment thread
sitaowang1998 marked this conversation as resolved.
Resolved
Empty file added python/README.md
Empty file.
77 changes: 77 additions & 0 deletions python/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
[project]
name = "spider"
version = "0.0.1"
description = "Spider is a distributed task execution framework"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"lark>=1.2.2",
"mariadb>=1.1.13",
"msgpack-types>=0.5.0",
]
Comment thread
sitaowang1998 marked this conversation as resolved.
Resolved

[project.scripts]
spider_task_executor = "spider.task_executor.task_executor:main"

[dependency-groups]
dev = [
"mypy>=1.12.0",
"pytest>=8.4.1",
"ruff>=0.4.4",
]

[build-system]
requires = ["hatchling>=1.18.0"]
build-backend = "hatchling.build"

Comment thread
sitaowang1998 marked this conversation as resolved.
Resolved
[tool.mypy]
strict = true
# Additional output
pretty = true
show_error_code_links = true
show_error_context = true

[tool.pytest.ini_options]
testpaths = ["tests"]

[tool.ruff]
line-length = 100

[tool.ruff.format]
docstring-code-format = true
docstring-code-line-length = 100

[tool.ruff.lint]
select = ["ALL"]
extend-select = ["PT"]
ignore = [
"COM812", # Redundant and conflicts with ruff format
"D203", # No blank line before docstrings (D211)
"D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists
"D212", # Enforce docstring summary line on the next line after quotes (D213)
"D400", # First line of docstrings may not end in period
"D401", # Docstrings should be written in present tense (not imperative)
"D415", # First line of docstrings may not end in a period, question mark, or exclamation point
"FA102", # Allow use of PEP 604 union in type annotations
"FBT", # Allow bool positional parameters since other value positions are allowed
"FIX002", # Allow todo statements
"PERF401", # Allow for loops when creating lists
"PERF403", # Allow for loops when creating dicts
"S311", # Allow usage of `random` package
"SIM102", # Allow collapsible if statements for readability
"TD002", # Author unnecessary for todo statement
"TD003", # Issue link unnecessary for todo statement
"UP015", # Explicit open modes are helpful
]
isort.order-by-type = false

[tool.ruff.lint.per-file-ignores]
"tests/**" = [
"INP001", # Allow implicit namespace package for tests
"S101", # Allow use of `assert` (security warning)
"S603", # Allow use of `subprocess.Popen` (security warning)
"T201", # Allow use of `print` (testing)
]

[tool.ruff.lint.pydocstyle]
ignore-decorators = ["typing.override"]
12 changes: 12 additions & 0 deletions python/src/spider/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
"""Spider package root."""

from spider.type import Double, Float, Int8, Int16, Int32, Int64

__all__ = [
"Double",
"Float",
"Int8",
"Int16",
"Int32",
"Int64",
]
1 change: 1 addition & 0 deletions python/src/spider/client/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Spider python client."""
1 change: 1 addition & 0 deletions python/src/spider/core/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Spider core package."""
5 changes: 5 additions & 0 deletions python/src/spider/core/task.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
"""Task module for Spider."""


class Task:
"""Represents a task in Spider."""
1 change: 1 addition & 0 deletions python/src/spider/task_executor/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Python task executor for running Python task function."""
9 changes: 9 additions & 0 deletions python/src/spider/task_executor/task_executor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
"""Executes a Spider Python task."""


def main() -> None:
"""Main function to execute the task."""


if __name__ == "__main__":
main()
6 changes: 6 additions & 0 deletions python/src/spider/type/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
"""Spider type package."""

from spider.type.tdl_convert import to_tdl_type_str
from spider.type.type import Double, Float, Int8, Int16, Int32, Int64

__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64", "to_tdl_type_str"]
101 changes: 101 additions & 0 deletions python/src/spider/type/tdl_convert.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
"""Converts native types to TDL types."""

from collections.abc import Collection
from types import GenericAlias
from typing import get_args, get_origin

from spider.type.tdl_type import (
BoolType,
ClassType,
DoubleType,
FloatType,
Int8Type,
Int16Type,
Int32Type,
Int64Type,
ListType,
MapType,
TdlType,
)
from spider.type.type import Double, Float, Int8, Int16, Int32, Int64
from spider.type.utils import get_class_name


def to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None:
"""
Converts a native type to primitive TDL type.
:param native_type:
:return: Converted TDL primitive. None if `native_type` is not a supported primitive type.
"""
tdl_type: TdlType | None = None
if native_type is Int8:
tdl_type = Int8Type()
elif native_type is Int16:
tdl_type = Int16Type()
elif native_type is Int32:
tdl_type = Int32Type()
elif native_type is Int64:
tdl_type = Int64Type()
elif native_type is Float:
tdl_type = FloatType()
elif native_type is Double:
tdl_type = DoubleType()
elif native_type is bool:
tdl_type = BoolType()
return tdl_type


def to_tdl_type(native_type: type | GenericAlias) -> TdlType:
"""
Converts a Python type to TDL type.
:param native_type:
:return:
:raise: TypeError if `native_type` is not a valid TDL type.
"""
primitive_tdl_type = to_primitive_tdl_type(native_type)
if primitive_tdl_type is not None:
return primitive_tdl_type

if native_type in (int, float, str, complex, bytes):
msg = f"{native_type} is not a valid TDL type."
raise TypeError(msg)

if isinstance(native_type, GenericAlias):
origin = get_origin(native_type)
if origin is list:
arg = get_args(native_type)
if arg is None:
msg = "List does not have a key type."
raise TypeError(msg)
arg = arg[0]
return ListType(to_tdl_type(arg))

if origin is dict:
arg = get_args(native_type)
msg = "Dict does not have a key/value type."
if arg is None:
raise TypeError(msg)
if len(arg) != 2: # noqa: PLR2004
raise TypeError(msg)
key = arg[0]
value = arg[1]
return MapType(to_tdl_type(key), to_tdl_type(value))

msg = f"{native_type} is not a valid TDL type."
raise TypeError(msg)

if issubclass(native_type, Collection):
msg = f"{native_type} is not a valid TDL type."
raise TypeError(msg)

return ClassType(get_class_name(native_type))


def to_tdl_type_str(native_type: type | GenericAlias) -> str:
"""
Converts a Python type to TDL type string.
:param native_type:
:return:
:raise: TypeError if `native_type` is not a valid TDL type.
"""
return to_tdl_type(native_type).type_str()
84 changes: 84 additions & 0 deletions python/src/spider/type/tdl_parse.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
"""Parse TDL type string."""

from lark import Lark, LarkError, Token, Transformer, v_args

from spider.type.tdl_type import (
BoolType,
ClassType,
DoubleType,
FloatType,
Int8Type,
Int16Type,
Int32Type,
Int64Type,
ListType,
MapType,
TdlType,
)

grammar = r"""
type: map_type | list_type | base_type

map_type: "Map" "<" type "," type ">"
list_type: "List" "<" type ">"
base_type: ID

ID: /[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*/

%import common.WS
%ignore WS
"""

primitive_type_map = {
"bool": BoolType,
"double": DoubleType,
"float": FloatType,
"int8": Int8Type,
"int16": Int16Type,
"int32": Int32Type,
"int64": Int64Type,
}


class TypeTransformer(Transformer[Token, TdlType]):
"""Transform Lark parse tree into TDL type."""

@v_args(inline=True)
def type(self, value: TdlType) -> TdlType:
"""Unwraps the type node to return the TdlType."""
return value

@v_args(inline=True)
def map_type(self, key: TdlType, value: TdlType) -> TdlType:
"""Transforms map node into Map type."""
return MapType(key, value)

@v_args(inline=True)
def list_type(self, key: TdlType) -> TdlType:
"""Transforms list node into List type."""
return ListType(key)

def base_type(self, children: list[Token]) -> TdlType:
"""Transforms primitive node into primitive type."""
name = str(children[0])
if name in primitive_type_map:
return primitive_type_map[name]() # type: ignore[abstract]
return ClassType(name)


parser = Lark(grammar, start="type", parser="lalr")


def parse_tdl_type(string: str) -> TdlType:
"""
Parses TDL type string into TDL type.
:param string: TDL type string.
:return: Parsed TDL type.
:raise: TypeError if string is not a valid TDL type.
"""
try:
tree = parser.parse(string)
return TypeTransformer(visit_tokens=False).transform(tree)
except LarkError as ecx:
msg = f"Cannot parse TDL type '{string}'"
raise TypeError(msg) from ecx
Loading