Set up uv workspace

.pre-commit-config.yaml

@@ -25,3 +25,18 @@ repos:
       pass_filenames: false
       args: [--python=.venv/]
       additional_dependencies: [ty]
+- repo: https://github.com/fpgmaas/deptry.git
+  rev: "0.23.0"
+  hooks:
+    - id: deptry
+      name: deptry for lynxkite-app
+      entry: bash -c 'cd lynxkite-app && deptry .'
+    - id: deptry
+      name: deptry for lynxkite-core
+      entry: bash -c 'cd lynxkite-core && deptry .'
+    - id: deptry
+      name: deptry for lynxkite-graph-analytics
+      entry: bash -c 'cd lynxkite-graph-analytics && deptry .'
+    - id: deptry
+      name: deptry for lynxkite-pillow-example
+      entry: bash -c 'cd lynxkite-pillow-example && deptry .'

docs/contributing.md

@@ -19,7 +19,7 @@ Install everything like this:
 uv venv
 source .venv/bin/activate
 uvx pre-commit install
-uv pip install -e 'lynxkite-core/[dev]' -e 'lynxkite-app/[dev]' -e 'lynxkite-graph-analytics/[dev]' -e lynxkite-pillow-example/
+uv sync
 ```
 
 This also builds the frontend, hopefully very quickly. To run it:
@@ -38,17 +38,16 @@ npm run dev
 
 ## Executing tests
 
-Run all tests with a single command, or look inside to see how to run them individually:
-
 ```bash
-./test.sh
+pytest                                # Runs all backend unit tests.
+pytest lynxkite-core                  # Runs tests for one package.
+cd lynxkite-app/web && npm run test   # Runs frontend tests.
 ```
 
 ## Documentation
 
 To work on the documentation:
 
 ```bash
-uv pip install mkdocs-material mkdocstrings[python]
 mkdocs serve
 ```

lynxkite-app/pyproject.toml

@@ -6,25 +6,28 @@ readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
     "fastapi[standard]>=0.115.6",
+    "griffe>=1.7.3",
+    "joblib>=1.5.1",
     "lynxkite-core",
-    "orjson>=3.10.13",
     "pycrdt-websocket>=0.16",
+    "pycrdt>=0.12.26",
+    "pydantic>=2.11.7",
     "sse-starlette>=2.2.1",
-    "griffe>=1.7.3",
+    "uvicorn>=0.35.0",
 ]
 classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"]
 
 [project.urls]
 Homepage = "https://github.com/lynxkite/lynxkite-2000/"
 
-[project.optional-dependencies]
+[dependency-groups]
 dev = [
     "pydantic-to-typescript>=2.0.0",
-    "pytest>=8.3.4",
+    "setuptools>=80.9.0",
 ]
 
 [tool.uv.sources]
-lynxkite-core = { path = "../lynxkite-core" }
+lynxkite-core = { workspace = true }
 
 [build-system]
 requires = ["setuptools", "wheel", "setuptools-scm"]
@@ -47,3 +50,11 @@ build_py = "build_frontend.build_py"
 
 [project.scripts]
 lynxkite = "lynxkite_app.__main__:main"
+
+[tool.deptry.package_module_name_map]
+lynxkite-core = "lynxkite"
+sse-starlette = "starlette"
+
+[tool.deptry.per_rule_ignores]
+DEP002 = ["pycrdt-websocket", "griffe"]
+DEP004 = ["setuptools"]

lynxkite-app/src/lynxkite_app/main.py

@@ -4,6 +4,7 @@
 import pydantic
 import fastapi
 import importlib
+import joblib
 import pathlib
 import pkgutil
 from fastapi.staticfiles import StaticFiles
@@ -13,11 +14,14 @@
 from lynxkite.core import workspace
 from . import crdt
 
+mem = joblib.Memory(".joblib-cache")
+ops.CACHE_WRAPPER = mem.cache
+
 
 def detect_plugins():
     plugins = {}
     for _, name, _ in pkgutil.iter_modules():
-        if name.startswith("lynxkite_"):
+        if name.startswith("lynxkite_") and name != "lynxkite_app":
             print(f"Importing {name}")
             plugins[name] = importlib.import_module(name)
     if not plugins:

lynxkite-core/pyproject.toml

@@ -5,16 +5,21 @@ description = "A lightweight dependency for authoring LynxKite operations and ex
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
+    "pydantic>=2.11.7",
 ]
 classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"]
 
 [project.urls]
 Homepage = "https://github.com/lynxkite/lynxkite-2000/"
 
-[project.optional-dependencies]
-dev = [
-    "pytest",
-]
+[tool.deptry.per_rule_ignores]
+DEP001 = ["matplotlib", "griffe", "pycrdt"]
+DEP003 = ["matplotlib", "griffe", "pycrdt"]
+
+[build-system]
+requires = ["setuptools", "wheel", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
 
-[tool.pytest.ini_options]
-asyncio_mode = "auto"
+[tool.setuptools.packages.find]
+namespaces = true
+where = ["src"]

lynxkite-core/src/lynxkite/core/executors/one_by_one.py

@@ -4,9 +4,6 @@
 
 from .. import ops
 from .. import workspace
-import orjson
-import pandas as pd
-import pydantic
 import traceback
 import inspect
 import typing
@@ -35,9 +32,6 @@ def _has_ctx(op):
     return "_ctx" in sig.parameters
 
 
-CACHES = {}
-
-
 def register(env: str, cache: bool = True):
     """Registers the one-by-one executor.
 
@@ -46,12 +40,7 @@ def register(env: str, cache: bool = True):
         from lynxkite.core.executors import one_by_one
         one_by_one.register("My Environment")
     """
-    if cache:
-        CACHES[env] = {}
-        _cache = CACHES[env]
-    else:
-        _cache = None
-    ops.EXECUTORS[env] = lambda ws: _execute(ws, ops.CATALOGS[env], cache=_cache)
+    ops.EXECUTORS[env] = lambda ws: _execute(ws, ops.CATALOGS[env])
 
 
 def _get_stages(ws, catalog: ops.Catalog):
@@ -83,28 +72,13 @@ def _get_stages(ws, catalog: ops.Catalog):
     return stages
 
 
-def _default_serializer(obj):
-    if isinstance(obj, pydantic.BaseModel):
-        return obj.dict()
-    return {"__nonserializable__": id(obj)}
-
-
-def _make_cache_key(obj):
-    return orjson.dumps(obj, default=_default_serializer)
-
-
-EXECUTOR_OUTPUT_CACHE = {}
-
-
 async def _await_if_needed(obj):
     if inspect.isawaitable(obj):
         return await obj
     return obj
 
 
-async def _execute(
-    ws: workspace.Workspace, catalog: ops.Catalog, cache: typing.Optional[dict] = None
-):
+async def _execute(ws: workspace.Workspace, catalog: ops.Catalog):
     nodes = {n.id: n for n in ws.nodes}
     contexts = {n.id: Context(node=n) for n in ws.nodes}
     edges = {n.id: [] for n in ws.nodes}
@@ -157,23 +131,15 @@ async def _execute(
                     if missing:
                         node.publish_error(f"Missing input: {', '.join(missing)}")
                         break
-                    if cache is not None:
-                        key = _make_cache_key((inputs, params))
-                        if key not in cache:
-                            result: ops.Result = op(*inputs, **params)
-                            result.output = await _await_if_needed(result.output)
-                            cache[key] = result
-                        result = cache[key]
-                    else:
-                        result = op(*inputs, **params)
+                    result = op(*inputs, **params)
                     output = await _await_if_needed(result.output)
                 except Exception as e:
                     traceback.print_exc()
                     node.publish_error(e)
                     break
                 contexts[node.id].last_result = output
                 # Returned lists and DataFrames are considered multiple tasks.
-                if isinstance(output, pd.DataFrame):
+                if hasattr(output, "to_dict"):
                     output = _df_to_list(output)
                 elif not isinstance(output, list):
                     output = [output]

lynxkite-core/src/lynxkite/core/ops.py

@@ -15,9 +15,7 @@
 import typing
 from dataclasses import dataclass
 
-import joblib
 import pydantic
-from typing_extensions import Annotated
 
 if typing.TYPE_CHECKING:
     from . import workspace
@@ -26,10 +24,17 @@
 Catalogs = dict[str, Catalog]
 CATALOGS: Catalogs = {}
 EXECUTORS = {}
-mem = joblib.Memory(".joblib-cache")
 
 typeof = type  # We have some arguments called "type".
 
+CACHE_WRAPPER = None  # Overwrite this to configure a caching mechanism.
+
+
+def _cache_wrap(func):
+    if CACHE_WRAPPER is None:
+        return func
+    return CACHE_WRAPPER(func)
+
 
 def type_to_json(t):
     if isinstance(t, type) and issubclass(t, enum.Enum):
@@ -39,10 +44,10 @@ def type_to_json(t):
     return {"type": str(t)}
 
 
-Type = Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)]
-LongStr = Annotated[str, {"format": "textarea"}]
+Type = typing.Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)]
+LongStr = typing.Annotated[str, {"format": "textarea"}]
 """LongStr is a string type for parameters that will be displayed as a multiline text area in the UI."""
-PathStr = Annotated[str, {"format": "path"}]
+PathStr = typing.Annotated[str, {"format": "path"}]
 # https://github.com/python/typing/issues/182#issuecomment-1320974824
 ReadOnlyJSON: typing.TypeAlias = (
     typing.Mapping[str, "ReadOnlyJSON"]
@@ -239,12 +244,12 @@ def compute_id(self):
 def op(
     env: str,
     *names: str,
-    view="basic",
-    outputs=None,
-    params=None,
-    slow=False,
-    color=None,
-    cache=None,
+    view: str = "basic",
+    outputs: list[str] | None = None,
+    params: list[Parameter] | None = None,
+    slow: bool = False,
+    color: str | None = None,
+    cache: bool | None = None,
 ):
     """
     Decorator for defining an operation.
@@ -275,7 +280,7 @@ def decorator(func):
         if slow:
             func = make_async(func)
             if cache is not False:
-                func = mem.cache(func)
+                func = _cache_wrap(func)
         # Positional arguments are inputs.
         inputs = [
             Input(name=name, type=param.annotation)

lynxkite-core/src/lynxkite/core/workspace.py

@@ -5,12 +5,12 @@
 import dataclasses
 import enum
 import os
-import pycrdt
 import pydantic
 import tempfile
 from . import ops
 
 if TYPE_CHECKING:
+    import pycrdt
     from lynxkite.core import ops
 
 
@@ -65,7 +65,7 @@ class WorkspaceNode(BaseConfig):
     position: Position
     width: Optional[float] = None
     height: Optional[float] = None
-    _crdt: Optional[pycrdt.Map] = None
+    _crdt: Optional["pycrdt.Map"] = None
 
     def publish_started(self):
         """Notifies the frontend that work has started on this node."""
@@ -118,7 +118,7 @@ class Workspace(BaseConfig):
     env: str = ""
     nodes: list[WorkspaceNode] = dataclasses.field(default_factory=list)
     edges: list[WorkspaceEdge] = dataclasses.field(default_factory=list)
-    _crdt: Optional[pycrdt.Map] = None
+    _crdt: Optional["pycrdt.Map"] = None
 
     def normalize(self):
         if self.env not in ops.CATALOGS:
@@ -219,7 +219,9 @@ def update_metadata(self):
                     node._crdt["data"]["meta"] = {}
                     node._crdt["data"]["error"] = "Unknown operation."
 
-    def connect_crdt(self, ws_crdt: pycrdt.Map):
+    def connect_crdt(self, ws_crdt: "pycrdt.Map"):
+        import pycrdt
+
         self._crdt = ws_crdt
         with ws_crdt.doc.transaction():
             for nc, np in zip(ws_crdt["nodes"], self.nodes):