From 20d727f532ac30833a56435d8855a1544ea2faad Mon Sep 17 00:00:00 2001 From: Daniel Darabos Date: Mon, 21 Jul 2025 01:34:33 +0200 Subject: [PATCH] Set up a "workspace" package, fix up dependencies, set up Deptry. --- .pre-commit-config.yaml | 15 +++++++ docs/contributing.md | 9 ++-- lynxkite-app/pyproject.toml | 21 +++++++--- lynxkite-app/src/lynxkite_app/main.py | 6 ++- lynxkite-core/pyproject.toml | 17 +++++--- .../src/lynxkite/core/executors/one_by_one.py | 42 ++----------------- lynxkite-core/src/lynxkite/core/ops.py | 31 ++++++++------ lynxkite-core/src/lynxkite/core/workspace.py | 10 +++-- lynxkite-graph-analytics/pyproject.toml | 29 ++++++++++--- lynxkite-pillow-example/pyproject.toml | 16 +++++-- pyproject.toml | 33 +++++++++++++++ test.sh | 6 --- 12 files changed, 148 insertions(+), 87 deletions(-) create mode 100644 pyproject.toml delete mode 100755 test.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8bdd2b3e..d04374e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,3 +25,18 @@ repos: pass_filenames: false args: [--python=.venv/] additional_dependencies: [ty] +- repo: https://github.com/fpgmaas/deptry.git + rev: "0.23.0" + hooks: + - id: deptry + name: deptry for lynxkite-app + entry: bash -c 'cd lynxkite-app && deptry .' + - id: deptry + name: deptry for lynxkite-core + entry: bash -c 'cd lynxkite-core && deptry .' + - id: deptry + name: deptry for lynxkite-graph-analytics + entry: bash -c 'cd lynxkite-graph-analytics && deptry .' + - id: deptry + name: deptry for lynxkite-pillow-example + entry: bash -c 'cd lynxkite-pillow-example && deptry .' diff --git a/docs/contributing.md b/docs/contributing.md index 9ca9d0f1..3a33fdc8 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -19,7 +19,7 @@ Install everything like this: uv venv source .venv/bin/activate uvx pre-commit install -uv pip install -e 'lynxkite-core/[dev]' -e 'lynxkite-app/[dev]' -e 'lynxkite-graph-analytics/[dev]' -e lynxkite-pillow-example/ +uv sync ``` This also builds the frontend, hopefully very quickly. To run it: @@ -38,10 +38,10 @@ npm run dev ## Executing tests -Run all tests with a single command, or look inside to see how to run them individually: - ```bash -./test.sh +pytest # Runs all backend unit tests. +pytest lynxkite-core # Runs tests for one package. +cd lynxkite-app/web && npm run test # Runs frontend tests. ``` ## Documentation @@ -49,6 +49,5 @@ Run all tests with a single command, or look inside to see how to run them indiv To work on the documentation: ```bash -uv pip install mkdocs-material mkdocstrings[python] mkdocs serve ``` diff --git a/lynxkite-app/pyproject.toml b/lynxkite-app/pyproject.toml index 43570e5f..cf40de55 100644 --- a/lynxkite-app/pyproject.toml +++ b/lynxkite-app/pyproject.toml @@ -6,25 +6,28 @@ readme = "README.md" requires-python = ">=3.11" dependencies = [ "fastapi[standard]>=0.115.6", + "griffe>=1.7.3", + "joblib>=1.5.1", "lynxkite-core", - "orjson>=3.10.13", "pycrdt-websocket>=0.16", + "pycrdt>=0.12.26", + "pydantic>=2.11.7", "sse-starlette>=2.2.1", - "griffe>=1.7.3", + "uvicorn>=0.35.0", ] classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"] [project.urls] Homepage = "https://github.com/lynxkite/lynxkite-2000/" -[project.optional-dependencies] +[dependency-groups] dev = [ "pydantic-to-typescript>=2.0.0", - "pytest>=8.3.4", + "setuptools>=80.9.0", ] [tool.uv.sources] -lynxkite-core = { path = "../lynxkite-core" } +lynxkite-core = { workspace = true } [build-system] requires = ["setuptools", "wheel", "setuptools-scm"] @@ -47,3 +50,11 @@ build_py = "build_frontend.build_py" [project.scripts] lynxkite = "lynxkite_app.__main__:main" + +[tool.deptry.package_module_name_map] +lynxkite-core = "lynxkite" +sse-starlette = "starlette" + +[tool.deptry.per_rule_ignores] +DEP002 = ["pycrdt-websocket", "griffe"] +DEP004 = ["setuptools"] diff --git a/lynxkite-app/src/lynxkite_app/main.py b/lynxkite-app/src/lynxkite_app/main.py index 5facea36..75a28dc9 100644 --- a/lynxkite-app/src/lynxkite_app/main.py +++ b/lynxkite-app/src/lynxkite_app/main.py @@ -4,6 +4,7 @@ import pydantic import fastapi import importlib +import joblib import pathlib import pkgutil from fastapi.staticfiles import StaticFiles @@ -13,11 +14,14 @@ from lynxkite.core import workspace from . import crdt +mem = joblib.Memory(".joblib-cache") +ops.CACHE_WRAPPER = mem.cache + def detect_plugins(): plugins = {} for _, name, _ in pkgutil.iter_modules(): - if name.startswith("lynxkite_"): + if name.startswith("lynxkite_") and name != "lynxkite_app": print(f"Importing {name}") plugins[name] = importlib.import_module(name) if not plugins: diff --git a/lynxkite-core/pyproject.toml b/lynxkite-core/pyproject.toml index 55d6fab0..43541d25 100644 --- a/lynxkite-core/pyproject.toml +++ b/lynxkite-core/pyproject.toml @@ -5,16 +5,21 @@ description = "A lightweight dependency for authoring LynxKite operations and ex readme = "README.md" requires-python = ">=3.11" dependencies = [ + "pydantic>=2.11.7", ] classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"] [project.urls] Homepage = "https://github.com/lynxkite/lynxkite-2000/" -[project.optional-dependencies] -dev = [ - "pytest", -] +[tool.deptry.per_rule_ignores] +DEP001 = ["matplotlib", "griffe", "pycrdt"] +DEP003 = ["matplotlib", "griffe", "pycrdt"] + +[build-system] +requires = ["setuptools", "wheel", "setuptools-scm"] +build-backend = "setuptools.build_meta" -[tool.pytest.ini_options] -asyncio_mode = "auto" +[tool.setuptools.packages.find] +namespaces = true +where = ["src"] diff --git a/lynxkite-core/src/lynxkite/core/executors/one_by_one.py b/lynxkite-core/src/lynxkite/core/executors/one_by_one.py index e0eeebb3..6aeb39f6 100644 --- a/lynxkite-core/src/lynxkite/core/executors/one_by_one.py +++ b/lynxkite-core/src/lynxkite/core/executors/one_by_one.py @@ -4,9 +4,6 @@ from .. import ops from .. import workspace -import orjson -import pandas as pd -import pydantic import traceback import inspect import typing @@ -35,9 +32,6 @@ def _has_ctx(op): return "_ctx" in sig.parameters -CACHES = {} - - def register(env: str, cache: bool = True): """Registers the one-by-one executor. @@ -46,12 +40,7 @@ def register(env: str, cache: bool = True): from lynxkite.core.executors import one_by_one one_by_one.register("My Environment") """ - if cache: - CACHES[env] = {} - _cache = CACHES[env] - else: - _cache = None - ops.EXECUTORS[env] = lambda ws: _execute(ws, ops.CATALOGS[env], cache=_cache) + ops.EXECUTORS[env] = lambda ws: _execute(ws, ops.CATALOGS[env]) def _get_stages(ws, catalog: ops.Catalog): @@ -83,28 +72,13 @@ def _get_stages(ws, catalog: ops.Catalog): return stages -def _default_serializer(obj): - if isinstance(obj, pydantic.BaseModel): - return obj.dict() - return {"__nonserializable__": id(obj)} - - -def _make_cache_key(obj): - return orjson.dumps(obj, default=_default_serializer) - - -EXECUTOR_OUTPUT_CACHE = {} - - async def _await_if_needed(obj): if inspect.isawaitable(obj): return await obj return obj -async def _execute( - ws: workspace.Workspace, catalog: ops.Catalog, cache: typing.Optional[dict] = None -): +async def _execute(ws: workspace.Workspace, catalog: ops.Catalog): nodes = {n.id: n for n in ws.nodes} contexts = {n.id: Context(node=n) for n in ws.nodes} edges = {n.id: [] for n in ws.nodes} @@ -157,15 +131,7 @@ async def _execute( if missing: node.publish_error(f"Missing input: {', '.join(missing)}") break - if cache is not None: - key = _make_cache_key((inputs, params)) - if key not in cache: - result: ops.Result = op(*inputs, **params) - result.output = await _await_if_needed(result.output) - cache[key] = result - result = cache[key] - else: - result = op(*inputs, **params) + result = op(*inputs, **params) output = await _await_if_needed(result.output) except Exception as e: traceback.print_exc() @@ -173,7 +139,7 @@ async def _execute( break contexts[node.id].last_result = output # Returned lists and DataFrames are considered multiple tasks. - if isinstance(output, pd.DataFrame): + if hasattr(output, "to_dict"): output = _df_to_list(output) elif not isinstance(output, list): output = [output] diff --git a/lynxkite-core/src/lynxkite/core/ops.py b/lynxkite-core/src/lynxkite/core/ops.py index 47c193be..0fd395a9 100644 --- a/lynxkite-core/src/lynxkite/core/ops.py +++ b/lynxkite-core/src/lynxkite/core/ops.py @@ -15,9 +15,7 @@ import typing from dataclasses import dataclass -import joblib import pydantic -from typing_extensions import Annotated if typing.TYPE_CHECKING: from . import workspace @@ -26,10 +24,17 @@ Catalogs = dict[str, Catalog] CATALOGS: Catalogs = {} EXECUTORS = {} -mem = joblib.Memory(".joblib-cache") typeof = type # We have some arguments called "type". +CACHE_WRAPPER = None # Overwrite this to configure a caching mechanism. + + +def _cache_wrap(func): + if CACHE_WRAPPER is None: + return func + return CACHE_WRAPPER(func) + def type_to_json(t): if isinstance(t, type) and issubclass(t, enum.Enum): @@ -39,10 +44,10 @@ def type_to_json(t): return {"type": str(t)} -Type = Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)] -LongStr = Annotated[str, {"format": "textarea"}] +Type = typing.Annotated[typing.Any, pydantic.PlainSerializer(type_to_json, return_type=dict)] +LongStr = typing.Annotated[str, {"format": "textarea"}] """LongStr is a string type for parameters that will be displayed as a multiline text area in the UI.""" -PathStr = Annotated[str, {"format": "path"}] +PathStr = typing.Annotated[str, {"format": "path"}] # https://github.com/python/typing/issues/182#issuecomment-1320974824 ReadOnlyJSON: typing.TypeAlias = ( typing.Mapping[str, "ReadOnlyJSON"] @@ -239,12 +244,12 @@ def compute_id(self): def op( env: str, *names: str, - view="basic", - outputs=None, - params=None, - slow=False, - color=None, - cache=None, + view: str = "basic", + outputs: list[str] | None = None, + params: list[Parameter] | None = None, + slow: bool = False, + color: str | None = None, + cache: bool | None = None, ): """ Decorator for defining an operation. @@ -275,7 +280,7 @@ def decorator(func): if slow: func = make_async(func) if cache is not False: - func = mem.cache(func) + func = _cache_wrap(func) # Positional arguments are inputs. inputs = [ Input(name=name, type=param.annotation) diff --git a/lynxkite-core/src/lynxkite/core/workspace.py b/lynxkite-core/src/lynxkite/core/workspace.py index 484c7fff..e3e7bed8 100644 --- a/lynxkite-core/src/lynxkite/core/workspace.py +++ b/lynxkite-core/src/lynxkite/core/workspace.py @@ -5,12 +5,12 @@ import dataclasses import enum import os -import pycrdt import pydantic import tempfile from . import ops if TYPE_CHECKING: + import pycrdt from lynxkite.core import ops @@ -65,7 +65,7 @@ class WorkspaceNode(BaseConfig): position: Position width: Optional[float] = None height: Optional[float] = None - _crdt: Optional[pycrdt.Map] = None + _crdt: Optional["pycrdt.Map"] = None def publish_started(self): """Notifies the frontend that work has started on this node.""" @@ -118,7 +118,7 @@ class Workspace(BaseConfig): env: str = "" nodes: list[WorkspaceNode] = dataclasses.field(default_factory=list) edges: list[WorkspaceEdge] = dataclasses.field(default_factory=list) - _crdt: Optional[pycrdt.Map] = None + _crdt: Optional["pycrdt.Map"] = None def normalize(self): if self.env not in ops.CATALOGS: @@ -219,7 +219,9 @@ def update_metadata(self): node._crdt["data"]["meta"] = {} node._crdt["data"]["error"] = "Unknown operation." - def connect_crdt(self, ws_crdt: pycrdt.Map): + def connect_crdt(self, ws_crdt: "pycrdt.Map"): + import pycrdt + self._crdt = ws_crdt with ws_crdt.doc.transaction(): for nc, np in zip(ws_crdt["nodes"], self.nodes): diff --git a/lynxkite-graph-analytics/pyproject.toml b/lynxkite-graph-analytics/pyproject.toml index e648d66d..1ee16c89 100644 --- a/lynxkite-graph-analytics/pyproject.toml +++ b/lynxkite-graph-analytics/pyproject.toml @@ -5,20 +5,23 @@ description = "The graph analytics executor and boxes for LynxKite" readme = "README.md" requires-python = ">=3.11" dependencies = [ + "cudf-cu12>=25.6.0", "fsspec>=2025.3.2", "grand-cypher>=0.13.0", - "joblib>=1.4.2", "lynxkite-core", "matplotlib>=3.10.1", "networkx[default]>=3.4.2", "numba>=0.61.2", + "numpy>=2.2.6", "osmnx>=2.0.2", "pandas>=2.2.3", "polars>=1.25.2", "pyarrow>=19.0.1", + "pydantic>=2.11.7", "torch>=2.7.0", "torch-geometric>=2.6.1", "torchdiffeq>=0.2.5", + "tqdm>=4.67.1", "umap-learn>=0.5.9.post2", ] classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"] @@ -27,10 +30,6 @@ classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 Homepage = "https://github.com/lynxkite/lynxkite-2000/" [project.optional-dependencies] -dev = [ - "pytest>=8.3.5", - "pytest-asyncio>=0.26.0", -] gpu = [ "cuml-cu12>=25.2.1", "nx-cugraph-cu12>=25.4.0", @@ -39,7 +38,7 @@ gpu = [ ] [tool.uv.sources] -lynxkite-core = { path = "../lynxkite-core" } +lynxkite-core = { workspace = true } pylibcugraph-cu12 = { index = "nvidia" } [tool.pytest.ini_options] @@ -48,3 +47,21 @@ asyncio_mode = "auto" [[tool.uv.index]] name = "nvidia" url = "https://pypi.nvidia.com" + +[tool.deptry.per_rule_ignores] +DEP002 = ["numba", "pyarrow", "nx-cugraph-cu12", "pylibcugraph-cu12"] + +[tool.deptry.package_module_name_map] +grand-cypher = "grandcypher" +lynxkite-core = "lynxkite" +umap-learn = "umap" +cuml-cu12 = "cuml" +cudf-cu12 = "cudf" + +[build-system] +requires = ["setuptools", "wheel", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +namespaces = true +where = ["src"] diff --git a/lynxkite-pillow-example/pyproject.toml b/lynxkite-pillow-example/pyproject.toml index 6b9dccab..af146534 100644 --- a/lynxkite-pillow-example/pyproject.toml +++ b/lynxkite-pillow-example/pyproject.toml @@ -8,8 +8,6 @@ dependencies = [ "fsspec>=2025.2.0", "lynxkite-core", "pillow>=11.1.0", - "requests>=2.32.3", - "aiohttp>=3.11.11", ] classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)"] @@ -17,4 +15,16 @@ classifiers = ["License :: OSI Approved :: GNU Affero General Public License v3 Homepage = "https://github.com/lynxkite/lynxkite-2000/" [tool.uv.sources] -lynxkite-core = { path = "../lynxkite-core" } +lynxkite-core = { workspace = true } + +[tool.deptry.package_module_name_map] +lynxkite-core = "lynxkite" +pillow = "PIL" + +[build-system] +requires = ["setuptools", "wheel", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +namespaces = true +where = ["src"] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8a22daf0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +# This is the workspace configuration. Each individual package has its own pyproject.toml. +[project] +name = "lynxkite-workspace" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = ["lynxkite", "lynxkite-core", "lynxkite-graph-analytics[gpu]", "lynxkite-pillow-example"] +classifiers = ["Private :: Do Not Upload"] + +[tool.uv.sources] +lynxkite = { workspace = true } +lynxkite-core = { workspace = true } +lynxkite-graph-analytics = { workspace = true } +lynxkite-pillow-example = { workspace = true } + +[tool.uv.workspace] +members = [ + "lynxkite-app", + "lynxkite-core", + "lynxkite-graph-analytics", + "lynxkite-pillow-example", +] + +[dependency-groups] +dev = [ + "deptry>=0.23.0", + "mkdocs-material>=9.6.15", + "mkdocstrings[python]>=0.29.1", + "pytest>=8.4.1", + "pytest-asyncio>=1.1.0", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/test.sh b/test.sh deleted file mode 100755 index 1f30c1e8..00000000 --- a/test.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -xue - -cd "$(dirname $0)" -pytest --asyncio-mode=auto -cd lynxkite-app/web -npm run test