diff --git a/.github/wordlist.txt b/.github/wordlist.txt index 5eacb0df..dc670b7e 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -120,4 +120,5 @@ pylint pytest Radix Zod +SDK Dependabot diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index f5700ecc..1ee961ea 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -53,7 +53,7 @@ jobs: # Install Python dependencies - name: Install Python dependencies - run: uv sync --locked + run: uv sync --locked --all-extras # Install Node dependencies (root - for Playwright) - name: Install root dependencies diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 90c1b01b..c0963d66 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -23,7 +23,7 @@ jobs: - name: Install dependencies run: | - uv sync + uv sync --locked --all-extras - name: Run pylint run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4107b985..da8f00a2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,7 +43,7 @@ jobs: - name: Install dependencies run: | - uv sync --locked + uv sync --locked --all-extras - name: Install frontend dependencies run: | @@ -63,8 +63,81 @@ jobs: - name: Run unit tests run: | - uv run python -m pytest tests/ -k "not e2e" --verbose + uv run python -m pytest tests/ -k "not e2e and not test_sdk" --verbose - name: Run lint run: | make lint + + sdk-tests: + runs-on: ubuntu-latest + + services: + falkordb: + image: falkordb/falkordb:latest + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + postgres: + image: postgres:15 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: testdb + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 + with: + version: "latest" + + - name: Install dependencies + run: | + uv sync --locked --all-extras + + - name: Create test environment file + run: | + cp .env.example .env + echo "FASTAPI_SECRET_KEY=test-secret-key" >> .env + echo "FALKORDB_URL=redis://localhost:6379" >> .env + + - name: Run SDK tests + env: + FALKORDB_URL: redis://localhost:6379 + TEST_POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/testdb + TEST_MYSQL_URL: mysql://root:root@localhost:3306/testdb + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + uv run python -m pytest tests/test_sdk/ -v diff --git a/Dockerfile b/Dockerfile index f7ae3720..4e6ea5b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ ENV UV_SYSTEM_PYTHON=1 ENV PATH="/app/.venv/bin:$PATH" # Install Python dependencies only (project itself installed after COPY) -RUN uv sync --frozen --no-dev --no-install-project +RUN uv sync --frozen --no-dev --extra server --no-install-project # Install Node.js (Node 22) so we can build the frontend inside the image. # Use NodeSource setup script to get a recent Node version on Debian-based images. @@ -78,7 +78,7 @@ RUN npm --prefix ./app run build COPY . . # Install the project package now that source code is available -RUN uv sync --frozen --no-dev +RUN uv sync --frozen --no-dev --extra server # Copy and make start.sh executable COPY start.sh /start.sh diff --git a/Makefile b/Makefile index 54b5ac9a..b8dbc443 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ -.PHONY: help install test test-unit test-e2e test-e2e-headed lint format clean setup-dev build lint-frontend +.PHONY: help install test test-unit test-e2e test-e2e-headed lint format clean setup-dev build lint-frontend test-sdk docker-test-services docker-test-stop build-package help: ## Show this help message @echo 'Usage: make [target]' @echo '' @echo 'Targets:' - @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-20s %s\n", $$1, $$2}' $(MAKEFILE_LIST) install: ## Install dependencies uv sync @@ -23,10 +23,14 @@ build-dev: build-prod: npm --prefix ./app run build +build-package: ## Build distributable package (wheel + sdist) + uv build + @echo "Built packages in dist/" + test: build-dev test-unit test-e2e ## Run all tests -test-unit: ## Run unit tests only - uv run python -m pytest tests/ -k "not e2e" --verbose +test-unit: ## Run unit tests only (excludes SDK and E2E tests) + uv run python -m pytest tests/ -k "not e2e and not test_sdk" --ignore=tests/test_sdk --verbose test-e2e: build-dev ## Run E2E tests headless @@ -57,6 +61,8 @@ clean: ## Clean up test artifacts rm -rf playwright-report/ rm -rf tests/e2e/screenshots/ rm -rf __pycache__/ + rm -rf dist/ + rm -rf *.egg-info/ find . -name "*.pyc" -delete find . -name "*.pyo" -delete @@ -72,3 +78,20 @@ docker-falkordb: ## Start FalkorDB in Docker for testing docker-stop: ## Stop test containers docker stop falkordb-test || true docker rm falkordb-test || true + +# SDK Testing +docker-test-services: ## Start all test services (FalkorDB + PostgreSQL + MySQL) + docker compose -f docker-compose.test.yml up -d + @echo "Waiting for services to be ready..." + @sleep 10 + +docker-test-stop: ## Stop all test services + docker compose -f docker-compose.test.yml down -v + +test-sdk: ## Run SDK integration tests (requires docker-test-services) + uv run python -m pytest tests/test_sdk/ -v + +test-sdk-quick: ## Run SDK tests without LLM (models and connection only) + uv run python -m pytest tests/test_sdk/test_queryweaver.py::TestModels tests/test_sdk/test_queryweaver.py::TestQueryWeaverInit -v + +test-all: test-unit test-sdk test-e2e ## Run all tests diff --git a/README.md b/README.md index 6d0f1a49..3f3c6273 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,124 @@ Notes & tips - The streaming response includes intermediate reasoning steps, follow-up questions (if the query is ambiguous or off-topic), and the final SQL. The frontend expects the boundary string `|||FALKORDB_MESSAGE_BOUNDARY|||` between messages. - For destructive SQL (INSERT/UPDATE/DELETE etc) the service will include a confirmation step in the stream; the frontend handles this flow. If you automate destructive operations, ensure you handle confirmation properly (see the `ConfirmRequest` model in the code). +## Python SDK + +The QueryWeaver Python SDK allows you to use Text2SQL functionality directly in your Python applications **without running a web server**. + +### Installation + +```bash +# SDK only (minimal dependencies) +pip install queryweaver + +# With server dependencies (FastAPI, etc.) +pip install queryweaver[server] + +# Development (includes testing tools) +pip install queryweaver[dev] +``` + +### Quick Start + +```python +import asyncio +from queryweaver import QueryWeaver + +async def main(): + # Initialize with FalkorDB connection + qw = QueryWeaver(falkordb_url="redis://localhost:6379") + + # Connect a PostgreSQL or MySQL database + conn = await qw.connect_database("postgresql://user:pass@host:5432/mydb") + print(f"Connected: {conn.database_id}") # "mydb" + + # Convert natural language to SQL and execute — pass the database_id + # returned by connect_database (un-prefixed; namespacing is internal). + result = await qw.query(conn.database_id, "Show me all customers from NYC") + print(result.sql_query) # SELECT * FROM customers WHERE city = 'NYC' + print(result.results) # [{"id": 1, "name": "Alice", "city": "NYC"}, ...] + print(result.ai_response) # "Found 42 customers from NYC..." + + await qw.close() + +asyncio.run(main()) +``` + +### Context Manager + +```python +async with QueryWeaver(falkordb_url="redis://localhost:6379") as qw: + conn = await qw.connect_database("postgresql://user:pass@host/mydb") + result = await qw.query(conn.database_id, "Count orders by status") +# close() runs automatically, awaiting any in-flight background memory writes. +``` + +### Multiple Instances + +Multiple `QueryWeaver` instances can run side-by-side in the same process. +Each holds its own FalkorDB connection and passes it explicitly through +every call, so there is no shared global state to collide over. + +```python +async with QueryWeaver(falkordb_url="redis://host-a:6379", user_id="tenant_a") as a, \ + QueryWeaver(falkordb_url="redis://host-b:6379", user_id="tenant_b") as b: + sales = await a.connect_database("postgresql://user:pass@host-a/sales") + ops = await b.connect_database("postgresql://user:pass@host-b/ops") + await a.query(sales.database_id, "Show top customers") + await b.query(ops.database_id, "Count open tickets") +``` + +### Available Methods + +| Method | Description | +|--------|-------------| +| `connect_database(db_url)` | Connect PostgreSQL/MySQL and load schema | +| `query(database, question)` | Convert natural language to SQL and execute | +| `get_schema(database)` | Retrieve database schema (tables and relationships) | +| `list_databases()` | List all connected databases | +| `delete_database(database)` | Remove database from FalkorDB | +| `refresh_schema(database)` | Re-sync schema after database changes | +| `execute_confirmed(database, sql)` | Execute confirmed destructive operations | + +### Advanced Query Options + +For multi-turn conversations, custom instructions, or per-request LLM overrides: + +```python +from queryweaver import QueryWeaver, QueryRequest + +request = QueryRequest( + question="Show their recent orders", + chat_history=["Show all customers from NYC"], + result_history=["Found 42 customers..."], + instructions="Use created_at for date filtering", + # Optional per-request LLM overrides — bypass env-based config + custom_api_key="sk-...", + custom_model="openai/gpt-4.1", +) + +result = await qw.query("mydb", request) +``` + +### Handling Destructive Operations + +INSERT, UPDATE, DELETE operations require confirmation: + +```python +result = await qw.query("mydb", "Delete inactive users") + +if result.requires_confirmation: + print(f"Destructive SQL: {result.sql_query}") + # Execute after user confirms + confirmed = await qw.execute_confirmed("mydb", result.sql_query) +``` + +### Requirements + +- Python 3.12+ +- FalkorDB instance (local or remote) +- OpenAI or Azure OpenAI API key (for LLM) +- Target SQL database (PostgreSQL or MySQL) ## Development diff --git a/api/core/__init__.py b/api/core/__init__.py index 25e418c5..f1f9d1db 100644 --- a/api/core/__init__.py +++ b/api/core/__init__.py @@ -8,7 +8,14 @@ from .errors import InternalError, GraphNotFoundError, InvalidArgumentError from .schema_loader import load_database, list_databases -from .text2sql import MESSAGE_DELIMITER +from .pipeline import ( + MESSAGE_DELIMITER, + graph_name, + get_database_type_and_loader, + sanitize_query, + sanitize_log_input, + is_general_graph, +) __all__ = [ "InternalError", @@ -17,4 +24,9 @@ "load_database", "list_databases", "MESSAGE_DELIMITER", + "graph_name", + "get_database_type_and_loader", + "sanitize_query", + "sanitize_log_input", + "is_general_graph", ] diff --git a/api/core/db_resolver.py b/api/core/db_resolver.py new file mode 100644 index 00000000..2c653c8d --- /dev/null +++ b/api/core/db_resolver.py @@ -0,0 +1,25 @@ +"""Resolve a FalkorDB handle, falling back to the server-side singleton. + +Core text2sql functions accept an optional ``db`` parameter so the SDK can +inject its own connection without mutating process globals. When ``db`` is +None (route handlers that haven't threaded it yet), we lazily import the +module-level singleton from ``api.extensions``. The import is deferred so +the SDK can use this module without triggering ``api.extensions``'s +import-time FalkorDB connect. +""" + +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + # Import only for type checking — avoids pulling falkordb at runtime when + # callers pass an explicit handle and never need the server default. + from falkordb.asyncio import FalkorDB + + +def resolve_db(db: Optional["FalkorDB"] = None) -> "FalkorDB": + """Return the given ``db`` handle, or lazily import the server default.""" + if db is not None: + return db + # pylint: disable=import-outside-toplevel + from api.extensions import db as _default_db + return _default_db diff --git a/api/core/pipeline.py b/api/core/pipeline.py new file mode 100644 index 00000000..643b8075 --- /dev/null +++ b/api/core/pipeline.py @@ -0,0 +1,428 @@ +"""Shared logic for text2sql streaming and SDK (sync) paths. + +This module contains pure functions and constants extracted from +``text2sql.py`` (canonical source) so that both the streaming API and the +SDK non-streaming path stay in sync. +""" + +import asyncio +import contextvars +import logging +import os +from typing import Any, Optional, Type + +from api.agents import ResponseFormatterAgent +from api.config import Config +from api.core.db_resolver import resolve_db +from api.core.errors import InvalidArgumentError +from api.loaders.postgres_loader import PostgresLoader +from api.loaders.mysql_loader import MySQLLoader +from api.loaders.base_loader import BaseLoader +from api.sql_utils import SQLIdentifierQuoter, DatabaseSpecificQuoter + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +# Delimiter used by the streaming route to frame JSON messages on the wire. +# Kept here so any caller that composes streaming payloads pulls the single +# source of truth rather than redefining it. +MESSAGE_DELIMITER = "|||FALKORDB_MESSAGE_BOUNDARY|||" + +GENERAL_PREFIX = os.getenv("GENERAL_PREFIX") + +# Verb → user-facing description for destructive operations. Single source of +# truth for both ``DESTRUCTIVE_OPS`` (membership test) and the confirmation +# message builder, so adding a verb in one place can't drift from the other. +_DESTRUCTIVE_VERBS = { + 'INSERT': 'Add new data to the database', + 'UPDATE': 'Modify existing data in the database', + 'DELETE': '**PERMANENTLY DELETE** data from the database', + 'DROP': '**PERMANENTLY DELETE** entire tables or database objects', + 'CREATE': 'Create new tables or database objects', + 'ALTER': 'Modify the structure of existing tables', + 'TRUNCATE': '**PERMANENTLY DELETE ALL DATA** from specified tables', +} + +DESTRUCTIVE_OPS = frozenset(_DESTRUCTIVE_VERBS) + +# Contextvar-scoped task sink. SDK code sets this for the duration of a +# query/execute call so ``save_memory_background`` (fire-and-forget) can +# be awaited at ``QueryWeaver.close()`` time. Unset in server contexts, +# where the event loop outlives the query and tasks drain naturally. +background_tasks_var: contextvars.ContextVar[Optional[set]] = ( + contextvars.ContextVar("queryweaver_background_tasks", default=None) +) + +# --------------------------------------------------------------------------- +# Graph helpers +# --------------------------------------------------------------------------- + + +def graph_name(user_id: str, graph_id: str) -> str: + """Return the namespaced graph name. + + Applies validation identical to the original ``_graph_name`` in + ``text2sql.py``: strip, truncate to 200 chars, reject empty, bypass + prefix for general/demo graphs. + + Raises: + InvalidArgumentError: If *graph_id* is empty after stripping. + """ + graph_id = graph_id.strip()[:200] + if not graph_id: + # Bad input is a 400, not a 404 — several callers map + # InvalidArgumentError → 400 in the HTTP layer. + raise InvalidArgumentError( + "Invalid graph_id, must be a non-empty string up to 200 characters." + ) + + if GENERAL_PREFIX and graph_id.startswith(GENERAL_PREFIX): + return graph_id + + return f"{user_id}_{graph_id}" + + +def is_general_graph(graph_id: str) -> bool: + """Return ``True`` when *graph_id* belongs to a demo/general graph.""" + return bool(GENERAL_PREFIX and graph_id.startswith(GENERAL_PREFIX)) + + +# --------------------------------------------------------------------------- +# Database type detection +# --------------------------------------------------------------------------- + + +def get_database_type_and_loader( + db_url: str, + *, + sdk_only: bool = False, +) -> tuple[Optional[str], Optional[Type[BaseLoader]]]: + """Determine database type from *db_url* and return the loader class. + + Performs null/empty check, case-insensitive matching and defaults to + PostgreSQL for backward compatibility on the server path. + + When ``sdk_only`` is True, raises ``InvalidArgumentError`` for vendors + that need the ``[server]`` extra (snowflake) or for unknown URL schemes, + so SDK callers get a clean error instead of a deferred ``ImportError``. + """ + if not db_url or db_url == "No URL available for this database.": + return None, None + + db_url_lower = db_url.lower() + + if db_url_lower.startswith('postgresql://') or db_url_lower.startswith('postgres://'): + return 'postgresql', PostgresLoader + if db_url_lower.startswith('mysql://'): + return 'mysql', MySQLLoader + if db_url_lower.startswith('snowflake://'): + if sdk_only: + raise InvalidArgumentError( + "Snowflake requires the [server] extra: " + "pip install queryweaver[server]" + ) + # Lazy-import: snowflake-connector-python is in the [server] extra, + # not in the core SDK install. + # pylint: disable=import-outside-toplevel + from api.loaders.snowflake_loader import SnowflakeLoader + return 'snowflake', SnowflakeLoader + + if sdk_only: + raise InvalidArgumentError( + "Invalid database URL format. Must be PostgreSQL or MySQL." + ) + # Server path keeps the historical default-to-PostgreSQL fallback. + return 'postgresql', PostgresLoader + + +def validate_custom_model(custom_model: Optional[str]) -> None: + """Validate the ``vendor/model`` format and supported vendor list. + + Raises: + InvalidArgumentError: If the format is wrong or the vendor is unsupported. + """ + if not custom_model: + return + # Lazy-import: SUPPORTED_VENDORS lives in api.config which pulls server-only + # symbols. Keeping the import here means the SDK doesn't need it at import time. + # pylint: disable=import-outside-toplevel + from api.config import SUPPORTED_VENDORS + parts = custom_model.split("/", 1) + if len(parts) != 2 or not parts[0] or not parts[1]: + raise InvalidArgumentError( + "Invalid model format. Expected 'vendor/model' (e.g. 'openai/gpt-4.1')" + ) + if parts[0] not in SUPPORTED_VENDORS: + raise InvalidArgumentError( + f"Unsupported vendor '{parts[0]}'. Supported: {', '.join(SUPPORTED_VENDORS)}" + ) + + +# --------------------------------------------------------------------------- +# Input sanitisation +# --------------------------------------------------------------------------- + + +def sanitize_query(query: str) -> str: + """Sanitize *query* for safe usage — strips newlines and truncates to 500 chars.""" + return query.replace('\n', ' ').replace('\r', ' ')[:500] + + +def sanitize_log_input(value: str) -> str: + """Sanitize *value* for safe logging — removes newlines, CRs, and tabs.""" + if not isinstance(value, str): + value = str(value) + return value.replace('\n', ' ').replace('\r', ' ').replace('\t', ' ') + + +def truncate_for_log(query: str, max_length: int = 200) -> str: + """Truncate *query* for compact log messages (SDK path).""" + if len(query) > max_length: + return query[:max_length] + "..." + return query + + +# --------------------------------------------------------------------------- +# SQL analysis helpers +# --------------------------------------------------------------------------- + + +def _strip_sql_comments_and_whitespace(sql_query: str) -> str: + """Strip leading SQL comments (-- line and /* block */) and whitespace. + + A naive ``strip().split()[0]`` lets ``-- evil\\nDROP TABLE x`` masquerade + as a non-destructive statement, bypassing confirmation. + """ + text = sql_query.lstrip() + while text: + if text.startswith("--"): + newline = text.find("\n") + if newline == -1: + return "" + text = text[newline + 1:].lstrip() + elif text.startswith("/*"): + end = text.find("*/") + if end == -1: + return "" + text = text[end + 2:].lstrip() + else: + break + return text + + +def detect_destructive_operation(sql_query: str) -> tuple[str, bool]: + """Return ``(sql_type, is_destructive)`` for a SQL statement. + + Strips leading SQL comments before classifying so attackers cannot + bypass destructive-op confirmation by prefixing a comment. + """ + if not sql_query: + return "", False + cleaned = _strip_sql_comments_and_whitespace(sql_query) + sql_type = cleaned.split()[0].upper() if cleaned else "" + return sql_type, sql_type in DESTRUCTIVE_OPS + + +def auto_quote_sql_identifiers( + sql_query: str, + known_tables: set, + db_type: Optional[str], +) -> tuple[str, bool]: + """Auto-quote table names containing special characters. + + Returns ``(sanitized_sql, was_modified)``. + """ + quote_char = DatabaseSpecificQuoter.get_quote_char(db_type or 'postgresql') + return SQLIdentifierQuoter.auto_quote_identifiers( + sql_query, known_tables, quote_char + ) + + +def check_schema_modification( + sql_query: str, + loader_class: Type[BaseLoader], +) -> tuple[bool, str]: + """Thin wrapper around ``loader_class.is_schema_modifying_query()``. + + Returns ``(is_schema_modifying, operation_type)``. + """ + return loader_class.is_schema_modifying_query(sql_query) + + +# --------------------------------------------------------------------------- +# Chat data validation & truncation +# --------------------------------------------------------------------------- + + +def validate_and_truncate_chat( + chat_data, +) -> tuple[list, Optional[list], Optional[str], bool]: + """Validate *chat_data* and truncate history to ``Config.SHORT_MEMORY_LENGTH``. + + Uses ``getattr`` for safe attribute access (works with both Pydantic + models and plain objects). + + Returns: + ``(queries_history, result_history, instructions, use_user_rules)`` + + Raises: + InvalidArgumentError: If chat data is invalid or empty. + """ + queries_history = getattr(chat_data, 'chat', None) + result_history = getattr(chat_data, 'result', None) + instructions = getattr(chat_data, 'instructions', None) + use_user_rules = getattr(chat_data, 'use_user_rules', True) + + if not queries_history or not isinstance(queries_history, list): + raise InvalidArgumentError("Invalid or missing chat history") + + if len(queries_history) == 0: + raise InvalidArgumentError("Empty chat history") + + # Truncate to configured window + if len(queries_history) > Config.SHORT_MEMORY_LENGTH: + queries_history = queries_history[-Config.SHORT_MEMORY_LENGTH:] + if result_history and len(result_history) > 0: + max_results = Config.SHORT_MEMORY_LENGTH - 1 + if max_results > 0: + result_history = result_history[-max_results:] + else: + result_history = [] + + return queries_history, result_history, instructions, use_user_rules + + +# --------------------------------------------------------------------------- +# Pipeline helpers used by run_query / run_confirmed +# --------------------------------------------------------------------------- + + +async def quote_identifiers_from_graph( + sql_query: str, + graph_id: str, + db_type: Optional[str], + db=None, + known_tables: Optional[set] = None, +) -> tuple[str, bool]: + """Auto-quote SQL identifiers using the Table list stored in FalkorDB. + + If *known_tables* is supplied, uses it directly; otherwise queries the + graph for the current Table names. Returns ``(sql, was_modified)``. + """ + if known_tables is None: + graph = resolve_db(db).select_graph(graph_id) + try: + tables_res = ( + await graph.query("MATCH (t:Table) RETURN t.name") + ).result_set + known_tables = ( + {row[0] for row in tables_res} if tables_res else set() + ) + except Exception: # pylint: disable=broad-exception-caught + known_tables = set() + + return auto_quote_sql_identifiers(sql_query, known_tables, db_type) + + +def format_ai_response( # pylint: disable=too-many-arguments,too-many-positional-arguments + queries_history: list, + result_history: Optional[list], + sql_query: str, + query_results: list, + db_description: str, + custom_api_key: Optional[str] = None, + custom_model: Optional[str] = None, +) -> str: + """Build a human-readable AI response for *query_results*.""" + agent = ResponseFormatterAgent( + queries_history, result_history, custom_api_key, custom_model, + ) + return agent.format_response( + user_query=queries_history[-1] if queries_history else "", + sql_query=sql_query, + query_results=query_results, + db_description=db_description, + ) + + +def build_destructive_confirmation_message(sql_type: str, sql_query: str) -> str: + """Return the rich confirmation prompt shown for destructive operations. + + Used by both the streaming confirmation event and the sync ``QueryResult`` + so users see the same warning wording regardless of transport. + """ + description = _DESTRUCTIVE_VERBS.get(sql_type, "Modify the database") + return ( + "⚠️ DESTRUCTIVE OPERATION DETECTED ⚠️\n\n" + f"The generated SQL query will perform a **{sql_type}** operation:\n\n" + f"SQL:\n{sql_query}\n\n" + f"What this will do:\n• {description}\n\n" + "⚠️ WARNING: This operation will make changes to your database and " + "may be irreversible." + ) + + +def save_memory_background( # pylint: disable=too-many-arguments,too-many-positional-arguments + memory_tool: Any, + question: str, + sql_query: str, + success: bool, + error: str, + full_response: Optional[dict] = None, + chat_histories: Optional[list] = None, + task_sink: Optional[set] = None, +) -> None: + """Schedule fire-and-forget memory persistence for the given query. + + Returns immediately; tasks run in the background with their own + error-logging callbacks so a failure to save never blocks the response. + + When ``task_sink`` is given, each scheduled task is added to it and + auto-removed on completion. The SDK uses this so ``QueryWeaver.close()`` + can await in-flight memory writes before disconnecting the pool. + """ + + sink = task_sink if task_sink is not None else background_tasks_var.get() + + def _track(task): + if sink is None: + return + sink.add(task) + task.add_done_callback(sink.discard) + + def _log_done(label: str): + # Done-callbacks must not call ``t.exception()`` on a cancelled task — + # it raises CancelledError and surfaces as a noisy "exception in callback" + # log line, which is misleading at shutdown. + def _cb(task): + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + logging.error("%s failed: %s", label, exc) # nosemgrep + else: + logging.info("%s completed successfully", label) + return _cb + + save_query_task = asyncio.create_task( + memory_tool.save_query_memory( + query=question, + sql_query=sql_query, + success=success, + error=error, + ) + ) + _track(save_query_task) + save_query_task.add_done_callback(_log_done("Query memory save")) + + if full_response is not None and chat_histories is not None: + save_task = asyncio.create_task( + memory_tool.add_new_memory(full_response, chat_histories) + ) + _track(save_task) + save_task.add_done_callback(_log_done("Memory save")) + + clean_task = asyncio.create_task(memory_tool.clean_memory()) + _track(clean_task) + clean_task.add_done_callback(_log_done("Memory cleanup")) diff --git a/api/core/request_models.py b/api/core/request_models.py new file mode 100644 index 00000000..2999424a --- /dev/null +++ b/api/core/request_models.py @@ -0,0 +1,36 @@ +"""Request dataclasses accepted by the core text2sql API. + +Split from ``result_models`` so request/response intents stay distinct. +``queryweaver.models`` re-exports these for the SDK's public surface. +""" + +from dataclasses import dataclass, field + + +@dataclass +class QueryRequest: # pylint: disable=too-many-instance-attributes + """Request parameters for a query operation.""" + + question: str + """The natural language question to convert to SQL.""" + + chat_history: list[str] = field(default_factory=list) + """Previous questions in the conversation for context.""" + + result_history: list[str] = field(default_factory=list) + """Previous results for context.""" + + instructions: str | None = None + """Additional instructions for query generation.""" + + use_user_rules: bool = True + """Whether to apply user-defined rules from the database.""" + + use_memory: bool = False + """Whether to use long-term memory for context.""" + + custom_api_key: str | None = None + """Per-request override for the LLM API key. Falls back to env config.""" + + custom_model: str | None = None + """Per-request override for the LLM model (``vendor/model`` format).""" diff --git a/api/core/result_models.py b/api/core/result_models.py new file mode 100644 index 00000000..088bfa84 --- /dev/null +++ b/api/core/result_models.py @@ -0,0 +1,192 @@ +"""Result dataclasses returned by the core text2sql API. + +Kept in ``api.core`` so both the server-side code and the SDK package +depend on the same definitions. ``queryweaver.models`` re-exports +these for the SDK's public surface. +""" + +from dataclasses import asdict, dataclass, field +from typing import Any, Optional + + +@dataclass +class QueryMetadata: + """Metadata about query execution.""" + + confidence: float = 0.0 + """Confidence score (0-1) for the generated SQL query.""" + + execution_time: float = 0.0 + """Total execution time in seconds.""" + + is_valid: bool = True + """Whether the query was successfully translated to valid SQL.""" + + is_destructive: bool = False + """Whether the query is a destructive operation (INSERT/UPDATE/DELETE/DROP).""" + + requires_confirmation: bool = False + """Whether the operation requires user confirmation before execution.""" + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return asdict(self) + + +@dataclass +class QueryAnalysis: + """Analysis information from query processing.""" + + missing_information: str = "" + """Any information that was missing to fully answer the query.""" + + ambiguities: str = "" + """Any ambiguities detected in the user's question.""" + + explanation: str = "" + """Explanation of the SQL query logic.""" + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return asdict(self) + + +@dataclass +class QueryResult: + """Result from a text-to-SQL query execution.""" + + sql_query: str + """The generated SQL query.""" + + results: list[dict[str, Any]] + """Query execution results as list of row dictionaries.""" + + ai_response: str + """Human-readable AI-generated response summarizing the results.""" + + metadata: QueryMetadata = field(default_factory=QueryMetadata) + """Query execution metadata (confidence, timing, flags).""" + + analysis: QueryAnalysis = field(default_factory=QueryAnalysis) + """Query analysis information (missing info, ambiguities, explanation).""" + + error_message: Optional[str] = None + """Execution error, if any. None on success.""" + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary with flattened structure for compatibility.""" + result = { + "sql_query": self.sql_query, + "results": self.results, + "ai_response": self.ai_response, + "error_message": self.error_message, + } + result.update(self.metadata.to_dict()) + result.update(self.analysis.to_dict()) + return result + + # Compatibility properties so callers can read metadata/analysis fields flat. + @property + def confidence(self) -> float: + """Confidence score (0-1) for the generated SQL query.""" + return self.metadata.confidence + + @property + def execution_time(self) -> float: + """Total execution time in seconds.""" + return self.metadata.execution_time + + @property + def is_valid(self) -> bool: + """Whether the query was successfully translated to valid SQL.""" + return self.metadata.is_valid + + @property + def is_destructive(self) -> bool: + """Whether the query is a destructive operation.""" + return self.metadata.is_destructive + + @property + def requires_confirmation(self) -> bool: + """Whether the operation requires user confirmation.""" + return self.metadata.requires_confirmation + + @property + def missing_information(self) -> str: + """Any information that was missing to fully answer the query.""" + return self.analysis.missing_information + + @property + def ambiguities(self) -> str: + """Any ambiguities detected in the user's question.""" + return self.analysis.ambiguities + + @property + def explanation(self) -> str: + """Explanation of the SQL query logic.""" + return self.analysis.explanation + + +@dataclass +class SchemaResult: + """Database schema representation.""" + + nodes: list[dict[str, Any]] + """Tables in the schema, each with id, name, and columns.""" + + links: list[dict[str, str]] + """Foreign key relationships between tables.""" + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return asdict(self) + + +@dataclass +class DatabaseConnection: + """Result from connecting to a database.""" + + database_id: str + """The identifier for the connected database.""" + + success: bool + """Whether the connection and schema loading succeeded.""" + + tables_loaded: int = 0 + """Number of tables loaded into the schema graph.""" + + message: str = "" + """Status message or error description.""" + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return asdict(self) + + +@dataclass +class RefreshResult: + """Result from refreshing a database schema.""" + + success: bool + """Whether the schema refresh succeeded.""" + + message: str = "" + """Status message or error description.""" + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return asdict(self) + + +@dataclass +class ChatMessage: + """A message in the conversation history.""" + + question: str + """The user's question.""" + + sql_query: str = "" + """The generated SQL query (if any).""" + + result: str = "" + """The result or response.""" diff --git a/api/core/schema_loader.py b/api/core/schema_loader.py index b1568514..edb44d6c 100644 --- a/api/core/schema_loader.py +++ b/api/core/schema_loader.py @@ -4,19 +4,16 @@ import json import time from typing import AsyncGenerator, Optional +from urllib.parse import urlparse from pydantic import BaseModel +from redis import RedisError -from api.extensions import db - +from api.core.db_resolver import resolve_db from api.core.errors import InvalidArgumentError +from api.core.pipeline import MESSAGE_DELIMITER, get_database_type_and_loader from api.loaders.base_loader import BaseLoader -from api.loaders.postgres_loader import PostgresLoader -from api.loaders.mysql_loader import MySQLLoader -from api.loaders.snowflake_loader import SnowflakeLoader - -# Use the same delimiter as in the JavaScript frontend for streaming chunks -MESSAGE_DELIMITER = "|||FALKORDB_MESSAGE_BOUNDARY|||" +from api.core.result_models import DatabaseConnection class DatabaseConnectionRequest(BaseModel): @@ -35,20 +32,22 @@ def _step_start(steps_counter: int) -> dict[str, str]: "message": f"Step {steps_counter}: Starting database connection", } +_KNOWN_DB_SCHEMES = ("postgresql://", "postgres://", "mysql://", "snowflake://") + + def _step_detect_db_type(steps_counter: int, url: str) -> tuple[type[BaseLoader], dict[str, str]]: - """Yield the database type detection step message.""" - db_type = None - loader: type[BaseLoader] = BaseLoader # type: ignore - if url.startswith("postgres://") or url.startswith("postgresql://"): - db_type = "postgresql" - loader = PostgresLoader - elif url.startswith("mysql://"): - db_type = "mysql" - loader = MySQLLoader - elif url.startswith("snowflake://"): - db_type = "snowflake" - loader = SnowflakeLoader - else: + """Yield the database type detection step message. + + Strictly validates the URL scheme — unlike ``get_database_type_and_loader``'s + server-path default-to-PostgreSQL fallback, schema loading must reject + ``sqlite://``/``invalid://``/etc. with a clean ``InvalidArgumentError`` + rather than misclassifying them. + """ + if not url or not any(url.lower().startswith(s) for s in _KNOWN_DB_SCHEMES): + raise InvalidArgumentError("Invalid database URL format") + + db_type, loader = get_database_type_and_loader(url) + if loader is None or db_type is None: raise InvalidArgumentError("Invalid database URL format") return loader, { @@ -59,13 +58,13 @@ def _step_detect_db_type(steps_counter: int, url: str) -> tuple[type[BaseLoader] async def _step_attempt_load( - steps_counter: int, loader: type[BaseLoader], user_id: str, url: str + steps_counter: int, loader: type[BaseLoader], user_id: str, url: str, db=None, ) -> AsyncGenerator[dict[str, str | bool], None]: """Yield the attempt to load schema step message.""" success, result = [False, ""] try: load_start = time.perf_counter() - async for progress in loader.load(user_id, url): + async for progress in loader.load(user_id, url, db=db): success, result = progress if success: steps_counter += 1 @@ -99,7 +98,7 @@ def _step_result(result) -> str: return json.dumps(result) + MESSAGE_DELIMITER -async def load_database(url: str, user_id: str): +async def load_database(url: str, user_id: str, db=None): """ Accepts a JSON payload with a database URL and attempts to connect. Supports both PostgreSQL and MySQL databases. @@ -126,7 +125,7 @@ async def generate(): # Step 3: Attempt to load schema using the loader async for progress in _step_attempt_load( - steps_counter, loader, user_id, url + steps_counter, loader, user_id, url, db=db, ): yield _step_result(progress) @@ -146,11 +145,11 @@ async def generate(): return generate() -async def list_databases(user_id: str, general_prefix: Optional[str] = None) -> list[str]: +async def list_databases(user_id: str, general_prefix: Optional[str] = None, db=None) -> list[str]: """ This route is used to list all the graphs (databases names) that are available in the database. """ - user_graphs = await db.list_graphs() + user_graphs = await resolve_db(db).list_graphs() # Only include graphs that start with user_id + '_', and strip the prefix filtered_graphs = [ @@ -166,3 +165,70 @@ async def list_databases(user_id: str, general_prefix: Optional[str] = None) -> filtered_graphs = filtered_graphs + demo_graphs return filtered_graphs + + +# ============================================================================= +# SDK Non-Streaming Functions +# ============================================================================= + +async def load_database_sync(url: str, user_id: str, db=None): + """ + Load a database schema and return structured result (non-streaming). + + SDK-friendly version that returns DatabaseConnection instead of streaming. + + Args: + url: Database connection URL (PostgreSQL or MySQL). + user_id: User identifier for namespacing. + db: Optional FalkorDB handle; falls back to the server singleton. + + Returns: + DatabaseConnection with connection status. + """ + # Validate URL format + if not url or len(url.strip()) == 0: + raise InvalidArgumentError("Invalid URL format") + + # Determine database type and loader. ``sdk_only=True`` rejects snowflake + # and unknown schemes with a clean InvalidArgumentError instead of letting + # an ImportError surface when the snowflake extra isn't installed. + _, loader = get_database_type_and_loader(url, sdk_only=True) + if loader is None: + raise InvalidArgumentError("Invalid database URL format. Must be PostgreSQL or MySQL.") + + success = False + + try: + async for progress_success, _progress_message in loader.load(user_id, url, db=db): + success = progress_success + + if success: + # SDK callers pass the un-prefixed database_id back into query/delete/etc., + # where graph_name(user_id, db_name) re-applies the user_id prefix. + # urlparse.path may carry trailing slashes or schema/path separators + # (e.g. ``/mydb/``), and the query string is already stripped from .path + # by urlparse — but a malformed URL may yield an empty .path, so we fall + # back to splitting the raw URL. + db_name = urlparse(url).path.strip("/").split("/")[0] + if not db_name: + db_name = url.rsplit("/", 1)[-1].split("?")[0].split("#")[0] + + return DatabaseConnection( + database_id=db_name, + success=True, + message="Database connected and schema loaded successfully", + ) + + return DatabaseConnection( + database_id="", + success=False, + message="Failed to load database schema", + ) + + except (RedisError, ConnectionError, OSError) as e: + logging.exception("Error loading database: %s", str(e)) + return DatabaseConnection( + database_id="", + success=False, + message="Error connecting to database", + ) diff --git a/api/core/text2sql.py b/api/core/text2sql.py index 65bc0b4f..3c914aff 100644 --- a/api/core/text2sql.py +++ b/api/core/text2sql.py @@ -2,32 +2,53 @@ # pylint: disable=line-too-long,trailing-whitespace import asyncio -import json import logging -import os import time +from dataclasses import dataclass +from typing import Any, AsyncGenerator, Optional, TYPE_CHECKING, Union from pydantic import BaseModel -from redis import ResponseError +from redis import ResponseError, RedisError from api.core.errors import GraphNotFoundError, InternalError, InvalidArgumentError from api.core.schema_loader import load_database -from api.agents import AnalysisAgent, RelevancyAgent, ResponseFormatterAgent, FollowUpAgent +from api.core.pipeline import ( + auto_quote_sql_identifiers, + build_destructive_confirmation_message, + check_schema_modification, + detect_destructive_operation, + format_ai_response, + get_database_type_and_loader, + graph_name, + is_general_graph, + quote_identifiers_from_graph, + sanitize_log_input, + sanitize_query, + save_memory_background, + validate_and_truncate_chat, + validate_custom_model, +) +from api.agents import AnalysisAgent, RelevancyAgent, FollowUpAgent from api.agents.healer_agent import HealerAgent -from api.config import Config -from api.config import SUPPORTED_VENDORS -from api.extensions import db +from api.core.db_resolver import resolve_db +from api.core.result_models import QueryAnalysis, QueryMetadata, QueryResult, RefreshResult from api.graph import find, get_db_description, get_user_rules -from api.loaders.postgres_loader import PostgresLoader -from api.loaders.mysql_loader import MySQLLoader -from api.loaders.snowflake_loader import SnowflakeLoader -from api.memory.graphiti_tool import MemoryTool -from api.sql_utils import SQLIdentifierQuoter, DatabaseSpecificQuoter -# Use the same delimiter as in the JavaScript -MESSAGE_DELIMITER = "|||FALKORDB_MESSAGE_BOUNDARY|||" +if TYPE_CHECKING: + from falkordb.asyncio import FalkorDB + + +async def _create_memory_tool(user_id: str, graph_id: str, db=None): + """Lazy-create a MemoryTool. + + ``graphiti_core`` lives in the ``[server]`` extra; deferring the import + keeps ``pip install queryweaver`` (no extras) working for SDK callers + that pass ``use_memory=False`` (the SDK default). + """ + # pylint: disable=import-outside-toplevel + from api.memory.graphiti_tool import MemoryTool + return await MemoryTool.create(user_id, graph_id, db=db) -GENERAL_PREFIX = os.getenv("GENERAL_PREFIX") class GraphData(BaseModel): """Graph data model. @@ -64,71 +85,23 @@ class ConfirmRequest(BaseModel): chat: list = [] custom_api_key: str | None = None custom_model: str | None = None + use_memory: bool = False -def get_database_type_and_loader(db_url: str): - """ - Determine the database type from URL and return appropriate loader class. - - Args: - db_url: Database connection URL - - Returns: - tuple: (database_type, loader_class) - """ - if not db_url or db_url == "No URL available for this database.": - return None, None - - db_url_lower = db_url.lower() - - if db_url_lower.startswith('postgresql://') or db_url_lower.startswith('postgres://'): - return 'postgresql', PostgresLoader - if db_url_lower.startswith('mysql://'): - return 'mysql', MySQLLoader - if db_url_lower.startswith('snowflake://'): - return 'snowflake', SnowflakeLoader - - # Default to PostgresLoader for backward compatibility - return 'postgresql', PostgresLoader -def sanitize_query(query: str) -> str: - """Sanitize the query to prevent injection attacks.""" - return query.replace('\n', ' ').replace('\r', ' ')[:500] - -def sanitize_log_input(value: str) -> str: - """ - Sanitize input for safe logging—remove newlines, - carriage returns, tabs, and wrap in repr(). - """ - if not isinstance(value, str): - value = str(value) - - return value.replace('\n', ' ').replace('\r', ' ').replace('\t', ' ') - -def _graph_name(user_id: str, graph_id:str) -> str: - - graph_id = graph_id.strip()[:200] - if not graph_id: - raise GraphNotFoundError("Invalid graph_id, must be less than 200 characters.") - - if GENERAL_PREFIX and graph_id.startswith(GENERAL_PREFIX): - return graph_id - - return f"{user_id}_{graph_id}" - -async def get_schema(user_id: str, graph_id: str): # pylint: disable=too-many-locals,too-many-branches,too-many-statements +async def get_schema(user_id: str, graph_id: str, db=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements """Return all nodes and edges for the specified database schema (namespaced to the user). This endpoint returns a JSON object with two keys: `nodes` and `edges`. Nodes contain a minimal set of properties (id, name, labels, props). Edges contain source and target node names (or internal ids), type and props. - + args: graph_id (str): The ID of the graph to query (the database name). """ - namespaced = _graph_name(user_id, graph_id) + namespaced = graph_name(user_id, graph_id) try: - graph = db.select_graph(namespaced) + graph = resolve_db(db).select_graph(namespaced) except Exception as e: # pylint: disable=broad-exception-caught logging.error("Failed to select graph %s: %s", sanitize_log_input(namespaced), e) raise GraphNotFoundError("Graph not found or database error") from e @@ -210,745 +183,677 @@ async def get_schema(user_id: str, graph_id: str): # pylint: disable=too-many-l return {"nodes": nodes, "links": links} -async def query_database(user_id: str, graph_id: str, chat_data: ChatRequest): # pylint: disable=too-many-statements - """ - Query the Database with the given graph_id and chat_data. - - Args: - graph_id (str): The ID of the graph to query. - chat_data (ChatRequest): The chat data containing user queries and context. - """ - graph_id = _graph_name(user_id, graph_id) - - queries_history = chat_data.chat if hasattr(chat_data, 'chat') else None - result_history = chat_data.result if hasattr(chat_data, 'result') else None - instructions = chat_data.instructions if hasattr(chat_data, 'instructions') else None - use_user_rules = chat_data.use_user_rules if hasattr(chat_data, 'use_user_rules') else True - - if not queries_history or not isinstance(queries_history, list): - raise InvalidArgumentError("Invalid or missing chat history") - - if len(queries_history) == 0: - raise InvalidArgumentError("Empty chat history") - - # Truncate history to keep only the last N questions maximum (configured in Config) - if len(queries_history) > Config.SHORT_MEMORY_LENGTH: - queries_history = queries_history[-Config.SHORT_MEMORY_LENGTH:] - # Keep corresponding results (one less than queries since current query has no result yet) - if result_history and len(result_history) > 0: - max_results = Config.SHORT_MEMORY_LENGTH - 1 - if max_results > 0: - result_history = result_history[-max_results:] - else: - result_history = [] - logging.info("User Query: %s", sanitize_query(queries_history[-1])) +# --------------------------------------------------------------------------- +# Unified text2sql pipeline +# +# ``run_query`` and ``run_confirmed`` are async generators that yield wire-format +# progress events as plain dicts and end with a ``_Final(QueryResult)`` sentinel. +# +# • Streaming consumers (api/routes/graphs.py): serialize each yielded dict as +# ``json + MESSAGE_DELIMITER`` and stop when ``_Final`` arrives — the user-facing +# "final" event was already emitted as a regular dict before the sentinel. +# • SDK consumers (queryweaver): use ``collect_result`` to drop progress +# events and return the final ``QueryResult``. +# +# This is the one source of truth for the text2sql pipeline. +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _Final: + """Sentinel terminating the pipeline generator with a structured result.""" + value: QueryResult - if chat_data.use_memory: - memory_tool_task = asyncio.create_task(MemoryTool.create(user_id, graph_id)) + +async def collect_result( + gen: AsyncGenerator[Union[dict, _Final], None], +) -> QueryResult: + """Drain a pipeline generator, returning the final ``QueryResult``. + + Used by SDK consumers that don't care about progress events. Streaming + consumers iterate manually so they can serialize each dict event. + """ + async for event in gen: + if isinstance(event, _Final): + return event.value + raise InternalError("Pipeline produced no final result") + + +async def _emit_schema_refresh( # pylint: disable=too-many-arguments,too-many-positional-arguments + loader_class, + namespaced: str, + db_url: str, + operation_type: str, + *, + db: Optional["FalkorDB"] = None, + mark_final_response: bool = False, +) -> AsyncGenerator[dict, None]: + """Refresh the graph schema and yield the standard wire events. + + ``mark_final_response`` adds the ``final_response: False`` field to events. + The streaming path (``run_query``) sets it; the confirm path historically + omits it. Threading the divergence through one parameter keeps the two + callers from drifting again. + """ + base = {"final_response": False} if mark_final_response else {} + yield { + **base, + "type": "reasoning_step", + "message": "Step 3: Schema change detected - refreshing graph...", + } + + refresh_success, refresh_message = await loader_class.refresh_graph_schema( + namespaced, db_url, db=db, + ) + if refresh_success: + yield { + **base, + "type": "schema_refresh", + "message": ( + f"✅ Schema change detected ({operation_type} operation)\n\n" + "🔄 Graph schema has been automatically refreshed with the " + "latest database structure." + ), + "refresh_status": "success", + } else: - memory_tool_task = None - - # Create a generator function for streaming - async def generate(): # pylint: disable=too-many-locals,too-many-branches,too-many-statements - # Start overall timing - overall_start = time.perf_counter() - logging.info("Starting query processing pipeline for query: %s", - sanitize_query(queries_history[-1])) # nosemgrep - - # Extract custom API key and model from chat_data - custom_api_key = chat_data.custom_api_key - custom_model = chat_data.custom_model - - # Validate custom model format (vendor/model) - if custom_model: - parts = custom_model.split("/", 1) - if len(parts) != 2 or not parts[0] or not parts[1]: - raise InvalidArgumentError( - "Invalid model format. Expected 'vendor/model' (e.g. 'openai/gpt-4.1')" - ) - if parts[0] not in SUPPORTED_VENDORS: - raise InvalidArgumentError( - f"Unsupported vendor '{parts[0]}'. Supported: {', '.join(SUPPORTED_VENDORS)}" - ) - - agent_rel = RelevancyAgent(queries_history, result_history, custom_api_key, custom_model) - agent_an = AnalysisAgent(queries_history, result_history, custom_api_key, custom_model) - follow_up_agent = FollowUpAgent(queries_history, result_history, custom_api_key, custom_model) - - step = {"type": "reasoning_step", - "final_response": False, - "message": "Step 1: Analyzing user query and generating SQL..."} - yield json.dumps(step) + MESSAGE_DELIMITER - # Ensure the database description is loaded - db_description, db_url = await get_db_description(graph_id) - # Fetch user rules from database only if toggle is enabled - user_rules_spec = await get_user_rules(graph_id) if use_user_rules else None - - # Determine database type and get appropriate loader - db_type, loader_class = get_database_type_and_loader(db_url) + yield { + **base, + "type": "schema_refresh", + "message": ( + f"⚠️ Schema was modified but graph refresh failed: " + f"{refresh_message}" + ), + "refresh_status": "failed", + } + + +def _build_query_result( # pylint: disable=too-many-arguments,too-many-positional-arguments + sql_query: str, + results: list, + ai_response: str, + *, + confidence: float = 0.0, + is_valid: bool = True, + is_destructive: bool = False, + requires_confirmation: bool = False, + execution_time: float = 0.0, + missing_information: str = "", + ambiguities: str = "", + explanation: str = "", + error_message: Optional[str] = None, +) -> QueryResult: + """Assemble a ``QueryResult`` from the pipeline's loose state.""" + return QueryResult( + sql_query=sql_query, + results=results, + ai_response=ai_response, + metadata=QueryMetadata( + confidence=confidence, + is_valid=is_valid, + is_destructive=is_destructive, + requires_confirmation=requires_confirmation, + execution_time=execution_time, + ), + analysis=QueryAnalysis( + missing_information=missing_information, + ambiguities=ambiguities, + explanation=explanation, + ), + error_message=error_message, + ) + + +async def run_query( # pylint: disable=too-many-locals,too-many-branches,too-many-statements + user_id: str, + graph_id: str, + chat_data: Any, + db: Optional["FalkorDB"] = None, +) -> AsyncGenerator[Union[dict, _Final], None]: + """Run the full text2sql pipeline. - if not loader_class: - overall_elapsed = time.perf_counter() - overall_start - logging.info("Query processing failed (no loader) - Total time: %.2f seconds", - overall_elapsed) - yield json.dumps({ - "type": "error", - "final_response": True, - "message": "Unable to determine database type" - }) + MESSAGE_DELIMITER - return + Yields wire-format progress dicts (matching the streaming JSON shapes the + React frontend already parses) and ends with a ``_Final(QueryResult)`` + sentinel carrying the structured result for SDK callers. + + Args: + user_id: Namespacing identifier. + graph_id: Un-prefixed graph id; namespacing is applied internally. + chat_data: Anything with ``chat`` / ``result`` / ``instructions`` / + ``custom_api_key`` / ``custom_model`` / ``use_user_rules`` / + ``use_memory`` attributes (Pydantic ``ChatRequest`` works). + db: Optional FalkorDB handle; resolves to the server singleton when + ``None``. + """ + overall_start = time.perf_counter() + namespaced = graph_name(user_id, graph_id) + queries_history, result_history, instructions, use_user_rules = ( + validate_and_truncate_chat(chat_data) + ) + custom_api_key = getattr(chat_data, "custom_api_key", None) + custom_model = getattr(chat_data, "custom_model", None) + use_memory = getattr(chat_data, "use_memory", False) + validate_custom_model(custom_model) - # Start both tasks concurrently - find_task = asyncio.create_task(find(graph_id, queries_history, db_description)) + logging.info("User Query: %s", sanitize_query(queries_history[-1])) - relevancy_task = asyncio.create_task(agent_rel.get_answer( - queries_history[-1], db_description + # Memory tool created concurrently with relevancy/find work — small perf + # win for streaming, harmless for SDK. Lazy-imported via _create_memory_tool. + memory_tool_task = ( + asyncio.create_task(_create_memory_tool(user_id, namespaced, db=db)) + if use_memory else None + ) + + yield { + "type": "reasoning_step", + "final_response": False, + "message": "Step 1: Analyzing user query and generating SQL...", + } + + db_description, db_url = await get_db_description(namespaced, db=db) + user_rules_spec = ( + await get_user_rules(namespaced, db=db) if use_user_rules else None + ) + db_type, loader_class = get_database_type_and_loader(db_url) + + if not loader_class: + yield {"type": "error", "final_response": True, + "message": "Unable to determine database type"} + yield _Final(_build_query_result( + sql_query="", results=[], + ai_response="Unable to determine database type", + is_valid=False, + execution_time=time.perf_counter() - overall_start, + error_message="Unable to determine database type", + )) + return + + # Concurrent: relevancy check + table-finding + find_task = asyncio.create_task( + find(namespaced, queries_history, db_description, db=db) + ) + agent_rel = RelevancyAgent( + queries_history, result_history, custom_api_key, custom_model, + ) + relevancy_task = asyncio.create_task( + agent_rel.get_answer(queries_history[-1], db_description) + ) + answer_rel = await relevancy_task + + if answer_rel["status"] != "On-topic": + find_task.cancel() + try: + await find_task + except asyncio.CancelledError: + logging.debug("Find task cancelled (off-topic query)") + msg = "Off topic question: " + answer_rel["reason"] + yield {"type": "followup_questions", "final_response": True, "message": msg} + yield _Final(_build_query_result( + sql_query="", results=[], ai_response=msg, + is_valid=False, + execution_time=time.perf_counter() - overall_start, + )) + return + + tables = await find_task + + memory_tool = None + memory_context = None + if memory_tool_task is not None: + memory_tool = await memory_tool_task + memory_context = await memory_tool.search_memories(query=queries_history[-1]) + + agent_an = AnalysisAgent( + queries_history, result_history, custom_api_key, custom_model, + ) + answer_an = agent_an.get_analysis( + queries_history[-1], tables, db_description, instructions, memory_context, + db_type, user_rules_spec, + ) + + yield { + "type": "sql_query", + "data": answer_an["sql_query"], + "conf": answer_an["confidence"], + "miss": answer_an["missing_information"], + "amb": answer_an["ambiguities"], + "exp": answer_an["explanation"], + "is_valid": answer_an["is_sql_translatable"], + "final_response": False, + } + + if not answer_an["is_sql_translatable"]: + follow_up_agent = FollowUpAgent( + queries_history, result_history, custom_api_key, custom_model, + ) + follow_up = follow_up_agent.generate_follow_up_question( + user_question=queries_history[-1], + analysis_result=answer_an, + ) + yield { + "type": "followup_questions", + "final_response": True, + "message": follow_up, + "missing_information": answer_an.get("missing_information", ""), + "ambiguities": answer_an.get("ambiguities", ""), + } + yield _Final(_build_query_result( + sql_query=answer_an.get("sql_query", ""), + results=[], + ai_response=follow_up, + confidence=answer_an.get("confidence", 0.0), + is_valid=False, + execution_time=time.perf_counter() - overall_start, + missing_information=answer_an.get("missing_information", ""), + ambiguities=answer_an.get("ambiguities", ""), + explanation=answer_an.get("explanation", ""), + )) + return + + # Auto-quote identifiers using the table set we already loaded. + known_tables = {t[0] for t in tables} if tables else set() + sanitized_sql, was_modified = auto_quote_sql_identifiers( + answer_an["sql_query"], known_tables, db_type, + ) + if was_modified: + logging.info( + "SQL query auto-sanitized: quoted table names with special characters" + ) + answer_an["sql_query"] = sanitized_sql + + sql_query = answer_an["sql_query"] + sql_type, is_destructive = detect_destructive_operation(sql_query) + on_demo = is_general_graph(namespaced) + + if is_destructive and on_demo: + yield { + "type": "error", + "final_response": True, + "message": "Destructive operation not allowed on demo graphs", + } + yield _Final(_build_query_result( + sql_query=sql_query, results=[], + ai_response="Destructive operation not allowed on demo graphs", + confidence=answer_an.get("confidence", 0.0), + is_valid=True, is_destructive=True, + execution_time=time.perf_counter() - overall_start, + error_message="Destructive operation not allowed on demo graphs", )) + return + + if is_destructive: + confirmation_msg = build_destructive_confirmation_message(sql_type, sql_query) + yield { + "type": "destructive_confirmation", + "message": confirmation_msg, + "sql_query": sql_query, + "operation_type": sql_type, + "final_response": False, + } + yield _Final(_build_query_result( + sql_query=sql_query, results=[], ai_response=confirmation_msg, + confidence=answer_an.get("confidence", 0.0), + is_valid=True, is_destructive=True, requires_confirmation=True, + execution_time=time.perf_counter() - overall_start, + )) + return - logging.info("Starting relevancy check and graph analysis concurrently") + yield { + "type": "reasoning_step", + "final_response": False, + "message": "Step 2: Executing SQL query", + } - # Wait for relevancy check first - answer_rel = await relevancy_task + is_schema_modifying, operation_type = check_schema_modification(sql_query, loader_class) - if answer_rel["status"] != "On-topic": # pylint: disable=too-many-nested-blocks - # Cancel the find task since query is off-topic - find_task.cancel() - try: - await find_task - except asyncio.CancelledError: - logging.info("Find task cancelled due to off-topic query") - - step = { - "type": "followup_questions", - "final_response": True, - "message": "Off topic question: " + answer_rel["reason"], + execution_error_msg = None + query_results: list = [] + user_readable_response = "" + + try: + try: + query_results = loader_class.execute_sql_query(sql_query, db_url) + except Exception as exec_error: # pylint: disable=broad-exception-caught + yield { + "type": "reasoning_step", + "final_response": False, + "message": "Step 2a: SQL execution failed, attempting to heal query...", } - logging.info("SQL Fail reason: %s", answer_rel["reason"]) # nosemgrep - yield json.dumps(step) + MESSAGE_DELIMITER - # Total time for off-topic query - overall_elapsed = time.perf_counter() - overall_start - logging.info("Query processing completed (off-topic) - Total time: %.2f seconds", - overall_elapsed) - else: - # Query is on-topic, wait for find results - result = await find_task - - logging.info("Calling to analysis agent with query: %s", - sanitize_query(queries_history[-1])) # nosemgrep - - memory_context = None - if memory_tool_task: - memory_tool = await memory_tool_task - memory_context = await memory_tool.search_memories( - query=queries_history[-1] - ) - - logging.info("Starting SQL generation with analysis agent") - answer_an = agent_an.get_analysis( - queries_history[-1], result, db_description, instructions, memory_context, - db_type, user_rules_spec + healer = HealerAgent(max_healing_attempts=3) + + def _run_sql(sql: str): + return loader_class.execute_sql_query(sql, db_url) + + healing_result = healer.heal_and_execute( + initial_sql=sql_query, + initial_error=str(exec_error), + execute_sql_func=_run_sql, + db_description=db_description, + question=queries_history[-1], + database_type=db_type, ) - # Initialize response variables - user_readable_response = "" - follow_up_result = "" - execution_error = False - - logging.info("Generated SQL query: %s", answer_an['sql_query']) # nosemgrep - yield json.dumps( - { - "type": "sql_query", - "data": answer_an["sql_query"], - "conf": answer_an["confidence"], - "miss": answer_an["missing_information"], - "amb": answer_an["ambiguities"], - "exp": answer_an["explanation"], - "is_valid": answer_an["is_sql_translatable"], + if not healing_result.get("success"): + yield { + "type": "healing_failed", "final_response": False, + "message": ( + f"❌ Failed to heal query after " + f"{healing_result.get('attempts', 0)} attempt(s)" + ), + "final_error": healing_result.get("final_error", str(exec_error)), } - ) + MESSAGE_DELIMITER - - # If the SQL query is valid, execute it using the configured database and db_url - if answer_an["is_sql_translatable"]: - # Auto-quote table names with special characters (like dashes) - # Extract known table names from the result schema - known_tables = {table[0] for table in result} if result else set() - - # Determine database type and get appropriate quote character - quote_char = DatabaseSpecificQuoter.get_quote_char( - db_type or 'postgresql' - ) - - # Auto-quote identifiers with special characters - sanitized_sql, was_modified = ( - SQLIdentifierQuoter.auto_quote_identifiers( - answer_an['sql_query'], known_tables, quote_char - ) - ) - - if was_modified: - msg = ( - "SQL query auto-sanitized: quoted table names with " - "special characters" - ) - logging.info(msg) - answer_an['sql_query'] = sanitized_sql - - # Check if this is a destructive operation that requires confirmation - sql_query = answer_an["sql_query"] - sql_type = sql_query.strip().split()[0].upper() if sql_query else "" - - destructive_ops = ['INSERT', 'UPDATE', 'DELETE', 'DROP', - 'CREATE', 'ALTER', 'TRUNCATE'] - is_destructive = sql_type in destructive_ops - general_graph = graph_id.startswith(GENERAL_PREFIX) if GENERAL_PREFIX else False - if is_destructive and not general_graph: - # This is a destructive operation - ask for user confirmation - confirmation_message = f"""⚠️ DESTRUCTIVE OPERATION DETECTED ⚠️ - -The generated SQL query will perform a **{sql_type}** operation: - -SQL: -{sql_query} - -What this will do: -""" - if sql_type == 'INSERT': - confirmation_message += "• Add new data to the database" - elif sql_type == 'UPDATE': - confirmation_message += ("• Modify existing data in the " - "database") - elif sql_type == 'DELETE': - confirmation_message += ("• **PERMANENTLY DELETE** data " - "from the database") - elif sql_type == 'DROP': - confirmation_message += ("• **PERMANENTLY DELETE** entire " - "tables or database objects") - elif sql_type == 'CREATE': - confirmation_message += ("• Create new tables or database " - "objects") - elif sql_type == 'ALTER': - confirmation_message += ("• Modify the structure of existing " - "tables") - elif sql_type == 'TRUNCATE': - confirmation_message += ("• **PERMANENTLY DELETE ALL DATA** " - "from specified tables") - confirmation_message += """ - -⚠️ WARNING: This operation will make changes to your database and may be irreversible. -""" - - yield json.dumps( - { - "type": "destructive_confirmation", - "message": confirmation_message, - "sql_query": sql_query, - "operation_type": sql_type, - "final_response": False, - } - ) + MESSAGE_DELIMITER - # Log end-to-end time for destructive operation that requires confirmation - overall_elapsed = time.perf_counter() - overall_start - logging.info( - "Query processing halted for confirmation - Total time: %.2f seconds", - overall_elapsed - ) - return # Stop here and wait for user confirmation - - try: - if is_destructive and general_graph: - yield json.dumps( - { - "type": "error", - "final_response": True, - "message": "Destructive operation not allowed on demo graphs" - }) + MESSAGE_DELIMITER - else: - step = {"type": "reasoning_step", - "final_response": False, - "message": "Step 2: Executing SQL query"} - yield json.dumps(step) + MESSAGE_DELIMITER - - # Check if this query modifies the database schema - # using the appropriate loader - is_schema_modifying, operation_type = ( - loader_class.is_schema_modifying_query(sql_query) - ) - - # Try executing the SQL query first - try: - query_results = loader_class.execute_sql_query( - answer_an["sql_query"], - db_url - ) - except Exception as exec_error: # pylint: disable=broad-exception-caught - # Initial execution failed - start iterative healing process - step = { - "type": "reasoning_step", - "final_response": False, - "message": "Step 2a: SQL execution failed, attempting to heal query..." - } - yield json.dumps(step) + MESSAGE_DELIMITER - - # Create healer agent and attempt iterative healing - healer_agent = HealerAgent(max_healing_attempts=3) - - # Create a wrapper function for execute_sql_query - def execute_sql(sql: str): - return loader_class.execute_sql_query(sql, db_url) - - healing_result = healer_agent.heal_and_execute( - initial_sql=answer_an["sql_query"], - initial_error=str(exec_error), - execute_sql_func=execute_sql, - db_description=db_description, - question=queries_history[-1], - database_type=db_type - ) - - if not healing_result.get("success"): - # Healing failed after all attempts - yield json.dumps({ - "type": "healing_failed", - "final_response": False, - "message": f"❌ Failed to heal query after {healing_result['attempts']} attempt(s)", - "final_error": healing_result.get("final_error", str(exec_error)), - "healing_log": healing_result.get("healing_log", []) - }) + MESSAGE_DELIMITER - raise exec_error - - # Healing succeeded! - healing_log = healing_result.get("healing_log", []) - - # Show healing progress - for log_entry in healing_log: - if log_entry.get("status") == "healed": - changes_msg = ", ".join(log_entry.get("changes_made", [])) - yield json.dumps({ - "type": "healing_attempt", - "final_response": False, - "message": f"Attempt {log_entry['attempt']}: {changes_msg}", - "attempt": log_entry["attempt"], - "changes": log_entry.get("changes_made", []), - "confidence": log_entry.get("confidence", 0) - }) + MESSAGE_DELIMITER - - # Update the SQL query to the healed version - answer_an["sql_query"] = healing_result["sql_query"] - query_results = healing_result["query_results"] - - yield json.dumps({ - "type": "healing_success", - "final_response": False, - "message": f"✅ Query healed and executed successfully after {healing_result['attempts'] + 1} attempt(s)", - "healed_sql": healing_result["sql_query"], - "attempts": healing_result["attempts"] + 1 - }) + MESSAGE_DELIMITER - - if len(query_results) != 0: - yield json.dumps( - { - "type": "query_result", - "data": query_results, - "final_response": False - } - ) + MESSAGE_DELIMITER - - # If schema was modified, refresh the graph using the appropriate loader - if is_schema_modifying: - step = {"type": "reasoning_step", - "final_response": False, - "message": ("Step 3: Schema change detected - " - "refreshing graph...")} - yield json.dumps(step) + MESSAGE_DELIMITER - - refresh_result = await loader_class.refresh_graph_schema( - graph_id, db_url) - refresh_success, refresh_message = refresh_result - - if refresh_success: - refresh_msg = (f"✅ Schema change detected " - f"({operation_type} operation)\n\n" - f"🔄 Graph schema has been automatically " - f"refreshed with the latest database " - f"structure.") - yield json.dumps( - { - "type": "schema_refresh", - "final_response": False, - "message": refresh_msg, - "refresh_status": "success" - } - ) + MESSAGE_DELIMITER - else: - failure_msg = (f"⚠️ Schema was modified but graph " - f"refresh failed: {refresh_message}") - yield json.dumps( - { - "type": "schema_refresh", - "final_response": False, - "message": failure_msg, - "refresh_status": "failed" - } - ) + MESSAGE_DELIMITER - - # Generate user-readable response using AI - step_num = "4" if is_schema_modifying else "3" - step = {"type": "reasoning_step", - "final_response": False, - "message": f"Step {step_num}: Generating user-friendly response"} - yield json.dumps(step) + MESSAGE_DELIMITER - - response_agent = ResponseFormatterAgent( - queries_history, result_history, custom_api_key, custom_model - ) - user_readable_response = response_agent.format_response( - user_query=queries_history[-1], - sql_query=answer_an["sql_query"], - query_results=query_results, - db_description=db_description - ) - - yield json.dumps( - { - "type": "ai_response", - "final_response": True, - "message": user_readable_response, - } - ) + MESSAGE_DELIMITER - - # Log overall completion time - overall_elapsed = time.perf_counter() - overall_start - logging.info( - "Query processing completed successfully - Total time: %.2f seconds", - overall_elapsed - ) - - except Exception as e: # pylint: disable=broad-exception-caught - execution_error = str(e) - overall_elapsed = time.perf_counter() - overall_start - logging.error("Error executing SQL query: %s", str(e)) # nosemgrep - logging.info( - "Query processing failed during execution - Total time: %.2f seconds", - overall_elapsed - ) - yield json.dumps({ - "type": "error", - "final_response": True, - "message": "Error executing SQL query" - }) + MESSAGE_DELIMITER - else: - execution_error = "Missing information" - # SQL query is not valid/translatable - generate follow-up questions - follow_up_result = follow_up_agent.generate_follow_up_question( - user_question=queries_history[-1], - analysis_result=answer_an - ) - - # Send follow-up questions to help the user - yield json.dumps({ - "type": "followup_questions", - "final_response": True, - "message": follow_up_result, - "missing_information": answer_an.get("missing_information", ""), - "ambiguities": answer_an.get("ambiguities", "") - }) + MESSAGE_DELIMITER - - overall_elapsed = time.perf_counter() - overall_start - logging.info( - "Query processing completed (non-translatable SQL) - Total time: %.2f seconds", - overall_elapsed - ) - - # Save conversation to memory (only for on-topic queries) - # Only save to memory if use_memory is enabled - if memory_tool_task: - # Determine the final answer based on which path was taken - final_answer = user_readable_response if user_readable_response else follow_up_result - - # Build comprehensive response for memory - full_response = { - "question": queries_history[-1], - "generated_sql": answer_an.get('sql_query', ""), - "answer": final_answer - } + raise exec_error - # Add error information if SQL execution failed - if execution_error: - full_response["error"] = execution_error - full_response["success"] = False - else: - full_response["success"] = True - - - # Save query to memory - save_query_task = asyncio.create_task( - memory_tool.save_query_memory( - query=queries_history[-1], - sql_query=answer_an["sql_query"], - success=full_response["success"], - error=execution_error - ) - ) - save_query_task.add_done_callback( - lambda t: logging.error("Query memory save failed: %s", t.exception()) # nosemgrep - if t.exception() else logging.info("Query memory saved successfully") - ) - - # Save conversation with memory tool (run in background) - save_task = asyncio.create_task( - memory_tool.add_new_memory(full_response, - [queries_history, result_history]) - ) - # Add error handling callback to prevent silent failures - save_task.add_done_callback( - lambda t: logging.error("Memory save failed: %s", t.exception()) # nosemgrep - if t.exception() else logging.info("Conversation saved to memory tool") - ) - logging.info("Conversation save task started in background") - - # Clean old memory in background (once per week cleanup) - clean_memory_task = asyncio.create_task(memory_tool.clean_memory()) - clean_memory_task.add_done_callback( - lambda t: logging.error("Memory cleanup failed: %s", t.exception()) # nosemgrep - if t.exception() else logging.info("Memory cleanup completed successfully") - ) - - # Log timing summary at the end of processing - overall_elapsed = time.perf_counter() - overall_start - logging.info("Query processing pipeline completed - Total time: %.2f seconds", - overall_elapsed) - - return generate() - - -async def execute_destructive_operation( # pylint: disable=too-many-statements + sql_query = healing_result["sql_query"] + answer_an["sql_query"] = sql_query + query_results = healing_result["query_results"] + + yield { + "type": "healing_success", + "final_response": False, + "message": ( + f"✅ Query healed and executed successfully after " + f"{healing_result.get('attempts', 0)} attempt(s)" + ), + "healed_sql": sql_query, + "attempts": healing_result.get("attempts", 0), + } + + if query_results: + yield { + "type": "query_result", + "data": query_results, + "final_response": False, + } + + if is_schema_modifying: + async for ev in _emit_schema_refresh( + loader_class, namespaced, db_url, operation_type, + db=db, mark_final_response=True, + ): + yield ev + + step_num = "4" if is_schema_modifying else "3" + yield { + "type": "reasoning_step", + "final_response": False, + "message": f"Step {step_num}: Generating user-friendly response", + } + + user_readable_response = format_ai_response( + queries_history=queries_history, + result_history=result_history, + sql_query=sql_query, + query_results=query_results, + db_description=db_description, + custom_api_key=custom_api_key, + custom_model=custom_model, + ) + + yield { + "type": "ai_response", + "final_response": True, + "message": user_readable_response, + } + except Exception as e: # pylint: disable=broad-exception-caught + execution_error_msg = str(e) + logging.error("Error executing SQL query: %s", str(e)) # nosemgrep + yield { + "type": "error", + "final_response": True, + "message": "Error executing SQL query", + } + if not user_readable_response: + user_readable_response = f"Error executing SQL query: {execution_error_msg}" + + if memory_tool is not None: + full_response = { + "question": queries_history[-1], + "generated_sql": answer_an.get("sql_query", ""), + "answer": user_readable_response, + "success": execution_error_msg is None, + } + if execution_error_msg: + full_response["error"] = execution_error_msg + save_memory_background( + memory_tool=memory_tool, + question=queries_history[-1], + sql_query=answer_an.get("sql_query", ""), + success=execution_error_msg is None, + error=execution_error_msg or "", + full_response=full_response, + chat_histories=[queries_history, result_history], + ) + + yield _Final(_build_query_result( + sql_query=answer_an.get("sql_query", ""), + results=query_results if execution_error_msg is None else [], + ai_response=user_readable_response, + confidence=answer_an.get("confidence", 0.0), + is_valid=True, + is_destructive=is_destructive, + execution_time=time.perf_counter() - overall_start, + missing_information=answer_an.get("missing_information", ""), + ambiguities=answer_an.get("ambiguities", ""), + explanation=answer_an.get("explanation", ""), + error_message=execution_error_msg, + )) + + +async def run_confirmed( # pylint: disable=too-many-locals,too-many-branches,too-many-statements user_id: str, graph_id: str, - confirm_data: ConfirmRequest, -): - """ - Handle user confirmation for destructive SQL operations + confirm_data: Any, + db: Optional["FalkorDB"] = None, +) -> AsyncGenerator[Union[dict, _Final], None]: + """Execute a user-confirmed destructive SQL operation. + + Same wire-format-+-_Final shape as ``run_query``. Confirmed destructive + queries are NOT auto-healed (we just confirmed *this* SQL, not a healed + variant), so this path skips the healer entirely. """ + overall_start = time.perf_counter() + namespaced = graph_name(user_id, graph_id) + + if is_general_graph(namespaced): + # Match streaming refusal: even an explicit CONFIRM cannot run writes + # on a demo graph. + raise InvalidArgumentError( + "Destructive operations are not allowed on demo graphs" + ) + + confirmation = (getattr(confirm_data, "confirmation", "") or "").strip().upper() + sql_query = getattr(confirm_data, "sql_query", "") or "" + queries_history = getattr(confirm_data, "chat", []) or [] + custom_api_key = getattr(confirm_data, "custom_api_key", None) + custom_model = getattr(confirm_data, "custom_model", None) + validate_custom_model(custom_model) - graph_id = _graph_name(user_id, graph_id) + if not sql_query: + raise InvalidArgumentError("No SQL query provided") - if hasattr(confirm_data, 'confirmation'): - confirmation = confirm_data.confirmation.strip().upper() - else: - confirmation = "" + question = ( + queries_history[-1] if queries_history else "Destructive operation confirmation" + ) + + if confirmation != "CONFIRM": + yield { + "type": "operation_cancelled", + "message": ( + "Operation cancelled. The destructive SQL query was not executed." + ), + } + yield _Final(_build_query_result( + sql_query=sql_query, results=[], + ai_response="Operation cancelled. The destructive SQL query was not executed.", + is_valid=True, is_destructive=True, + execution_time=time.perf_counter() - overall_start, + )) + return - sql_query = confirm_data.sql_query if hasattr(confirm_data, 'sql_query') else "" - queries_history = confirm_data.chat if hasattr(confirm_data, 'chat') else [] - custom_api_key = confirm_data.custom_api_key - custom_model = confirm_data.custom_model + use_memory = bool(getattr(confirm_data, "use_memory", False)) + memory_tool = None + execution_error_msg = None + user_readable_response = "" + query_results: list = [] - if not sql_query: - raise InvalidArgumentError("No SQL query provided") + try: + # Only create the MemoryTool when the caller asks for it. graphiti_core + # is in the [server] extra; SDK installs without it would otherwise + # ImportError here at runtime. + if use_memory: + memory_tool = await _create_memory_tool(user_id, namespaced, db=db) + db_description, db_url = await get_db_description(namespaced, db=db) + db_type, loader_class = get_database_type_and_loader(db_url) + + if not loader_class: + yield {"type": "error", "message": "Unable to determine database type"} + yield _Final(_build_query_result( + sql_query=sql_query, results=[], + ai_response="Unable to determine database type", + is_valid=False, is_destructive=True, + execution_time=time.perf_counter() - overall_start, + error_message="Unable to determine database type", + )) + return - # Create a generator function for streaming the confirmation response - async def generate_confirmation(): # pylint: disable=too-many-locals,too-many-statements - # Create memory tool for saving query results - memory_tool = await MemoryTool.create(user_id, graph_id) - result_history = [] # Initialize result_history for this context + yield {"type": "reasoning_step", + "message": "Step 2: Executing confirmed SQL query"} + + sql_query, was_modified = await quote_identifiers_from_graph( + sql_query=sql_query, graph_id=namespaced, db_type=db_type, db=db, + ) + if was_modified: + logging.info("Confirmed SQL query auto-sanitized") + + is_schema_modifying, operation_type = check_schema_modification( + sql_query, loader_class, + ) + query_results = loader_class.execute_sql_query(sql_query, db_url) + yield {"type": "query_result", "data": query_results} + + if is_schema_modifying: + async for ev in _emit_schema_refresh( + loader_class, namespaced, db_url, operation_type, db=db, + ): + yield ev + + step_num = "4" if is_schema_modifying else "3" + yield {"type": "reasoning_step", + "message": f"Step {step_num}: Generating user-friendly response"} + + user_readable_response = format_ai_response( + queries_history=queries_history or [question], + result_history=None, + sql_query=sql_query, + query_results=query_results, + db_description=db_description, + custom_api_key=custom_api_key, + custom_model=custom_model, + ) + + yield {"type": "ai_response", "message": user_readable_response} - if confirmation == "CONFIRM": - try: - db_description, db_url = await get_db_description(graph_id) - - # Determine database type and get appropriate loader - _, loader_class = get_database_type_and_loader(db_url) - - if not loader_class: - yield json.dumps({ - "type": "error", - "message": "Unable to determine database type" - }) + MESSAGE_DELIMITER - return - - step = {"type": "reasoning_step", - "message": "Step 2: Executing confirmed SQL query"} - yield json.dumps(step) + MESSAGE_DELIMITER - - # Auto-quote table names for confirmed destructive operations - sql_query = confirm_data.sql_query if hasattr( - confirm_data, 'sql_query' - ) else "" - if sql_query: - # Get schema to extract known tables - graph = db.select_graph(graph_id) - tables_query = "MATCH (t:Table) RETURN t.name" - try: - tables_res = (await graph.query(tables_query)).result_set - known_tables = ( - {row[0] for row in tables_res} - if tables_res else set() - ) - except Exception: # pylint: disable=broad-exception-caught - known_tables = set() - - # Determine database type and get appropriate quote character - db_type, _ = get_database_type_and_loader(db_url) - quote_char = DatabaseSpecificQuoter.get_quote_char( - db_type or 'postgresql' - ) - - # Auto-quote identifiers - sanitized_sql, was_modified = ( - SQLIdentifierQuoter.auto_quote_identifiers( - sql_query, known_tables, quote_char - ) - ) - if was_modified: - logging.info("Confirmed SQL query auto-sanitized") - sql_query = sanitized_sql - - # Check if this query modifies the database schema using appropriate loader - is_schema_modifying, operation_type = ( - loader_class.is_schema_modifying_query(sql_query) - ) - query_results = loader_class.execute_sql_query(sql_query, db_url) - yield json.dumps( - { - "type": "query_result", - "data": query_results, - } - ) + MESSAGE_DELIMITER - - # If schema was modified, refresh the graph - if is_schema_modifying: - step = {"type": "reasoning_step", - "message": "Step 3: Schema change detected - refreshing graph..."} - yield json.dumps(step) + MESSAGE_DELIMITER - - refresh_success, refresh_message = ( - await loader_class.refresh_graph_schema(graph_id, db_url) - ) - - if refresh_success: - yield json.dumps( - { - "type": "schema_refresh", - "message": (f"✅ Schema change detected ({operation_type} " - "operation)\n\n🔄 Graph schema has been automatically " - "refreshed with the latest database structure."), - "refresh_status": "success" - } - ) + MESSAGE_DELIMITER - else: - yield json.dumps( - { - "type": "schema_refresh", - "message": (f"⚠️ Schema was modified but graph refresh failed: " - f"{refresh_message}"), - "refresh_status": "failed" - } - ) + MESSAGE_DELIMITER - - # Generate user-readable response using AI - step_num = "4" if is_schema_modifying else "3" - step = {"type": "reasoning_step", - "message": f"Step {step_num}: Generating user-friendly response"} - yield json.dumps(step) + MESSAGE_DELIMITER - - response_agent = ResponseFormatterAgent( - queries_history, result_history, custom_api_key, custom_model - ) - user_readable_response = response_agent.format_response( - user_query=queries_history[-1] if queries_history else "Destructive operation", - sql_query=sql_query, - query_results=query_results, - db_description=db_description - ) - - yield json.dumps( - { - "type": "ai_response", - "message": user_readable_response, - } - ) + MESSAGE_DELIMITER - - # Save successful confirmed query to memory - save_query_task = asyncio.create_task( - memory_tool.save_query_memory( - query=(queries_history[-1] if queries_history - else "Destructive operation confirmation"), - sql_query=sql_query, - success=True, - error="" - ) - ) - save_query_task.add_done_callback( - lambda t: logging.error("Confirmed query memory save failed: %s", - t.exception()) # nosemgrep - if t.exception() else logging.info("Confirmed query memory saved successfully") - ) - - except Exception as e: # pylint: disable=broad-exception-caught - logging.error("Error executing confirmed SQL query: %s", str(e)) # nosemgrep - error_message = str(e) if str(e) else "Error executing query" - - # Save failed confirmed query to memory - save_query_task = asyncio.create_task( - memory_tool.save_query_memory( - query=(queries_history[-1] if queries_history - else "Destructive operation confirmation"), - sql_query=sql_query, - success=False, - error=str(e) - ) - ) - save_query_task.add_done_callback( - lambda t: logging.error( # nosemgrep - "Failed confirmed query memory save failed: %s", t.exception() - ) if t.exception() else logging.info( - "Failed confirmed query memory saved successfully" - ) - ) - - yield json.dumps( - {"type": "error", "message": error_message} - ) + MESSAGE_DELIMITER - else: - # User cancelled or provided invalid confirmation - yield json.dumps( - { - "type": "operation_cancelled", - "message": "Operation cancelled. The destructive SQL query was not executed." - } - ) + MESSAGE_DELIMITER + except Exception as e: # pylint: disable=broad-exception-caught + # Wraps both MemoryTool.create failures and driver-specific execution errors. + execution_error_msg = str(e) or "Error executing query" + logging.error("Error executing confirmed SQL query: %s", str(e)) # nosemgrep + yield {"type": "error", "message": execution_error_msg} + if not user_readable_response: + user_readable_response = execution_error_msg + + if memory_tool is not None: + save_memory_background( + memory_tool=memory_tool, + question=question, + sql_query=sql_query, + success=execution_error_msg is None, + error=execution_error_msg or "", + ) + + yield _Final(_build_query_result( + sql_query=sql_query, + results=query_results if execution_error_msg is None else [], + ai_response=user_readable_response, + is_valid=True, is_destructive=True, + execution_time=time.perf_counter() - overall_start, + error_message=execution_error_msg, + )) + + + +async def _resolve_refresh_target( + user_id: str, graph_id: str, db: Optional["FalkorDB"] = None, +) -> tuple[str, str]: + """Validate refresh prerequisites and return ``(namespaced, db_url)``. + + Raises: + InvalidArgumentError: For demo graphs, which are read-only. + InternalError: When no source URL is on record for the graph. + """ + namespaced = graph_name(user_id, graph_id) + if is_general_graph(namespaced): + raise InvalidArgumentError("Demo graphs cannot be refreshed") + + _, db_url = await get_db_description(namespaced, db=db) + if not db_url or db_url == "No URL available for this database.": + raise InternalError("No database URL found for this graph") + + return namespaced, db_url - return generate_confirmation() -async def refresh_database_schema(user_id: str, graph_id: str): +async def refresh_database_schema(user_id: str, graph_id: str, db=None): """ Manually refresh the graph schema from the database. This endpoint allows users to manually trigger a schema refresh if they suspect the graph is out of sync with the database. """ - graph_id = _graph_name(user_id, graph_id) - - # Prevent refresh of demo databases - if GENERAL_PREFIX and graph_id.startswith(GENERAL_PREFIX): - raise InvalidArgumentError("Demo graphs cannot be refreshed") - try: - # Get database description and URL - _, db_url = await get_db_description(graph_id) - - if not db_url or db_url == "No URL available for this database.": - raise InternalError("No database URL found for this graph") - - # Call load_database to refresh the schema by reconnecting - return await load_database(db_url, user_id) - except InternalError: + _, db_url = await _resolve_refresh_target(user_id, graph_id, db=db) + return await load_database(db_url, user_id, db=db) + except (InvalidArgumentError, InternalError): raise except Exception as e: logging.error("Error in refresh_graph_schema: %s", str(e)) raise InternalError("Internal server error while refreshing schema") from e -async def delete_database(user_id: str, graph_id: str): + +async def refresh_schema_for_sdk( + user_id: str, graph_id: str, db: Optional["FalkorDB"] = None, +) -> RefreshResult: + """SDK-facing schema refresh that returns a structured ``RefreshResult``. + + The streaming ``refresh_database_schema`` returns a wire-format generator; + SDK callers want a single dataclass back. Both share the same underlying + reload via ``load_database_sync``. + """ + # Lazy import to break the circular dep with schema_loader. + from api.core.schema_loader import load_database_sync # pylint: disable=import-outside-toplevel + + try: + _, db_url = await _resolve_refresh_target(user_id, graph_id, db=db) + except InternalError as e: + # SDK contract is to return a RefreshResult, not raise, when the URL + # is missing. InvalidArgumentError (demo graph) still propagates. + return RefreshResult(success=False, message=str(e)) + + try: + connection_result = await load_database_sync(db_url, user_id, db=db) + return RefreshResult( + success=connection_result.success, + message=connection_result.message, + ) + except (RedisError, ConnectionError, OSError) as e: + logging.error("Error refreshing schema: %s", str(e)) + return RefreshResult( + success=False, + message=f"Failed to refresh schema: {str(e)}", + ) + + +async def delete_database(user_id: str, graph_id: str, db=None): """Delete the specified graph (namespaced to the user). This will attempt to delete the FalkorDB graph belonging to the @@ -956,17 +861,24 @@ async def delete_database(user_id: str, graph_id: str): namespace and will be namespaced using the user's id from the request state. """ - namespaced = _graph_name(user_id, graph_id) - if GENERAL_PREFIX and graph_id.startswith(GENERAL_PREFIX): + namespaced = graph_name(user_id, graph_id) + if is_general_graph(graph_id): raise InvalidArgumentError("Demo graphs cannot be deleted") try: # Select and delete the graph using the FalkorDB client API - graph = db.select_graph(namespaced) + graph = resolve_db(db).select_graph(namespaced) await graph.delete() return {"success": True, "graph": graph_id} except ResponseError as re: raise GraphNotFoundError("Failed to delete graph, Graph not found") from re - except Exception as e: # pylint: disable=broad-exception-caught + except (RedisError, ConnectionError) as e: logging.exception("Failed to delete graph %s: %s", sanitize_log_input(namespaced), e) raise InternalError("Failed to delete graph") from e + except Exception as e: # pylint: disable=broad-exception-caught + # Catch-all so any future driver-specific exception is wrapped into + # a consistent API/SDK error contract instead of leaking as a 500. + logging.exception( + "Unexpected error deleting graph %s: %s", sanitize_log_input(namespaced), e, + ) + raise InternalError("Failed to delete graph") from e diff --git a/api/graph.py b/api/graph.py index 2a9bb1a0..27e8ce05 100644 --- a/api/graph.py +++ b/api/graph.py @@ -10,7 +10,7 @@ from pydantic import BaseModel from api.config import Config -from api.extensions import db +from api.core.db_resolver import resolve_db logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") # pylint: disable=broad-exception-caught @@ -36,9 +36,9 @@ class Descriptions(BaseModel): columns_descriptions: list[ColumnDescription] -async def get_db_description(graph_id: str) -> tuple[str, str]: +async def get_db_description(graph_id: str, db=None) -> tuple[str, str]: """Get the database description from the graph.""" - graph = db.select_graph(graph_id) + graph = resolve_db(db).select_graph(graph_id) query_result = await graph.query( """ MATCH (d:Database) @@ -54,9 +54,9 @@ async def get_db_description(graph_id: str) -> tuple[str, str]: query_result.result_set[0][1]) # Return the first result's description -async def get_user_rules(graph_id: str) -> str: +async def get_user_rules(graph_id: str, db=None) -> str: """Get the user rules from the graph.""" - graph = db.select_graph(graph_id) + graph = resolve_db(db).select_graph(graph_id) query_result = await graph.query( """ MATCH (d:Database) @@ -70,9 +70,9 @@ async def get_user_rules(graph_id: str) -> str: return query_result.result_set[0][0] -async def set_user_rules(graph_id: str, user_rules: str) -> None: +async def set_user_rules(graph_id: str, user_rules: str, db=None) -> None: """Set the user rules in the graph.""" - graph = db.select_graph(graph_id) + graph = resolve_db(db).select_graph(graph_id) await graph.query( """ MERGE (d:Database) @@ -279,7 +279,8 @@ async def _find_connecting_tables( async def find( # pylint: disable=too-many-locals graph_id: str, queries_history: List[str], - db_description: str = None + db_description: str = None, + db=None, ) -> List[List[Any]]: """ Find the tables and columns relevant to the user's query. @@ -288,11 +289,12 @@ async def find( # pylint: disable=too-many-locals graph_id: The identifier for the graph database. queries_history: List of previous queries, with the last one being current. db_description: Optional description of the database. + db: Optional FalkorDB handle; falls back to the server singleton. Returns: Combined list of relevant tables. """ - graph = db.select_graph(graph_id) + graph = resolve_db(db).select_graph(graph_id) user_query = queries_history[-1] previous_queries = queries_history[:-1] diff --git a/api/index.py b/api/index.py index 1bb7d061..4c96938e 100644 --- a/api/index.py +++ b/api/index.py @@ -1,26 +1,56 @@ -"""Main entry point for the text2sql API.""" +"""Main entry point for the text2sql API. -# Load .env before any app imports that read os.getenv at module level -from dotenv import load_dotenv -load_dotenv() +Module-level imports of ``dotenv`` / ``api.app_factory`` are guarded so that +``pip install queryweaver`` (no ``[server]`` extra) can still resolve the +``queryweaver`` console script and surface a friendly install message. +``app`` is exposed only when the server extras are present so uvicorn's +``api.index:app`` reference keeps working. +""" -from api.app_factory import create_app # pylint: disable=wrong-import-position +try: + # Load .env before any app imports that read os.getenv at module level. + from dotenv import load_dotenv + load_dotenv() + from api.app_factory import create_app # pylint: disable=wrong-import-position -app = create_app() + app = create_app() + _SERVER_AVAILABLE = True + _SERVER_IMPORT_ERROR: Exception | None = None +except ImportError as _exc: + # SDK-only install: server extras are not present. Defer the failure to + # ``main()`` so importing ``api.index`` (e.g. via the console script + # entrypoint) does not crash before we can print the install message. + app = None # pylint: disable=invalid-name # type: ignore[assignment] + _SERVER_AVAILABLE = False + _SERVER_IMPORT_ERROR = _exc -if __name__ == "__main__": - import os - import uvicorn - # Read FASTAPI_DEBUG to determine debug mode +def main() -> None: + """Console-script entrypoint (``queryweaver`` after ``pip install``). + + Requires the ``[server]`` extra (FastAPI + uvicorn). Plain + ``pip install queryweaver`` installs the SDK only; the server is + available via ``pip install queryweaver[server]``. + """ + if not _SERVER_AVAILABLE: + raise SystemExit( + "queryweaver server requires the [server] extra. " + "Install with: pip install queryweaver[server]\n" + f"(missing: {_SERVER_IMPORT_ERROR})" + ) + + import os # pylint: disable=import-outside-toplevel + import uvicorn # pylint: disable=import-outside-toplevel + debug_mode = os.environ.get('FASTAPI_DEBUG', 'False').lower() == 'true' uvicorn.run( "api.index:app", - host="127.0.0.1", - port=5000, + host=os.environ.get("HOST", "127.0.0.1"), + port=int(os.environ.get("PORT", "5000")), reload=debug_mode, log_level="info" if debug_mode else "warning", ) -# This allows running the app with `uvicorn api.index:app` or directly with `python api/index.py` -# Ensure the environment variable FASTAPI_DEBUG is set to 'True' for debug mode -# or 'False' for production mode. + + +if __name__ == "__main__": + main() diff --git a/api/loaders/graph_loader.py b/api/loaders/graph_loader.py index 855b2201..b4f3fca5 100644 --- a/api/loaders/graph_loader.py +++ b/api/loaders/graph_loader.py @@ -5,7 +5,7 @@ import tqdm from api.config import Config -from api.extensions import db +from api.core.db_resolver import resolve_db from api.utils import generate_db_description, create_combined_description @@ -16,6 +16,7 @@ async def load_to_graph( # pylint: disable=too-many-arguments,too-many-position batch_size: int = 100, db_name: str = "TBD", db_url: str = "", + db=None, ) -> None: """ Load the graph data into the database. @@ -26,8 +27,9 @@ async def load_to_graph( # pylint: disable=too-many-arguments,too-many-position - relationships: A dictionary containing the relationships between entities. - batch_size: The size of the batch for embedding. - db_name: The name of the database. + - db: Optional FalkorDB handle; falls back to the server singleton. """ - graph = db.select_graph(graph_id) + graph = resolve_db(db).select_graph(graph_id) embedding_model = Config.EMBEDDING_MODEL vec_len = embedding_model.get_vector_size() diff --git a/api/loaders/mysql_loader.py b/api/loaders/mysql_loader.py index 9825b2e4..2e8b40fa 100644 --- a/api/loaders/mysql_loader.py +++ b/api/loaders/mysql_loader.py @@ -152,13 +152,18 @@ def _parse_mysql_url(connection_url: str) -> Dict[str, str]: } @staticmethod - async def load(prefix: str, connection_url: str) -> AsyncGenerator[tuple[bool, str], None]: + async def load( # pylint: disable=arguments-differ + prefix: str, + connection_url: str, + db=None, + ) -> AsyncGenerator[tuple[bool, str], None]: """ Load the graph data from a MySQL database into the graph database. Args: connection_url: MySQL connection URL in format: mysql://username:password@host:port/database + db: Optional FalkorDB handle; falls back to the server singleton. Returns: Tuple[bool, str]: Success status and message @@ -189,7 +194,7 @@ async def load(prefix: str, connection_url: str) -> AsyncGenerator[tuple[bool, s # Load data into graph yield True, "Loading data into graph..." await load_to_graph(f"{prefix}_{db_name}", entities, relationships, - db_name=db_name, db_url=connection_url) + db_name=db_name, db_url=connection_url, db=db) yield True, (f"MySQL schema loaded successfully. " f"Found {len(entities)} tables.") @@ -442,13 +447,14 @@ def is_schema_modifying_query(sql_query: str) -> Tuple[bool, str]: return False, "" @staticmethod - async def refresh_graph_schema(graph_id: str, db_url: str) -> Tuple[bool, str]: + async def refresh_graph_schema(graph_id: str, db_url: str, db=None) -> Tuple[bool, str]: """ Refresh the graph schema by clearing existing data and reloading from the database. Args: graph_id: The graph ID to refresh db_url: Database connection URL + db: Optional FalkorDB handle; falls back to the server singleton. Returns: Tuple of (success, message) @@ -456,12 +462,11 @@ async def refresh_graph_schema(graph_id: str, db_url: str) -> Tuple[bool, str]: try: logging.info("Schema modification detected. Refreshing graph schema.") - # Import here to avoid circular imports - from api.extensions import db # pylint: disable=import-error,import-outside-toplevel + from api.core.db_resolver import resolve_db # pylint: disable=import-outside-toplevel # Clear existing graph data # Drop current graph before reloading - graph = db.select_graph(graph_id) + graph = resolve_db(db).select_graph(graph_id) await graph.delete() # Extract prefix from graph_id (remove database name part) @@ -474,7 +479,7 @@ async def refresh_graph_schema(graph_id: str, db_url: str) -> Tuple[bool, str]: prefix = graph_id # Reuse the existing load method to reload the schema - success, message = await MySQLLoader.load(prefix, db_url) + success, message = await MySQLLoader.load(prefix, db_url, db=db) if success: logging.info("Graph schema refreshed successfully.") diff --git a/api/loaders/postgres_loader.py b/api/loaders/postgres_loader.py index 9d58f4e2..c5dff6fa 100644 --- a/api/loaders/postgres_loader.py +++ b/api/loaders/postgres_loader.py @@ -140,7 +140,11 @@ def parse_schema_from_url(connection_url: str) -> str: return 'public' @staticmethod - async def load(prefix: str, connection_url: str) -> AsyncGenerator[tuple[bool, str], None]: + async def load( # pylint: disable=arguments-differ + prefix: str, + connection_url: str, + db=None, + ) -> AsyncGenerator[tuple[bool, str], None]: """ Load the graph data from a PostgreSQL database into the graph database. @@ -149,6 +153,7 @@ async def load(prefix: str, connection_url: str) -> AsyncGenerator[tuple[bool, s postgresql://username:password@host:port/database Optionally with schema via options parameter: postgresql://...?options=-csearch_path%3Dschema_name + db: Optional FalkorDB handle; falls back to the server singleton. Returns: Tuple[bool, str]: Success status and message @@ -191,7 +196,7 @@ async def load(prefix: str, connection_url: str) -> AsyncGenerator[tuple[bool, s yield True, "Loading data into graph..." # Load data into graph await load_to_graph(f"{prefix}_{db_name}", entities, relationships, - db_name=db_name, db_url=connection_url) + db_name=db_name, db_url=connection_url, db=db) yield True, (f"PostgreSQL schema loaded successfully. " f"Found {len(entities)} tables.") @@ -483,13 +488,14 @@ def is_schema_modifying_query(sql_query: str) -> Tuple[bool, str]: return False, "" @staticmethod - async def refresh_graph_schema(graph_id: str, db_url: str) -> Tuple[bool, str]: + async def refresh_graph_schema(graph_id: str, db_url: str, db=None) -> Tuple[bool, str]: """ Refresh the graph schema by clearing existing data and reloading from the database. Args: graph_id: The graph ID to refresh db_url: Database connection URL + db: Optional FalkorDB handle; falls back to the server singleton. Returns: Tuple of (success, message) @@ -497,12 +503,11 @@ async def refresh_graph_schema(graph_id: str, db_url: str) -> Tuple[bool, str]: try: logging.info("Schema modification detected. Refreshing graph schema.") - # Import here to avoid circular imports - from api.extensions import db # pylint: disable=import-error,import-outside-toplevel + from api.core.db_resolver import resolve_db # pylint: disable=import-outside-toplevel # Clear existing graph data # Drop current graph before reloading - graph = db.select_graph(graph_id) + graph = resolve_db(db).select_graph(graph_id) await graph.delete() # Extract prefix from graph_id (remove database name part) @@ -515,7 +520,7 @@ async def refresh_graph_schema(graph_id: str, db_url: str) -> Tuple[bool, str]: prefix = graph_id # Reuse the existing load method to reload the schema - success, message = await PostgresLoader.load(prefix, db_url) + success, message = await PostgresLoader.load(prefix, db_url, db=db) if success: logging.info("Graph schema refreshed successfully.") diff --git a/api/memory/graphiti_tool.py b/api/memory/graphiti_tool.py index 00a1b3e6..4838fe74 100644 --- a/api/memory/graphiti_tool.py +++ b/api/memory/graphiti_tool.py @@ -18,8 +18,8 @@ # Import Graphiti components from graphiti_core.driver.falkordb_driver import FalkorDriver from graphiti_core import Graphiti -from api.extensions import db from api.config import Config +from api.core.db_resolver import resolve_db from graphiti_core.nodes import EpisodeType from graphiti_core.llm_client import LLMConfig, OpenAIClient from graphiti_core.embedder import OpenAIEmbedder, OpenAIEmbedderConfig @@ -57,10 +57,11 @@ class MemoryTool: else None ) - def __init__(self, user_id: str, graph_id: str): + def __init__(self, user_id: str, graph_id: str, db=None): # Create FalkorDB driver with user-specific database self.memory_db_name = f"{user_id}-memory" - falkor_driver = FalkorDriver(falkor_db=db, database=self.memory_db_name) + self._db = resolve_db(db) + falkor_driver = FalkorDriver(falkor_db=self._db, database=self.memory_db_name) # Create Graphiti client with Azure OpenAI configuration @@ -74,14 +75,20 @@ def __init__(self, user_id: str, graph_id: str): async def _refresh_ttl(self) -> None: """Set a TTL on the memory graph key using Redis EXPIRE.""" try: - await db.execute_command("EXPIRE", self.memory_db_name, self.MEMORY_TTL_SECONDS) + await self._db.execute_command("EXPIRE", self.memory_db_name, self.MEMORY_TTL_SECONDS) except RedisError as e: logging.warning("Failed to refresh TTL for %s: %s", self.memory_db_name, e) @classmethod - async def create(cls, user_id: str, graph_id: str, use_direct_entities: bool = True) -> "MemoryTool": + async def create( + cls, + user_id: str, + graph_id: str, + use_direct_entities: bool = True, + db=None, + ) -> "MemoryTool": """Async factory to construct and initialize the tool.""" - self = cls(user_id, graph_id) + self = cls(user_id, graph_id, db=db) if not self.memory_enabled: return self diff --git a/api/routes/database.py b/api/routes/database.py index 293c63ad..e3287541 100644 --- a/api/routes/database.py +++ b/api/routes/database.py @@ -9,9 +9,6 @@ database_router = APIRouter(tags=["Database Connection"]) -# Use the same delimiter as in the JavaScript frontend for streaming chunks -MESSAGE_DELIMITER = "|||FALKORDB_MESSAGE_BOUNDARY|||" - class DatabaseConnectionRequest(BaseModel): """Database connection request model. @@ -29,7 +26,7 @@ async def connect_database(request: Request, db_request: DatabaseConnectionReque """ Accepts a JSON payload with a database URL and attempts to connect. Supports both PostgreSQL and MySQL databases. - Streams progress steps as a sequence of JSON messages separated by MESSAGE_DELIMITER. + Streams progress steps as a sequence of JSON messages separated by a delimiter. Requires authentication. """ generator = await load_database(db_request.url, request.state.user_id) diff --git a/api/routes/graphs.py b/api/routes/graphs.py index f0a7d036..95b6c08c 100644 --- a/api/routes/graphs.py +++ b/api/routes/graphs.py @@ -1,5 +1,6 @@ """Graph-related routes for the text2sql API.""" +import json import logging from fastapi import APIRouter, Request, HTTPException, UploadFile, File from fastapi.responses import JSONResponse, StreamingResponse @@ -7,19 +8,24 @@ from api.core.schema_loader import list_databases from api.core.text2sql import ( - GENERAL_PREFIX, ChatRequest, ConfirmRequest, - GraphNotFoundError, - InternalError, - InvalidArgumentError, + _Final, delete_database, - execute_destructive_operation, get_schema, - query_database, refresh_database_schema, - _graph_name, + run_confirmed, + run_query, +) +from api.core.pipeline import ( + GENERAL_PREFIX, + MESSAGE_DELIMITER, + graph_name, + is_general_graph, + validate_and_truncate_chat, + validate_custom_model, ) +from api.core.errors import GraphNotFoundError, InternalError, InvalidArgumentError from api.graph import get_user_rules, set_user_rules from api.auth.user_management import token_required from api.routes.tokens import UNAUTHORIZED_RESPONSE @@ -27,6 +33,20 @@ graphs_router = APIRouter(tags=["Graphs & Databases"]) +async def _serialize_pipeline(gen): + """Serialize pipeline events to the wire format and stop on ``_Final``. + + Pure encoding loop — no exception handling here. Each route handler + wraps iteration in its own ``try/except`` so the broad-except (which + emits a generic error event without leaking stack data) lives in the + route function CodeQL already accepts, not in a shared helper. + """ + async for event in gen: + if isinstance(event, _Final): + return + yield json.dumps(event) + MESSAGE_DELIMITER + + class GraphData(BaseModel): """Graph data model. @@ -145,13 +165,36 @@ async def query_graph( graph_id (str): The ID of the graph to query. chat_data (ChatRequest): The chat data containing user queries and context. """ + # Eager validation: ``run_query`` is an async generator, so its body + # (including ``validate_and_truncate_chat``/``graph_name``) only runs once + # the StreamingResponse is iterated. Surfacing client errors as HTTP 400 + # requires a synchronous check before we hand the stream to the response. try: - generator = await query_database(request.state.user_id, graph_id, chat_data) - return StreamingResponse(generator, media_type="application/json") + graph_name(request.state.user_id, graph_id) + validate_and_truncate_chat(chat_data) + validate_custom_model(getattr(chat_data, "custom_model", None)) except InvalidArgumentError as iae: logging.warning("Invalid argument in query: %s", str(iae)) return JSONResponse(content={"error": "Invalid query request"}, status_code=400) + async def stream(): + try: + async for chunk in _serialize_pipeline( + run_query(request.state.user_id, graph_id, chat_data) + ): + yield chunk + except Exception: # pylint: disable=broad-exception-caught + # Don't leak stack traces (CodeQL: information exposure through + # exception). Log internally; emit a generic error event. + logging.exception("Streaming query failed") + yield json.dumps({ + "type": "error", + "final_response": True, + "message": "Internal error while processing query", + }) + MESSAGE_DELIMITER + + return StreamingResponse(stream(), media_type="application/json") + @graphs_router.post("/{graph_id}/confirm", responses={401: UNAUTHORIZED_RESPONSE}) @token_required @@ -165,15 +208,37 @@ async def confirm_destructive_operation( Requires authentication. """ + # Eager validation — see note on the query endpoint above. try: - generator = await execute_destructive_operation( - request.state.user_id, graph_id, confirm_data - ) - return StreamingResponse(generator, media_type="application/json") + namespaced = graph_name(request.state.user_id, graph_id) + if is_general_graph(namespaced): + raise InvalidArgumentError( + "Destructive operations are not allowed on demo graphs" + ) + if not (getattr(confirm_data, "sql_query", "") or "").strip(): + raise InvalidArgumentError("No SQL query provided") + validate_custom_model(getattr(confirm_data, "custom_model", None)) except InvalidArgumentError as iae: logging.warning("Invalid argument in destructive operation: %s", str(iae)) return JSONResponse(content={"error": "Invalid confirmation request"}, status_code=400) + async def stream(): + try: + async for chunk in _serialize_pipeline( + run_confirmed(request.state.user_id, graph_id, confirm_data) + ): + yield chunk + except Exception: # pylint: disable=broad-exception-caught + # See note on the query endpoint above (CodeQL). + logging.exception("Streaming confirmed-destructive query failed") + yield json.dumps({ + "type": "error", + "final_response": True, + "message": "Internal error while processing confirmation", + }) + MESSAGE_DELIMITER + + return StreamingResponse(stream(), media_type="application/json") + @graphs_router.post("/{graph_id}/refresh", responses={401: UNAUTHORIZED_RESPONSE}) @token_required @@ -239,7 +304,7 @@ class UserRulesRequest(BaseModel): async def get_graph_user_rules(request: Request, graph_id: str): """Get user rules for the specified graph.""" try: - full_graph_id = _graph_name(request.state.user_id, graph_id) + full_graph_id = graph_name(request.state.user_id, graph_id) user_rules = await get_user_rules(full_graph_id) logging.info("Retrieved user rules length: %d", len(user_rules) if user_rules else 0) return JSONResponse(content={"user_rules": user_rules}) @@ -265,7 +330,7 @@ async def update_graph_user_rules(request: Request, graph_id: str, data: UserRul logging.info( "Received request to update user rules, content length: %d", len(data.user_rules) ) - full_graph_id = _graph_name(request.state.user_id, graph_id) + full_graph_id = graph_name(request.state.user_id, graph_id) await set_user_rules(full_graph_id, data.user_rules) logging.info("User rules updated successfully") return JSONResponse(content={"success": True, "user_rules": data.user_rules}) diff --git a/app/package-lock.json b/app/package-lock.json index a8249fbd..e522352f 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -3169,6 +3169,7 @@ "version": "22.19.7", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -3182,6 +3183,7 @@ "version": "18.3.27", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -3191,6 +3193,7 @@ "version": "18.3.7", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^18.0.0" } @@ -3240,6 +3243,7 @@ "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.57.2", "@typescript-eslint/types": "8.57.2", @@ -3502,6 +3506,7 @@ "version": "8.15.0", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -3689,6 +3694,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -4244,6 +4250,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -4331,6 +4338,7 @@ "node_modules/date-fns": { "version": "3.6.0", "license": "MIT", + "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/kossnocorp" @@ -4397,7 +4405,8 @@ }, "node_modules/embla-carousel": { "version": "8.6.0", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/embla-carousel-react": { "version": "8.6.0", @@ -4480,6 +4489,7 @@ "version": "9.39.2", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -4990,6 +5000,7 @@ "node_modules/jiti": { "version": "1.21.7", "license": "MIT", + "peer": true, "bin": { "jiti": "bin/jiti.js" } @@ -5344,6 +5355,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -5541,6 +5553,7 @@ "node_modules/react": { "version": "18.3.1", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -5563,6 +5576,7 @@ "node_modules/react-dom": { "version": "18.3.1", "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -5576,6 +5590,7 @@ "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.71.2.tgz", "integrity": "sha512-1CHvcDYzuRUNOflt4MOq3ZM46AronNJtQ1S7tnX6YN4y72qhgiUItpacZUAQ0TyWYci3yz1X+rXaSxiuEm86PA==", "license": "MIT", + "peer": true, "engines": { "node": ">=18.0.0" }, @@ -6022,6 +6037,7 @@ "node_modules/tailwindcss": { "version": "3.4.18", "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -6133,6 +6149,7 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -6186,6 +6203,7 @@ "version": "5.9.3", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6347,6 +6365,7 @@ "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -6438,6 +6457,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..b94c2391 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,41 @@ +# Test services for QueryWeaver SDK integration tests +# Usage: docker compose -f docker-compose.test.yml up -d + +services: + falkordb: + # Pin the major to avoid surprise breaks; bump deliberately when needed. + image: falkordb/falkordb:v4 + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + postgres: + image: postgres:15 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: testdb + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 3s + retries: 5 + + mysql: + image: mysql:8 + environment: + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: testdb + ports: + - "3306:3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 5s + timeout: 3s + retries: 5 diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..4391cff4 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,124 @@ +# QueryWeaver SDK — E-commerce Demo + +End-to-end walkthrough of using the `queryweaver` Python package against +a realistic 12-table PostgreSQL schema. + +## What's in here + +| File | Purpose | +|---|---| +| `ecommerce_example.sql` | Creates a `queryweaver_demo` database with 12 tables and ~120 rows of seed data | +| `ecommerce_example.py` | SDK script: connect → introspect schema → run a one-shot query → run a follow-up with chat history → tear down | + +The schema models a small e-commerce business: + +```text +users ───┬── addresses + ├── orders ── order_items ── product_variants ── products ── categories (self-ref) + │ └── payments │ │ + │ │ └── product_suppliers ── suppliers (M:N) + └── reviews ─────────────────────────────────────── products + inventory ──┘ +``` + +12 tables, 13 foreign-key relationships including one self-reference +(`categories.parent_id`) and one many-to-many (`product_suppliers`). + +## Prerequisites + +- Docker +- Python 3.12+ +- An LLM provider key — OpenAI, Azure OpenAI, Gemini, Anthropic, or Cohere +- A FalkorDB instance (the steps below run one in Docker) + +## 1. Start PostgreSQL + +```bash +docker run -d --name queryweaver-pg \ + -e POSTGRES_USER=root \ + -e POSTGRES_PASSWORD=123123 \ + -p 5432:5432 \ + postgres:15 +``` + +## 2. Start FalkorDB + +```bash +docker run -d --name queryweaver-falkor \ + -p 6379:6379 \ + falkordb/falkordb:latest +``` + +(Or skip this step and point at a managed FalkorDB by setting `FALKORDB_URL` +in step 5.) + +## 3. Load the demo schema and seed data + +```bash +docker cp examples/ecommerce_example.sql queryweaver-pg:/tmp/demo.sql +docker exec queryweaver-pg psql -U root -d postgres -f /tmp/demo.sql +``` + +The script ends with a row-count summary — you should see all 12 tables +populated. + +## 4. Install the SDK + +```bash +pip install queryweaver +``` + +Or from a local checkout of this repo: + +```bash +pip install -e . +``` + +## 5. Set environment variables + +```bash +export FALKORDB_URL=redis://localhost:6379 +export OPENAI_API_KEY=sk-... +``` + +For Azure OpenAI: + +```bash +export AZURE_API_KEY=... +export AZURE_API_BASE=https://.openai.azure.com/ +export AZURE_API_VERSION=2024-12-01-preview +``` + +Other supported providers: `GEMINI_API_KEY`, `ANTHROPIC_API_KEY`, +`COHERE_API_KEY`. See `api/config.py` for provider detection logic. + +## 6. Run the example + +```bash +python examples/ecommerce_example.py +``` + +You will see: +- Connection + schema-load status +- Generated SQL for the natural-language question +- Rows returned by PostgreSQL +- An AI-formatted summary +- A follow-up query that uses chat history (`QueryRequest.chat_history` / + `result_history`) +- Final cleanup of the loaded schema + +## Cleanup + +```bash +docker rm -f queryweaver-pg queryweaver-falkor +``` + +## Sample questions to try + +Edit `ecommerce_example.py` (or open a Python REPL) and try: + +- "Which products are low on stock across all warehouses?" +- "Show the top 3 suppliers by total cost across all products they supply" +- "Find users who bought a laptop and also reviewed it" +- "List orders whose payment is still pending" +- "Show the category hierarchy" *(self-join on `categories`)* diff --git a/examples/ecommerce_example.py b/examples/ecommerce_example.py new file mode 100644 index 00000000..84cfdad1 --- /dev/null +++ b/examples/ecommerce_example.py @@ -0,0 +1,103 @@ +"""End-to-end QueryWeaver SDK example against the e-commerce schema. + +Connects to the demo PostgreSQL database (loaded from +``ecommerce_example.sql``), introspects the schema into FalkorDB, runs a +one-shot natural-language query, then a follow-up that uses chat history, +and finally tears down the loaded schema. + +Required environment variables: + FALKORDB_URL e.g. redis://localhost:6379 + OPENAI_API_KEY — or any other LiteLLM-supported provider + (AZURE_API_KEY+AZURE_API_BASE+AZURE_API_VERSION, + GEMINI_API_KEY, ANTHROPIC_API_KEY, COHERE_API_KEY) + +Required (for the demo Postgres loaded from ``ecommerce_example.sql``): + DEMO_POSTGRES_URL e.g. postgresql://USER:PASSWORD@localhost:5432/queryweaver_demo +""" + +import asyncio +import os +from urllib.parse import urlparse + +from queryweaver import QueryRequest, QueryWeaver + + +def _redact_url(url: str) -> str: + """Render a connection URL without leaking the password.""" + parsed = urlparse(url) + host = parsed.hostname or "" + if parsed.port: + host = f"{host}:{parsed.port}" + user = f"{parsed.username}@" if parsed.username else "" + return f"{parsed.scheme}://{user}{host}{parsed.path}" + + +POSTGRES_URL = os.environ.get("DEMO_POSTGRES_URL", "") +FALKORDB_URL = os.environ.get("FALKORDB_URL", "redis://localhost:6379") + +if not POSTGRES_URL: + raise SystemExit( + "Set DEMO_POSTGRES_URL, e.g. " + "postgresql://USER:PASSWORD@localhost:5432/queryweaver_demo" + ) + + +def _print_rows(rows, limit=10): + for row in rows[:limit]: + print(f" {row}") + if len(rows) > limit: + print(f" ... ({len(rows) - limit} more rows)") + + +async def main() -> None: + async with QueryWeaver(falkordb_url=FALKORDB_URL, user_id="demo") as qw: + print(f"Connecting to PostgreSQL at {_redact_url(POSTGRES_URL)}") + conn = await qw.connect_database(POSTGRES_URL) + if not conn.success: + raise SystemExit(f"connect failed: {conn.message}") + print(f" connected; database_id={conn.database_id}\n") + + schema = await qw.get_schema(conn.database_id) + print(f"Schema: {len(schema.nodes)} tables, {len(schema.links)} relationships\n") + + # 1) One-shot query + question = ( + "Show each customer's total spending and number of orders, " + "sorted by total spending descending" + ) + print(f"Q: {question}") + result = await qw.query(conn.database_id, question) + print(f"\n SQL: {result.sql_query}") + print(f" Rows ({len(result.results)}):") + _print_rows(result.results) + print(f"\n AI summary: {result.ai_response}") + print( + f" (confidence={result.metadata.confidence:.2f}, " + f"took {result.metadata.execution_time:.2f}s)\n" + ) + + # 2) Multi-turn: follow-up uses chat_history + result_history + first_q = "Show all products with their categories" + print(f"Q: {first_q}") + first = await qw.query(conn.database_id, first_q) + print(f" -> {len(first.results)} rows\n") + + followup_q = "Of those, which ones have an average review rating above 4?" + print(f"Q (follow-up): {followup_q}") + followup = QueryRequest( + question=followup_q, + chat_history=[first_q], + result_history=[first.ai_response], + ) + second = await qw.query(conn.database_id, followup) + print(f"\n SQL: {second.sql_query}") + print(f" Rows ({len(second.results)}):") + _print_rows(second.results) + print() + + await qw.delete_database(conn.database_id) + print(f"Cleaned up: removed schema for {conn.database_id} from FalkorDB") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/ecommerce_example.sql b/examples/ecommerce_example.sql new file mode 100644 index 00000000..52760949 --- /dev/null +++ b/examples/ecommerce_example.sql @@ -0,0 +1,275 @@ +-- QueryWeaver demo schema: 12-table e-commerce +DROP DATABASE IF EXISTS queryweaver_demo; +CREATE DATABASE queryweaver_demo; +\c queryweaver_demo + +-- 1. users +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email VARCHAR(120) UNIQUE NOT NULL, + full_name VARCHAR(120) NOT NULL, + signup_date DATE NOT NULL DEFAULT CURRENT_DATE, + is_active BOOLEAN NOT NULL DEFAULT TRUE +); + +-- 2. addresses +CREATE TABLE addresses ( + id SERIAL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + street VARCHAR(160) NOT NULL, + city VARCHAR(80) NOT NULL, + state VARCHAR(40), + postal_code VARCHAR(20), + country VARCHAR(60) NOT NULL, + is_default BOOLEAN NOT NULL DEFAULT FALSE +); + +-- 3. categories (self-referencing) +CREATE TABLE categories ( + id SERIAL PRIMARY KEY, + name VARCHAR(80) NOT NULL, + parent_id INTEGER REFERENCES categories(id) ON DELETE SET NULL +); + +-- 4. products +CREATE TABLE products ( + id SERIAL PRIMARY KEY, + sku VARCHAR(40) UNIQUE NOT NULL, + name VARCHAR(160) NOT NULL, + description TEXT, + category_id INTEGER REFERENCES categories(id) ON DELETE SET NULL, + base_price NUMERIC(10,2) NOT NULL CHECK (base_price >= 0) +); + +-- 5. product_variants +CREATE TABLE product_variants ( + id SERIAL PRIMARY KEY, + product_id INTEGER NOT NULL REFERENCES products(id) ON DELETE CASCADE, + variant_name VARCHAR(80) NOT NULL, + sku VARCHAR(40) UNIQUE NOT NULL, + price NUMERIC(10,2) NOT NULL CHECK (price >= 0) +); + +-- 6. suppliers +CREATE TABLE suppliers ( + id SERIAL PRIMARY KEY, + name VARCHAR(120) NOT NULL, + contact_email VARCHAR(120), + country VARCHAR(60) +); + +-- 7. product_suppliers (M:N) +CREATE TABLE product_suppliers ( + product_id INTEGER NOT NULL REFERENCES products(id) ON DELETE CASCADE, + supplier_id INTEGER NOT NULL REFERENCES suppliers(id) ON DELETE CASCADE, + lead_time_days INTEGER NOT NULL DEFAULT 7, + cost NUMERIC(10,2), + PRIMARY KEY (product_id, supplier_id) +); + +-- 8. inventory +CREATE TABLE inventory ( + id SERIAL PRIMARY KEY, + variant_id INTEGER NOT NULL REFERENCES product_variants(id) ON DELETE CASCADE, + warehouse VARCHAR(40) NOT NULL, + quantity INTEGER NOT NULL DEFAULT 0 CHECK (quantity >= 0), + last_updated TIMESTAMP NOT NULL DEFAULT NOW() +); + +-- 9. orders +CREATE TABLE orders ( + id SERIAL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + address_id INTEGER REFERENCES addresses(id), + status VARCHAR(20) NOT NULL DEFAULT 'pending', + total NUMERIC(12,2) NOT NULL DEFAULT 0, + ordered_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +-- 10. order_items +CREATE TABLE order_items ( + id SERIAL PRIMARY KEY, + order_id INTEGER NOT NULL REFERENCES orders(id) ON DELETE CASCADE, + variant_id INTEGER NOT NULL REFERENCES product_variants(id), + quantity INTEGER NOT NULL CHECK (quantity > 0), + unit_price NUMERIC(10,2) NOT NULL +); + +-- 11. payments +CREATE TABLE payments ( + id SERIAL PRIMARY KEY, + order_id INTEGER NOT NULL REFERENCES orders(id) ON DELETE CASCADE, + method VARCHAR(30) NOT NULL, + amount NUMERIC(12,2) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + paid_at TIMESTAMP +); + +-- 12. reviews +CREATE TABLE reviews ( + id SERIAL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + product_id INTEGER NOT NULL REFERENCES products(id) ON DELETE CASCADE, + rating SMALLINT NOT NULL CHECK (rating BETWEEN 1 AND 5), + comment TEXT, + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + UNIQUE (user_id, product_id) +); + +-- ============================================================ +-- Fake data +-- ============================================================ + +INSERT INTO users (email, full_name, signup_date, is_active) VALUES + ('alice@example.com', 'Alice Cooper', '2024-01-15', TRUE), + ('bob@example.com', 'Bob Dylan', '2024-02-03', TRUE), + ('carol@example.com', 'Carol King', '2024-02-19', TRUE), + ('dave@example.com', 'Dave Grohl', '2024-03-08', FALSE), + ('eve@example.com', 'Eve Polastri', '2024-04-21', TRUE), + ('frank@example.com', 'Frank Ocean', '2024-05-30', TRUE), + ('grace@example.com', 'Grace Hopper', '2024-06-12', TRUE), + ('hank@example.com', 'Hank Williams', '2024-07-04', FALSE); + +INSERT INTO addresses (user_id, street, city, state, postal_code, country, is_default) VALUES + (1, '123 Main St', 'New York', 'NY', '10001', 'USA', TRUE), + (2, '456 Oak Ave', 'Los Angeles', 'CA', '90001', 'USA', TRUE), + (3, '789 Pine Rd', 'New York', 'NY', '10002', 'USA', TRUE), + (4, '12 Maple Dr', 'Chicago', 'IL', '60601', 'USA', TRUE), + (5, '34 Birch Ln', 'London', NULL, 'EC1A', 'UK', TRUE), + (6, '56 Cedar St', 'Toronto', 'ON', 'M5H', 'Canada', TRUE), + (7, '78 Elm Way', 'Austin', 'TX', '73301', 'USA', TRUE), + (8, '90 Spruce Ct', 'Berlin', NULL, '10115', 'Germany',TRUE), + (1, '200 Wall St', 'New York', 'NY', '10005', 'USA', FALSE); + +INSERT INTO categories (name, parent_id) VALUES + ('Electronics', NULL), -- 1 + ('Computers', 1), -- 2 + ('Audio', 1), -- 3 + ('Home', NULL), -- 4 + ('Kitchen', 4), -- 5 + ('Furniture', 4), -- 6 + ('Books', NULL); -- 7 + +INSERT INTO products (sku, name, description, category_id, base_price) VALUES + ('LAP-001', 'UltraBook 14', 'Lightweight 14-inch laptop', 2, 1199.00), + ('LAP-002', 'GamerPro 17', '17-inch gaming laptop', 2, 1899.00), + ('HDP-010', 'StudioPhones X', 'Over-ear studio headphones', 3, 299.00), + ('SPK-020', 'BoomBox Mini', 'Portable bluetooth speaker', 3, 89.00), + ('CHR-001', 'ErgoChair Pro', 'Ergonomic office chair', 6, 549.00), + ('TBL-001', 'Walnut Desk', 'Solid walnut writing desk', 6, 799.00), + ('PAN-001', 'CastIron Pan 12in', '12-inch cast iron skillet', 5, 59.00), + ('KNF-001', 'Chef Knife 8in', 'High-carbon chef knife', 5, 129.00), + ('BK-001', 'Database Internals', 'Book on database systems', 7, 45.00), + ('BK-002', 'Designing Data-Intensive Apps', 'The DDIA classic', 7, 55.00); + +INSERT INTO product_variants (product_id, variant_name, sku, price) VALUES + (1, '16GB / 512GB', 'LAP-001-A', 1199.00), + (1, '32GB / 1TB', 'LAP-001-B', 1499.00), + (2, 'RTX 4070', 'LAP-002-A', 1899.00), + (2, 'RTX 4080', 'LAP-002-B', 2299.00), + (3, 'Black', 'HDP-010-K', 299.00), + (3, 'White', 'HDP-010-W', 299.00), + (4, 'Black', 'SPK-020-K', 89.00), + (4, 'Red', 'SPK-020-R', 89.00), + (5, 'Standard', 'CHR-001-S', 549.00), + (6, 'Standard', 'TBL-001-S', 799.00), + (7, 'Standard', 'PAN-001-S', 59.00), + (8, 'Standard', 'KNF-001-S', 129.00), + (9, 'Paperback', 'BK-001-P', 45.00), + (10, 'Paperback', 'BK-002-P', 55.00); + +INSERT INTO suppliers (name, contact_email, country) VALUES + ('Acme Components', 'sales@acme.com', 'USA'), + ('Global Audio Ltd', 'orders@globalaudio.co.uk', 'UK'), + ('NordKitchen AB', 'hello@nordkitchen.se', 'Sweden'), + ('TimberWorks', 'contact@timberworks.ca', 'Canada'), + ('PaperPress Co', 'sales@paperpress.com', 'USA'); + +INSERT INTO product_suppliers (product_id, supplier_id, lead_time_days, cost) VALUES + (1, 1, 10, 850.00), + (2, 1, 14, 1300.00), + (3, 2, 7, 180.00), + (4, 2, 5, 45.00), + (5, 4, 21, 320.00), + (6, 4, 28, 450.00), + (7, 3, 14, 28.00), + (8, 3, 10, 65.00), + (9, 5, 3, 18.00), + (10, 5, 3, 22.00), + (3, 1, 14, 175.00); -- StudioPhones also supplied by Acme + +INSERT INTO inventory (variant_id, warehouse, quantity) VALUES + (1, 'NYC-1', 42), + (2, 'NYC-1', 15), + (3, 'LA-1', 8), + (4, 'LA-1', 3), + (5, 'NYC-1', 60), + (6, 'NYC-1', 25), + (7, 'LA-1', 120), + (8, 'LA-1', 80), + (9, 'NYC-1', 12), + (10, 'NYC-1', 7), + (11, 'CHI-1', 200), + (12, 'CHI-1', 95), + (13, 'NYC-1', 300), + (14, 'NYC-1', 280); + +INSERT INTO orders (user_id, address_id, status, total, ordered_at) VALUES + (1, 1, 'shipped', 1499.00, '2024-09-01 10:15:00'), + (1, 1, 'delivered', 388.00, '2024-09-12 14:30:00'), + (2, 2, 'delivered', 1899.00, '2024-09-15 09:00:00'), + (3, 3, 'pending', 549.00, '2024-10-01 11:45:00'), + (5, 5, 'shipped', 799.00, '2024-10-05 16:20:00'), + (6, 6, 'delivered', 188.00, '2024-10-08 12:10:00'), + (7, 7, 'cancelled', 299.00, '2024-10-10 18:00:00'), + (1, 1, 'delivered', 100.00, '2024-10-15 08:30:00'); + +INSERT INTO order_items (order_id, variant_id, quantity, unit_price) VALUES + (1, 2, 1, 1499.00), + (2, 5, 1, 299.00), + (2, 7, 1, 89.00), + (3, 3, 1, 1899.00), + (4, 9, 1, 549.00), + (5, 10, 1, 799.00), + (6, 8, 1, 89.00), + (6, 11, 1, 59.00), + (6, 12, 1, 129.00), -- intentional small mismatch from total above + (7, 6, 1, 299.00), + (8, 13, 1, 45.00), + (8, 14, 1, 55.00); + +INSERT INTO payments (order_id, method, amount, status, paid_at) VALUES + (1, 'credit_card', 1499.00, 'completed', '2024-09-01 10:16:00'), + (2, 'paypal', 388.00, 'completed', '2024-09-12 14:31:00'), + (3, 'credit_card', 1899.00, 'completed', '2024-09-15 09:01:00'), + (4, 'credit_card', 549.00, 'pending', NULL), + (5, 'bank_transfer',799.00, 'completed', '2024-10-05 17:00:00'), + (6, 'credit_card', 277.00, 'completed', '2024-10-08 12:11:00'), + (7, 'credit_card', 299.00, 'refunded', '2024-10-10 18:05:00'), + (8, 'paypal', 100.00, 'completed', '2024-10-15 08:31:00'); + +INSERT INTO reviews (user_id, product_id, rating, comment, created_at) VALUES + (1, 1, 5, 'Fast and light, perfect for travel.', '2024-09-20'), + (1, 3, 4, 'Great sound, ear cushions wear quickly.', '2024-09-25'), + (2, 2, 5, 'Runs every game on max settings.', '2024-09-22'), + (3, 5, 3, 'Comfortable but expensive.', '2024-10-12'), + (5, 6, 5, 'Beautiful desk, sturdy and well finished.', '2024-10-15'), + (6, 4, 4, 'Loud and clear, battery life could be better.','2024-10-18'), + (6, 7, 5, 'Heats evenly, sears beautifully.', '2024-10-20'), + (6, 8, 4, 'Sharp out of the box.', '2024-10-21'), + (1, 9, 5, 'Best technical book of the year.', '2024-10-25'), + (1, 10, 5, 'A modern classic, highly recommend.', '2024-10-26'); + +-- Quick sanity counts +SELECT 'users' AS table, COUNT(*) FROM users UNION ALL +SELECT 'addresses', COUNT(*) FROM addresses UNION ALL +SELECT 'categories', COUNT(*) FROM categories UNION ALL +SELECT 'products', COUNT(*) FROM products UNION ALL +SELECT 'product_variants', COUNT(*) FROM product_variants UNION ALL +SELECT 'suppliers', COUNT(*) FROM suppliers UNION ALL +SELECT 'product_suppliers', COUNT(*) FROM product_suppliers UNION ALL +SELECT 'inventory', COUNT(*) FROM inventory UNION ALL +SELECT 'orders', COUNT(*) FROM orders UNION ALL +SELECT 'order_items', COUNT(*) FROM order_items UNION ALL +SELECT 'payments', COUNT(*) FROM payments UNION ALL +SELECT 'reviews', COUNT(*) FROM reviews; diff --git a/package-lock.json b/package-lock.json index 2de71fc3..a1b318c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,9 @@ { - "name": "QueryWeaver3", + "name": "QueryWeaver", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "QueryWeaver3", "dependencies": { "queryweaver-app": "file:app" }, @@ -65,10 +64,10 @@ "react-dom": "^18.3.1", "react-hook-form": "^7.71.2", "react-resizable-panels": "^2.1.9", - "react-router-dom": "^7.13.2", + "react-router-dom": "^7.14.0", "recharts": "^2.15.4", "sonner": "^1.7.4", - "tailwind-merge": "^2.6.0", + "tailwind-merge": "^3.5.0", "tailwindcss-animate": "^1.0.7", "vaul": "^0.9.9", "zod": "^3.25.76" @@ -89,7 +88,7 @@ "tailwindcss": "^3.4.17", "typescript": "^5.8.3", "typescript-eslint": "^8.57.0", - "vite": "^7.3.0" + "vite": "^7.3.2" } }, "app/node_modules/@alloc/quick-lru": { @@ -110,21 +109,6 @@ "node": ">=6.9.0" } }, - "app/node_modules/@esbuild/linux-x64": { - "version": "0.27.2", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, "app/node_modules/@floating-ui/core": { "version": "1.7.3", "license": "MIT", @@ -2038,7 +2022,6 @@ "version": "22.19.7", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -2052,7 +2035,6 @@ "version": "18.3.27", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -2062,7 +2044,6 @@ "version": "18.3.7", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^18.0.0" } @@ -2194,7 +2175,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -2319,7 +2299,6 @@ "app/node_modules/date-fns": { "version": "3.6.0", "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/kossnocorp" @@ -2358,8 +2337,7 @@ }, "app/node_modules/embla-carousel": { "version": "8.6.0", - "license": "MIT", - "peer": true + "license": "MIT" }, "app/node_modules/embla-carousel-react": { "version": "8.6.0", @@ -2379,46 +2357,6 @@ "embla-carousel": "8.6.0" } }, - "app/node_modules/esbuild": { - "version": "0.27.2", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.27.2", - "@esbuild/android-arm": "0.27.2", - "@esbuild/android-arm64": "0.27.2", - "@esbuild/android-x64": "0.27.2", - "@esbuild/darwin-arm64": "0.27.2", - "@esbuild/darwin-x64": "0.27.2", - "@esbuild/freebsd-arm64": "0.27.2", - "@esbuild/freebsd-x64": "0.27.2", - "@esbuild/linux-arm": "0.27.2", - "@esbuild/linux-arm64": "0.27.2", - "@esbuild/linux-ia32": "0.27.2", - "@esbuild/linux-loong64": "0.27.2", - "@esbuild/linux-mips64el": "0.27.2", - "@esbuild/linux-ppc64": "0.27.2", - "@esbuild/linux-riscv64": "0.27.2", - "@esbuild/linux-s390x": "0.27.2", - "@esbuild/linux-x64": "0.27.2", - "@esbuild/netbsd-arm64": "0.27.2", - "@esbuild/netbsd-x64": "0.27.2", - "@esbuild/openbsd-arm64": "0.27.2", - "@esbuild/openbsd-x64": "0.27.2", - "@esbuild/openharmony-arm64": "0.27.2", - "@esbuild/sunos-x64": "0.27.2", - "@esbuild/win32-arm64": "0.27.2", - "@esbuild/win32-ia32": "0.27.2", - "@esbuild/win32-x64": "0.27.2" - } - }, "app/node_modules/escalade": { "version": "3.2.0", "dev": true, @@ -2611,7 +2549,6 @@ "version": "1.21.7", "dev": true, "license": "MIT", - "peer": true, "bin": { "jiti": "bin/jiti.js" } @@ -3157,19 +3094,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "app/node_modules/tailwind-merge": { - "version": "2.6.0", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/dcastil" - } - }, "app/node_modules/tailwindcss": { "version": "3.4.18", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -3368,197 +3296,552 @@ "d3-timer": "^3.0.1" } }, - "app/node_modules/vite": { - "version": "7.3.1", - "dev": true, + "app/node_modules/zod": { + "version": "3.25.76", "license": "MIT", - "peer": true, - "dependencies": { - "esbuild": "^0.27.0", - "fdir": "^6.5.0", - "picomatch": "^4.0.3", - "postcss": "^8.5.6", - "rollup": "^4.43.0", - "tinyglobby": "^0.2.15" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "lightningcss": "^1.21.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "lightningcss": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } + "url": "https://github.com/sponsors/colinhacks" } }, - "app/node_modules/vite/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "aix" + ], "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" + "node": ">=18" } }, - "app/node_modules/zod": { - "version": "3.25.76", + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" } }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", - "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^3.4.3" - }, + "optional": true, + "os": [ + "android" + ], "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + "node": ">=18" } }, - "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], "dev": true, - "license": "Apache-2.0", + "license": "MIT", + "optional": true, + "os": [ + "android" + ], "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" + "node": ">=18" } }, - "node_modules/@eslint-community/regexpp": { - "version": "4.12.2", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", - "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + "node": ">=18" } }, - "node_modules/@eslint/config-array": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", - "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/object-schema": "^2.1.7", - "debug": "^4.3.1", - "minimatch": "^3.1.5" - }, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": ">=18" } }, - "node_modules/@eslint/config-helpers": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", - "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.17.0" - }, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": ">=18" } }, - "node_modules/@eslint/core": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", - "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@types/json-schema": "^7.0.15" - }, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": ">=18" } }, - "node_modules/@eslint/eslintrc": { - "version": "3.3.5", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", - "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], "dev": true, "license": "MIT", - "dependencies": { - "ajv": "^6.14.0", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.1", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", + "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.5" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", + "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.14.0", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.1", "minimatch": "^3.1.5", "strip-json-comments": "^3.1.1" }, @@ -4133,7 +4416,6 @@ "integrity": "sha512-30ScMRHIAD33JJQkgfGW1t8CURZtjc2JpTrq5n2HFhOefbAhb7ucc7xJwdWcrEtqUIYJ73Nybpsggii6GtAHjA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.57.2", "@typescript-eslint/types": "8.57.2", @@ -4386,7 +4668,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4920,7 +5201,6 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", - "peer": true, "engines": { "node": ">=12" } @@ -5039,6 +5319,48 @@ "robust-predicates": "^3.0.2" } }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -5058,7 +5380,6 @@ "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -5759,7 +6080,6 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -5815,7 +6135,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -5864,7 +6183,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -5877,7 +6195,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "license": "MIT", - "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -5891,7 +6208,6 @@ "resolved": "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.71.2.tgz", "integrity": "sha512-1CHvcDYzuRUNOflt4MOq3ZM46AronNJtQ1S7tnX6YN4y72qhgiUItpacZUAQ0TyWYci3yz1X+rXaSxiuEm86PA==", "license": "MIT", - "peer": true, "engines": { "node": ">=18.0.0" }, @@ -6101,6 +6417,16 @@ "node": ">=8" } }, + "node_modules/tailwind-merge": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.5.0.tgz", + "integrity": "sha512-I8K9wewnVDkL1NTGoqWmVEIlUcB9gFriAEkXkfCjX5ib8ezGxtR3xD7iZIxrfArjEsH7F1CHD4RFUtxefdqV/A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/dcastil" + } + }, "node_modules/tailwindcss": { "version": "4.1.17", "license": "MIT", @@ -6168,7 +6494,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6216,6 +6541,96 @@ "punycode": "^2.1.0" } }, + "node_modules/vite": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.2.tgz", + "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.27.0", + "fdir": "^6.5.0", + "picomatch": "^4.0.3", + "postcss": "^8.5.6", + "rollup": "^4.43.0", + "tinyglobby": "^0.2.15" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "lightningcss": "^1.21.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/pyproject.toml b/pyproject.toml index e05d2839..f4b702d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,29 +1,75 @@ [project] name = "queryweaver" -version = "0.1.0" -description = "QueryWeaver - Text2SQL using graph-powered schema understanding" +version = "0.2.0" +description = "Text2SQL tool that transforms natural language into SQL using graph-powered schema understanding" readme = "README.md" +license = "AGPL-3.0-or-later" requires-python = ">=3.12" +authors = [ + { name = "FalkorDB", email = "support@falkordb.com" } +] +keywords = ["text2sql", "sql", "nlp", "llm", "database", "falkordb"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Database", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +# Core dependencies required for SDK (minimal) dependencies = [ - "fastapi~=0.136.0", - "uvicorn~=0.44.0", "litellm>=1.83.0", "falkordb~=1.6.0", "psycopg2-binary~=2.9.11", "pymysql~=1.1.0", - "authlib~=1.7.0", - "itsdangerous~=2.2.0", "jsonschema~=4.26.0", "tqdm~=4.67.3", +] + +[project.optional-dependencies] +# Server dependencies (FastAPI, auth, etc.) +server = [ + "fastapi~=0.136.0", + "uvicorn~=0.44.0", + "authlib~=1.7.0", + "itsdangerous~=2.2.0", "python-multipart~=0.0.10", "jinja2~=3.1.4", - "graphiti-core>=0.28.1", "fastmcp>=3.2.4", + "graphiti-core>=0.28.1", "snowflake-connector-python~=4.4.0", "python-dotenv~=1.2.2", "aiohttp>=3.13.5", ] +# Development dependencies +dev = [ + "pytest~=9.0.3", + "pytest-asyncio~=1.3.0", + "pylint~=4.0.3", + "playwright~=1.58.0", + "pytest-playwright~=0.7.1", +] + +# All dependencies (server + dev) +all = [ + "queryweaver[server]", + "queryweaver[dev]", +] + +[project.urls] +Homepage = "https://github.com/FalkorDB/QueryWeaver" +Documentation = "https://github.com/FalkorDB/QueryWeaver#readme" +Repository = "https://github.com/FalkorDB/QueryWeaver" +Issues = "https://github.com/FalkorDB/QueryWeaver/issues" + +[project.scripts] +queryweaver = "api.index:main" + [dependency-groups] dev = [ "pytest~=9.0.3", @@ -41,7 +87,15 @@ build-backend = "hatchling.build" allow-direct-references = true [tool.hatch.build.targets.wheel] -packages = ["api"] +packages = ["queryweaver", "api"] + +[tool.hatch.build.targets.sdist] +include = [ + "/queryweaver", + "/api", + "/README.md", + "/LICENSE", +] [tool.uv] package = true @@ -52,6 +106,8 @@ python_files = ["test_*.py"] python_classes = ["Test*"] python_functions = ["test_*"] addopts = "--verbose --tb=short --strict-markers --disable-warnings" +asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" markers = [ "e2e: End-to-end tests using Playwright", "slow: Tests that take a long time to run", @@ -65,10 +121,15 @@ filterwarnings = [ ] [tool.pylint.main] +max-line-length = 120 +ignore-patterns = ["test_.*\\.py", "conftest\\.py"] + +[tool.pylint.messages_control] disable = [ "C0114", # missing-module-docstring "C0115", # missing-class-docstring "C0116", # missing-function-docstring + "R0903", # too-few-public-methods ] [tool.pylint.format] diff --git a/queryweaver/__init__.py b/queryweaver/__init__.py new file mode 100644 index 00000000..933a9d39 --- /dev/null +++ b/queryweaver/__init__.py @@ -0,0 +1,53 @@ +"""QueryWeaver SDK - Text2SQL without a server. + +This package provides a Python SDK for QueryWeaver's text-to-SQL +functionality, allowing you to convert natural language questions +to SQL queries directly in your Python applications. + +Example: + ```python + from queryweaver import QueryWeaver + + async def main(): + qw = QueryWeaver(falkordb_url="redis://localhost:6379") + await qw.connect_database("postgresql://user:pass@host/mydb") + + result = await qw.query("mydb", "Show me all customers from NYC") + print(result.sql_query) # SELECT * FROM customers WHERE city = 'NYC' + print(result.results) # [{"id": 1, "name": "John", "city": "NYC"}, ...] + print(result.ai_response) # "Found 42 customers from New York City..." + ``` + +Requirements: + - FalkorDB instance (local or remote) + - OpenAI or Azure OpenAI API key + - Target SQL database (PostgreSQL or MySQL) +""" + +from queryweaver.client import QueryWeaver +from queryweaver.models import ( + QueryResult, + QueryMetadata, + QueryAnalysis, + SchemaResult, + DatabaseConnection, + RefreshResult, + QueryRequest, + ChatMessage, +) +from queryweaver.connection import FalkorDBConnection + +__all__ = [ + "QueryWeaver", + "QueryResult", + "QueryMetadata", + "QueryAnalysis", + "SchemaResult", + "DatabaseConnection", + "RefreshResult", + "QueryRequest", + "ChatMessage", + "FalkorDBConnection", +] + +__version__ = "0.2.0" diff --git a/queryweaver/client.py b/queryweaver/client.py new file mode 100644 index 00000000..cd539a81 --- /dev/null +++ b/queryweaver/client.py @@ -0,0 +1,317 @@ +"""QueryWeaver SDK - Python client for Text2SQL functionality. + +This module provides the main QueryWeaver class for converting natural +language questions to SQL queries without requiring a web server. + +Note: This module uses lazy imports (import-outside-toplevel) intentionally. +The api.* modules do not need to be loaded until an SDK method is called, +so deferring their import keeps `from queryweaver import QueryWeaver` +cheap and side-effect-free. + +Example usage: + ```python + from queryweaver import QueryWeaver + + async def main(): + qw = QueryWeaver(falkordb_url="redis://localhost:6379") + await qw.connect_database("postgresql://user:pass@host/mydb") + + result = await qw.query("mydb", "Show me all customers from NYC") + print(result.sql_query) + print(result.results) + ``` +""" +# pylint: disable=import-outside-toplevel +# Lazy imports are required - see module docstring for explanation + +import asyncio +from contextlib import contextmanager +from typing import Optional, Union + +from queryweaver.connection import FalkorDBConnection +from queryweaver.models import ( + QueryResult, + SchemaResult, + DatabaseConnection, + RefreshResult, + QueryRequest, +) + + +class QueryWeaver: + """Python SDK for Text2SQL functionality. + + This class provides a programmatic interface to QueryWeaver's text-to-SQL + capabilities without requiring a running web server. + + Attributes: + user_id: Identifier for namespacing databases (default: "default"). + """ + + def __init__( + self, + falkordb_url: Optional[str] = None, + user_id: str = "default", + ): + """Initialize QueryWeaver SDK. + + Multiple QueryWeaver instances can coexist in the same process — + each holds its own FalkorDB connection and passes it explicitly + into core functions, so there is no shared global state to collide. + + Args: + falkordb_url: Redis URL for FalkorDB connection. + Falls back to FALKORDB_URL environment variable. + user_id: User identifier for database namespacing. + Defaults to "default" for single-user scenarios. + + Raises: + ConnectionError: If FalkorDB connection cannot be established. + """ + self._user_id = user_id + self._connection = FalkorDBConnection(url=falkordb_url) + # Set of in-flight background tasks (memory writes) so close() can + # await them. Populated via the ``background_tasks_var`` contextvar + # in ``api.core.pipeline``. + self._pending_tasks: set = set() + + @property + def _db(self): + """The FalkorDB handle for this SDK instance.""" + return self._connection.db + + @contextmanager + def _bind_task_sink(self): + """Bind this instance's task sink to the current contextvar scope. + + Use as a context manager around any call that may schedule + background memory writes; close() then awaits them before the pool + is disconnected. + """ + from api.core.pipeline import background_tasks_var + token = background_tasks_var.set(self._pending_tasks) + try: + yield + finally: + background_tasks_var.reset(token) + + @property + def user_id(self) -> str: + """Get the user ID used for database namespacing.""" + return self._user_id + + async def connect_database(self, db_url: str) -> DatabaseConnection: + """Connect to a SQL database and load its schema. + + This method connects to the specified database, introspects its schema, + and loads it into FalkorDB for query processing. + + Args: + db_url: Database connection URL. Supported formats: + - PostgreSQL: "postgresql://user:pass@host:port/dbname" + - MySQL: "mysql://user:pass@host:port/dbname" + + Returns: + DatabaseConnection with connection status and details. + + Raises: + api.core.errors.InvalidArgumentError: If the database URL format is + invalid (empty, unknown scheme, or unsupported vendor for the + installed extras). + """ + from api.core.schema_loader import load_database_sync + return await load_database_sync(db_url, self._user_id, db=self._db) + + async def query( + self, + database: str, + question: Union[str, QueryRequest], + ) -> QueryResult: + """Convert natural language to SQL and execute. + + Can be called with a simple question string or a QueryRequest for advanced options. + + Args: + database: The database identifier to query. + question: Either a natural language question string, or a QueryRequest + object with full conversation context and options. + + Returns: + QueryResult with SQL query, results, and AI response. + + Raises: + ValueError: If the question is empty or database not found. + + Examples: + Simple usage: + result = await qw.query("mydb", "Show all customers") + + Advanced usage with context: + request = QueryRequest( + question="Show their orders", + chat_history=["Show all customers"], + result_history=["Found 10 customers"], + instructions="Use customer_id for joins", + ) + result = await qw.query("mydb", request) + """ + from api.core.text2sql import ChatRequest, collect_result, run_query + + # Handle both string and QueryRequest inputs + if isinstance(question, str): + if not question or not question.strip(): + raise ValueError("Question cannot be empty") + request = QueryRequest(question=question) + else: + request = question + if not request.question or not request.question.strip(): + raise ValueError("Question cannot be empty") + + # Build chat history with current question + history = list(request.chat_history or []) + history.append(request.question) + + chat_data = ChatRequest( + chat=history, + result=request.result_history, + instructions=request.instructions, + use_user_rules=request.use_user_rules, + use_memory=request.use_memory, + custom_api_key=request.custom_api_key, + custom_model=request.custom_model, + ) + + with self._bind_task_sink(): + return await collect_result( + run_query(self._user_id, database, chat_data, db=self._db) + ) + + async def get_schema(self, database: str) -> SchemaResult: + """Get the schema for a connected database. + + Args: + database: The database identifier. + + Returns: + SchemaResult with tables (nodes) and relationships (links). + + Raises: + ValueError: If the database is not found. + """ + from api.core.text2sql import get_schema as _get_schema + schema = await _get_schema(self._user_id, database, db=self._db) + return SchemaResult( + nodes=schema.get("nodes", []), + links=schema.get("links", []), + ) + + async def list_databases(self) -> list[str]: + """List all available databases for this user. + + Returns: + List of database identifiers. + """ + from api.core.schema_loader import list_databases as _list_databases # pylint: disable=import-outside-toplevel + from api.core.pipeline import GENERAL_PREFIX # pylint: disable=import-outside-toplevel + return await _list_databases(self._user_id, GENERAL_PREFIX, db=self._db) + + async def delete_database(self, database: str) -> bool: + """Delete a connected database. + + This removes the database schema from FalkorDB. It does not + affect the actual SQL database. + + Args: + database: The database identifier to delete. + + Returns: + True if deletion was successful. + + Raises: + ValueError: If the database is not found or cannot be deleted. + """ + from api.core.text2sql import delete_database as _delete_database + result = await _delete_database(self._user_id, database, db=self._db) + return result.get("success", False) + + async def refresh_schema(self, database: str) -> RefreshResult: + """Refresh the schema for a connected database. + + Re-introspects the source database and updates the schema graph. + Useful after schema changes in the source database. + + Args: + database: The database identifier to refresh. + + Returns: + RefreshResult with refresh status. + + Raises: + ValueError: If the database is not found. + """ + from api.core.text2sql import refresh_schema_for_sdk + return await refresh_schema_for_sdk(self._user_id, database, db=self._db) + + async def execute_confirmed( # pylint: disable=too-many-arguments,too-many-positional-arguments + self, + database: str, + sql_query: str, + chat_history: Optional[list[str]] = None, + custom_api_key: Optional[str] = None, + custom_model: Optional[str] = None, + ) -> QueryResult: + """Execute a confirmed destructive SQL operation. + + Use this method to execute INSERT, UPDATE, DELETE, or other + destructive operations that were flagged for confirmation. + + Args: + database: The database identifier. + sql_query: The SQL query to execute. + chat_history: Conversation context. + custom_api_key: Per-request override for the LLM API key. + custom_model: Per-request override for the LLM model + (``vendor/model`` format, e.g. ``openai/gpt-4.1``). + + Returns: + QueryResult with execution results. + """ + from api.core.text2sql import ConfirmRequest, collect_result, run_confirmed + + confirm_data = ConfirmRequest( + sql_query=sql_query, + confirmation="CONFIRM", + chat=chat_history or [], + custom_api_key=custom_api_key, + custom_model=custom_model, + ) + + with self._bind_task_sink(): + return await collect_result( + run_confirmed(self._user_id, database, confirm_data, db=self._db) + ) + + async def close(self) -> None: + """Close the SDK connection and release resources. + + Awaits any in-flight background memory writes so they land before + the FalkorDB connection pool is released. Drains in a loop because + ``save_memory_background`` registers ``sink.discard`` as a done + callback and any awaited task can schedule further tasks via the + same contextvar sink. + """ + while self._pending_tasks: + # Snapshot before awaiting — the live set mutates from done + # callbacks (sink.discard) and would raise "set changed size + # during iteration" if unpacked directly into gather(). + tasks = list(self._pending_tasks) + await asyncio.gather(*tasks, return_exceptions=True) + await self._connection.close() + + async def __aenter__(self) -> "QueryWeaver": + """Async context manager entry.""" + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + """Async context manager exit.""" + await self.close() diff --git a/queryweaver/connection.py b/queryweaver/connection.py new file mode 100644 index 00000000..c5609c36 --- /dev/null +++ b/queryweaver/connection.py @@ -0,0 +1,152 @@ +"""FalkorDB connection management for QueryWeaver SDK.""" + +import os +from typing import Optional + +from falkordb.asyncio import FalkorDB +from redis.asyncio import BlockingConnectionPool + + +class FalkorDBConnection: + """Manages FalkorDB connection lifecycle for the SDK. + + This class provides explicit connection management, allowing users + to initialize connections with specific parameters rather than + relying solely on environment variables. + """ + + def __init__( + self, + url: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + ): + """Initialize FalkorDB connection. + + Args: + url: Redis connection URL (e.g., "redis://localhost:6379"). + Takes precedence over host/port if provided. + host: FalkorDB host (default: "localhost"). + port: FalkorDB port (default: 6379). + + Raises: + ConnectionError: If connection cannot be established. + """ + self._url = url + self._host = host + self._port = port + self._db: Optional[FalkorDB] = None + self._pool: Optional[BlockingConnectionPool] = None + self._closed = False + + @property + def db(self) -> FalkorDB: + """Get the FalkorDB client instance. + + Lazily initializes the connection on first access. + + Returns: + FalkorDB client instance. + + Raises: + ConnectionError: If connection cannot be established. + RuntimeError: If accessed after ``close()`` — prevents silently + spinning up a fresh pool that would never be torn down. + """ + if self._closed: + raise RuntimeError( + "FalkorDBConnection is closed; create a new QueryWeaver instance" + ) + if self._db is None: + self._db = self._create_connection() + return self._db + + def _create_connection(self) -> FalkorDB: + """Create and return a FalkorDB connection. + + Returns: + FalkorDB client instance. + + Raises: + ConnectionError: If connection cannot be established. + """ + # Priority: explicit URL > explicit host/port > env URL > env host/port > defaults + url = self._url or os.getenv("FALKORDB_URL") + + if url: + try: + self._pool = BlockingConnectionPool.from_url( + url, + decode_responses=True + ) + return FalkorDB(connection_pool=self._pool) + except Exception as e: + raise ConnectionError(f"Failed to connect to FalkorDB with URL: {e}") from e + + # Fall back to host/port + host = self._host or os.getenv("FALKORDB_HOST", "localhost") + port = self._port or int(os.getenv("FALKORDB_PORT", "6379")) + + try: + return FalkorDB(host=host, port=port) + except Exception as e: + raise ConnectionError(f"Failed to connect to FalkorDB at {host}:{port}: {e}") from e + + @classmethod + def from_env(cls) -> "FalkorDBConnection": + """Create connection from environment variables. + + Uses FALKORDB_URL if set, otherwise FALKORDB_HOST and FALKORDB_PORT. + + Returns: + FalkorDBConnection instance. + """ + return cls() + + @classmethod + def from_url(cls, url: str) -> "FalkorDBConnection": + """Create connection from a Redis URL. + + Args: + url: Redis connection URL (e.g., "redis://localhost:6379"). + + Returns: + FalkorDBConnection instance. + """ + return cls(url=url) + + async def close(self) -> None: + """Close the connection and release resources. + + Idempotent — repeated calls are safe. After close, the ``db`` + property raises ``RuntimeError`` rather than silently reconnecting. + """ + if self._closed: + return + if self._pool is not None: + await self._pool.disconnect() + self._pool = None + elif self._db is not None: + # Non-pooled connection (created via host/port) — close directly + await self._db.connection.aclose() + self._db = None + self._closed = True + + def select_graph(self, graph_id: str): + """Select a graph by ID. + + Args: + graph_id: The graph identifier. + + Returns: + Graph instance for the specified ID. + """ + return self.db.select_graph(graph_id) + + async def list_graphs(self) -> list[str]: + """List all available graphs. + + Returns: + List of graph names. + """ + return await self.db.list_graphs() diff --git a/queryweaver/models.py b/queryweaver/models.py new file mode 100644 index 00000000..b689622e --- /dev/null +++ b/queryweaver/models.py @@ -0,0 +1,29 @@ +"""Data models for QueryWeaver SDK results. + +Thin re-export shim. The dataclasses live in ``api.core.result_models`` / +``api.core.request_models`` so the server code and the SDK share a single +source of truth. Importing from ``queryweaver.models`` still works for +external consumers. +""" + +from api.core.request_models import QueryRequest +from api.core.result_models import ( + ChatMessage, + DatabaseConnection, + QueryAnalysis, + QueryMetadata, + QueryResult, + RefreshResult, + SchemaResult, +) + +__all__ = [ + "ChatMessage", + "DatabaseConnection", + "QueryAnalysis", + "QueryMetadata", + "QueryRequest", + "QueryResult", + "RefreshResult", + "SchemaResult", +] diff --git a/tests/test_sdk/__init__.py b/tests/test_sdk/__init__.py new file mode 100644 index 00000000..db46e476 --- /dev/null +++ b/tests/test_sdk/__init__.py @@ -0,0 +1 @@ +"""Test SDK module marker.""" diff --git a/tests/test_sdk/conftest.py b/tests/test_sdk/conftest.py new file mode 100644 index 00000000..81496679 --- /dev/null +++ b/tests/test_sdk/conftest.py @@ -0,0 +1,168 @@ +"""Test fixtures for QueryWeaver SDK integration tests.""" + +import os +import pytest +from urllib.parse import urlparse + + +def pytest_configure(config): + """Configure pytest with custom markers.""" + config.addinivalue_line( + "markers", "requires_llm: mark test as requiring LLM API key" + ) + config.addinivalue_line( + "markers", "requires_postgres: mark test as requiring PostgreSQL" + ) + config.addinivalue_line( + "markers", "requires_mysql: mark test as requiring MySQL" + ) + + +@pytest.fixture(scope="session") +def falkordb_url(): + """Provide FalkorDB connection URL. + + Expects FalkorDB running (via `make docker-test-services` or CI service). + """ + url = os.getenv("FALKORDB_URL", "redis://localhost:6379") + + # Verify connection + from falkordb import FalkorDB + try: + db = FalkorDB.from_url(url) + db.connection.ping() + except Exception as e: + pytest.skip(f"FalkorDB not available at {url}: {e}") + + return url + + +@pytest.fixture(scope="session") +def postgres_url(): + """Provide PostgreSQL connection URL with test database. + + Expects PostgreSQL running (via `make docker-test-services` or CI service). + """ + url = os.getenv("TEST_POSTGRES_URL", "postgresql://postgres:postgres@localhost:5432/testdb") + + # Verify connection and create test schema + import psycopg2 + conn = None + try: + conn = psycopg2.connect(url) + cursor = conn.cursor() + + # Create test tables (DROP + CREATE ensures a clean slate) + cursor.execute(""" + DROP TABLE IF EXISTS orders CASCADE; + DROP TABLE IF EXISTS customers CASCADE; + + CREATE TABLE customers ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + email VARCHAR(100) UNIQUE, + city VARCHAR(100) + ); + + CREATE TABLE orders ( + id SERIAL PRIMARY KEY, + customer_id INTEGER REFERENCES customers(id), + product VARCHAR(100), + amount DECIMAL(10,2), + order_date DATE + ); + + -- Insert test data (UNIQUE on email allows ON CONFLICT) + INSERT INTO customers (name, email, city) VALUES + ('Alice Smith', 'alice@example.com', 'New York'), + ('Bob Jones', 'bob@example.com', 'Los Angeles'), + ('Carol White', 'carol@example.com', 'New York') + ON CONFLICT (email) DO NOTHING; + + INSERT INTO orders (customer_id, product, amount, order_date) VALUES + (1, 'Widget', 29.99, '2024-01-15'), + (1, 'Gadget', 49.99, '2024-01-20'), + (2, 'Widget', 29.99, '2024-02-01'); + """) + conn.commit() + except Exception as e: + pytest.skip(f"PostgreSQL not available: {e}") + finally: + if conn is not None: + conn.close() + + return url + + +@pytest.fixture(scope="session") +def mysql_url(): + """Provide MySQL connection URL with test database. + + Expects MySQL running (via `make docker-test-services` or CI service). + """ + url = os.getenv("TEST_MYSQL_URL", "mysql://root:root@localhost:3306/testdb") + + # Parse connection parameters from the URL + parsed = urlparse(url) + host = parsed.hostname or "localhost" + port = parsed.port or 3306 + user = parsed.username or "root" + password = parsed.password or "root" + database = parsed.path.lstrip("/") or "testdb" + + # Verify connection and create test schema + import pymysql + conn = None + try: + conn = pymysql.connect( + host=host, + port=port, + user=user, + password=password, + database=database, + ) + cursor = conn.cursor() + + # Create test tables + cursor.execute("DROP TABLE IF EXISTS products") + cursor.execute(""" + CREATE TABLE IF NOT EXISTS products ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(100) NOT NULL, + category VARCHAR(50), + price DECIMAL(10,2) + ) + """) + + cursor.execute(""" + INSERT INTO products (name, category, price) VALUES + ('Laptop', 'Electronics', 999.99), + ('Mouse', 'Electronics', 29.99), + ('Desk', 'Furniture', 199.99) + """) + conn.commit() + except Exception as e: + pytest.skip(f"MySQL not available: {e}") + finally: + if conn is not None: + conn.close() + + return url + + +@pytest.fixture +async def queryweaver(falkordb_url): + """Provide initialized QueryWeaver instance with proper teardown.""" + from queryweaver import QueryWeaver + + qw = QueryWeaver(falkordb_url=falkordb_url, user_id="test_user") + yield qw + await qw.close() + + +@pytest.fixture +def has_llm_key(): + """Check if LLM API key is available.""" + if not os.getenv("OPENAI_API_KEY") and not os.getenv("AZURE_API_KEY"): + pytest.skip("LLM API key required (OPENAI_API_KEY or AZURE_API_KEY)") + return True diff --git a/tests/test_sdk/test_queryweaver.py b/tests/test_sdk/test_queryweaver.py new file mode 100644 index 00000000..cbf806ac --- /dev/null +++ b/tests/test_sdk/test_queryweaver.py @@ -0,0 +1,366 @@ +"""SDK integration tests for QueryWeaver. + +Most integration tests create QueryWeaver instances via ``async with`` so +the connection pool is closed before the test function returns. This +prevents stale Redis futures from leaking into subsequent tests and +surfacing as spurious "Event loop is closed" errors. The ``TestModels`` +class is pure-unit (no FalkorDB/Postgres/LLM) and runs without fixtures. +""" + +import pytest + +from api.core.errors import InvalidArgumentError +from queryweaver import QueryWeaver +from queryweaver.models import ( + DatabaseConnection, + QueryMetadata, + QueryRequest, + QueryResult, + SchemaResult, +) + + +class TestQueryWeaverInit: + """Construction and lifecycle.""" + + def test_init_defaults(self, falkordb_url): + qw = QueryWeaver(falkordb_url=falkordb_url) + assert qw.user_id == "default" + + def test_init_with_custom_user_id(self, falkordb_url): + qw = QueryWeaver(falkordb_url=falkordb_url, user_id="custom_user") + assert qw.user_id == "custom_user" + + @pytest.mark.asyncio + async def test_context_manager(self, falkordb_url): + async with QueryWeaver(falkordb_url=falkordb_url) as qw: + assert qw.user_id == "default" + + @pytest.mark.asyncio + async def test_two_instances_isolated(self, falkordb_url): + """Two SDK instances must not share state (no global mutation). + + This is the regression test for the ``api.extensions.db`` global + that older SDK versions mutated on every __init__. + """ + async with QueryWeaver(falkordb_url=falkordb_url, user_id="a") as qw1: + async with QueryWeaver(falkordb_url=falkordb_url, user_id="b") as qw2: + assert qw1._db is not qw2._db # pylint: disable=protected-access + assert qw1.user_id == "a" + assert qw2.user_id == "b" + + +class TestListDatabases: + """Database listing.""" + + @pytest.mark.asyncio + async def test_list_databases_returns_list(self, queryweaver): + databases = await queryweaver.list_databases() + assert isinstance(databases, list) + + +class TestConnectDatabase: + """Database connection and schema loading.""" + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_connect_postgres(self, falkordb_url, postgres_url, has_llm_key): + # connect_database loads embeddings via the LLM, so it actually requires + # an LLM key — without one the load fails before the schema is persisted. + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_connect_pg") as qw: + result = await qw.connect_database(postgres_url) + try: + assert result.success is True + assert result.database_id == "testdb" + assert "successfully" in result.message.lower() + finally: + # Only clean up if the connect actually persisted a graph; + # otherwise database_id is empty and delete_database rejects it. + if result.success and result.database_id: + await qw.delete_database(result.database_id) + + @pytest.mark.asyncio + @pytest.mark.requires_mysql + async def test_connect_mysql(self, falkordb_url, mysql_url, has_llm_key): + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_connect_mysql") as qw: + result = await qw.connect_database(mysql_url) + try: + assert result.success is True + assert result.database_id == "testdb" + assert "successfully" in result.message.lower() + finally: + if result.success and result.database_id: + await qw.delete_database(result.database_id) + + @pytest.mark.asyncio + async def test_connect_invalid_url(self, queryweaver): + with pytest.raises(InvalidArgumentError): + await queryweaver.connect_database("invalid://url") + + +class TestGetSchema: + """Schema retrieval after connect.""" + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_get_schema(self, falkordb_url, postgres_url, has_llm_key): + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_schema_user") as qw: + conn_result = await qw.connect_database(postgres_url) + try: + assert conn_result.success + + schema = await qw.get_schema(conn_result.database_id) + + assert isinstance(schema.nodes, list) + assert len(schema.nodes) >= 2 + + table_names = [n.get("name", "").lower() for n in schema.nodes] + assert "customers" in table_names + assert "orders" in table_names + assert isinstance(schema.links, list) + finally: + await qw.delete_database(conn_result.database_id) + + +class TestQuery: + """End-to-end query paths.""" + + @pytest.mark.asyncio + async def test_query_empty_question_raises(self, queryweaver): + with pytest.raises(ValueError, match="cannot be empty"): + await queryweaver.query("testdb", "") + + @pytest.mark.asyncio + async def test_query_whitespace_question_raises(self, queryweaver): + with pytest.raises(ValueError, match="cannot be empty"): + await queryweaver.query("testdb", " ") + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_query_select_all_customers(self, falkordb_url, postgres_url, has_llm_key): + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_query_all") as qw: + conn_result = await qw.connect_database(postgres_url) + try: + assert conn_result.success + + result = await qw.query(conn_result.database_id, "Show me all customers") + + sql_lower = (result.sql_query or "").lower() + assert "select" in sql_lower + assert "customers" in sql_lower + assert len(result.results) == 3 + + names = {r.get("name") for r in result.results} + assert {"Alice Smith", "Bob Jones", "Carol White"} <= names + assert result.ai_response + finally: + await qw.delete_database(conn_result.database_id) + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_query_filter_by_city(self, falkordb_url, postgres_url, has_llm_key): + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_query_filter") as qw: + conn_result = await qw.connect_database(postgres_url) + try: + assert conn_result.success + + result = await qw.query( + conn_result.database_id, "Show me customers from New York", + ) + + sql_lower = (result.sql_query or "").lower() + assert "select" in sql_lower + assert "customers" in sql_lower + assert "new york" in sql_lower or "where" in sql_lower + + assert len(result.results) == 2 + names = {r.get("name") for r in result.results} + assert {"Alice Smith", "Carol White"} == names + assert "Bob Jones" not in names + finally: + await qw.delete_database(conn_result.database_id) + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_query_count_aggregation(self, falkordb_url, postgres_url, has_llm_key): + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_query_count") as qw: + conn_result = await qw.connect_database(postgres_url) + try: + assert conn_result.success + + result = await qw.query( + conn_result.database_id, "How many customers are there?", + ) + + sql_lower = (result.sql_query or "").lower() + assert "select" in sql_lower + assert len(result.results) >= 1 + + first = result.results[0] + count_value = next( + (v for v in first.values() if isinstance(v, int)), None, + ) + if count_value is not None: + assert count_value == 3 + else: + assert len(result.results) == 3 + finally: + await qw.delete_database(conn_result.database_id) + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_query_join_orders(self, falkordb_url, postgres_url, has_llm_key): + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_query_join") as qw: + conn_result = await qw.connect_database(postgres_url) + try: + assert conn_result.success + + result = await qw.query( + conn_result.database_id, "Show me all orders with customer names", + ) + + sql_lower = (result.sql_query or "").lower() + assert "select" in sql_lower + assert "order" in sql_lower + assert len(result.results) == 3 + + first = result.results[0] + assert any( + k.lower() in {"product", "amount", "order_date", "order_id", "id"} + for k in first.keys() + ) + finally: + await qw.delete_database(conn_result.database_id) + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_query_with_history(self, falkordb_url, postgres_url, has_llm_key): + """Chat history threads through via QueryRequest.""" + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_query_history") as qw: + conn_result = await qw.connect_database(postgres_url) + try: + assert conn_result.success + + first = await qw.query(conn_result.database_id, "Show me all customers") + assert first.sql_query + + follow_up = QueryRequest( + question="How many are from New York?", + chat_history=["Show me all customers"], + result_history=[first.ai_response or ""], + ) + second = await qw.query(conn_result.database_id, follow_up) + assert second is not None + assert isinstance(second.results, list) + finally: + await qw.delete_database(conn_result.database_id) + + +class TestDeleteDatabase: + """Database deletion.""" + + @pytest.mark.asyncio + @pytest.mark.requires_postgres + async def test_delete_database(self, falkordb_url, postgres_url, has_llm_key): + async with QueryWeaver(falkordb_url=falkordb_url, user_id="test_delete_user") as qw: + conn_result = await qw.connect_database(postgres_url) + assert conn_result.success + assert conn_result.database_id == "testdb" + + deleted = await qw.delete_database(conn_result.database_id) + assert deleted is True + + databases = await qw.list_databases() + assert conn_result.database_id not in databases + + +@pytest.mark.unit +class TestModels: + """Dataclass serialization / defaults — pure unit, no external services.""" + + def test_query_result_to_dict(self): + result = QueryResult( + sql_query="SELECT * FROM customers", + results=[{"id": 1, "name": "Alice"}], + ai_response="Found 1 customer", + metadata=QueryMetadata( + confidence=0.95, + is_destructive=False, + requires_confirmation=False, + execution_time=0.5, + ), + ) + + d = result.to_dict() + assert d["sql_query"] == "SELECT * FROM customers" + assert d["confidence"] == 0.95 + assert d["results"] == [{"id": 1, "name": "Alice"}] + assert d["ai_response"] == "Found 1 customer" + assert d["is_destructive"] is False + assert d["requires_confirmation"] is False + assert d["execution_time"] == 0.5 + + def test_schema_result_to_dict(self): + result = SchemaResult( + nodes=[{"id": "customers", "name": "customers"}], + links=[{"source": "orders", "target": "customers"}], + ) + + d = result.to_dict() + assert d["nodes"][0]["name"] == "customers" + assert d["links"][0]["source"] == "orders" + assert d["links"][0]["target"] == "customers" + + def test_database_connection_to_dict(self): + result = DatabaseConnection( + database_id="testdb", + success=True, + tables_loaded=5, + message="Connected successfully", + ) + + d = result.to_dict() + assert d["database_id"] == "testdb" + assert d["success"] is True + assert d["tables_loaded"] == 5 + assert d["message"] == "Connected successfully" + + def test_query_result_default_values(self): + result = QueryResult( + sql_query="SELECT 1", + results=[], + ai_response="Test", + metadata=QueryMetadata(confidence=0.8), + ) + + assert result.is_destructive is False + assert result.requires_confirmation is False + assert result.execution_time == 0.0 + assert result.is_valid is True + assert result.missing_information == "" + assert result.ambiguities == "" + assert result.explanation == "" + + def test_database_connection_failure(self): + result = DatabaseConnection( + database_id="", + success=False, + tables_loaded=0, + message="Connection refused", + ) + + d = result.to_dict() + assert d["database_id"] == "" + assert d["success"] is False + assert d["tables_loaded"] == 0 + assert "refused" in d["message"].lower() + + def test_query_request_custom_model_fields(self): + """custom_api_key/custom_model are threaded through to agents.""" + req = QueryRequest( + question="test", + custom_api_key="sk-test-123", + custom_model="openai/gpt-4.1", + ) + assert req.custom_api_key == "sk-test-123" + assert req.custom_model == "openai/gpt-4.1" diff --git a/uv.lock b/uv.lock index b639a971..c2399ad6 100644 --- a/uv.lock +++ b/uv.lock @@ -2155,25 +2155,54 @@ wheels = [ [[package]] name = "queryweaver" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } dependencies = [ + { name = "falkordb" }, + { name = "jsonschema" }, + { name = "litellm" }, + { name = "psycopg2-binary" }, + { name = "pymysql" }, + { name = "tqdm" }, +] + +[package.optional-dependencies] +all = [ + { name = "aiohttp" }, + { name = "authlib" }, + { name = "fastapi" }, + { name = "fastmcp" }, + { name = "graphiti-core" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "playwright" }, + { name = "pylint" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-playwright" }, + { name = "python-dotenv" }, + { name = "python-multipart" }, + { name = "snowflake-connector-python" }, + { name = "uvicorn" }, +] +dev = [ + { name = "playwright" }, + { name = "pylint" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-playwright" }, +] +server = [ { name = "aiohttp" }, { name = "authlib" }, - { name = "falkordb" }, { name = "fastapi" }, { name = "fastmcp" }, { name = "graphiti-core" }, { name = "itsdangerous" }, { name = "jinja2" }, - { name = "jsonschema" }, - { name = "litellm" }, - { name = "psycopg2-binary" }, - { name = "pymysql" }, { name = "python-dotenv" }, { name = "python-multipart" }, { name = "snowflake-connector-python" }, - { name = "tqdm" }, { name = "uvicorn" }, ] @@ -2188,24 +2217,32 @@ dev = [ [package.metadata] requires-dist = [ - { name = "aiohttp", specifier = ">=3.13.5" }, - { name = "authlib", specifier = "~=1.7.0" }, + { name = "aiohttp", marker = "extra == 'server'", specifier = ">=3.13.5" }, + { name = "authlib", marker = "extra == 'server'", specifier = "~=1.7.0" }, { name = "falkordb", specifier = "~=1.6.0" }, - { name = "fastapi", specifier = "~=0.136.0" }, - { name = "fastmcp", specifier = ">=3.2.4" }, - { name = "graphiti-core", specifier = ">=0.28.1" }, - { name = "itsdangerous", specifier = "~=2.2.0" }, - { name = "jinja2", specifier = "~=3.1.4" }, + { name = "fastapi", marker = "extra == 'server'", specifier = "~=0.136.0" }, + { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.2.4" }, + { name = "graphiti-core", marker = "extra == 'server'", specifier = ">=0.28.1" }, + { name = "itsdangerous", marker = "extra == 'server'", specifier = "~=2.2.0" }, + { name = "jinja2", marker = "extra == 'server'", specifier = "~=3.1.4" }, { name = "jsonschema", specifier = "~=4.26.0" }, { name = "litellm", specifier = ">=1.83.0" }, + { name = "playwright", marker = "extra == 'dev'", specifier = "~=1.58.0" }, { name = "psycopg2-binary", specifier = "~=2.9.11" }, + { name = "pylint", marker = "extra == 'dev'", specifier = "~=4.0.3" }, { name = "pymysql", specifier = "~=1.1.0" }, - { name = "python-dotenv", specifier = "~=1.2.2" }, - { name = "python-multipart", specifier = "~=0.0.10" }, - { name = "snowflake-connector-python", specifier = "~=4.4.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = "~=9.0.3" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = "~=1.3.0" }, + { name = "pytest-playwright", marker = "extra == 'dev'", specifier = "~=0.7.1" }, + { name = "python-dotenv", marker = "extra == 'server'", specifier = "~=1.2.2" }, + { name = "python-multipart", marker = "extra == 'server'", specifier = "~=0.0.10" }, + { name = "queryweaver", extras = ["dev"], marker = "extra == 'all'" }, + { name = "queryweaver", extras = ["server"], marker = "extra == 'all'" }, + { name = "snowflake-connector-python", marker = "extra == 'server'", specifier = "~=4.4.0" }, { name = "tqdm", specifier = "~=4.67.3" }, - { name = "uvicorn", specifier = "~=0.44.0" }, + { name = "uvicorn", marker = "extra == 'server'", specifier = "~=0.44.0" }, ] +provides-extras = ["server", "dev", "all"] [package.metadata.requires-dev] dev = [