From c84ed9ac3ddd41b8e875c8d5a80836d22995ae6f Mon Sep 17 00:00:00 2001 From: POWERFULMOVES <142271328+POWERFULMOVES@users.noreply.github.com> Date: Wed, 17 Sep 2025 21:47:58 -0400 Subject: [PATCH] test: tidy hi-rag smoke test import --- pmoves/.github/workflows/ci.yml | 24 ++++- pmoves/README.md | 4 + pmoves/tests/__init__.py | 1 + pmoves/tests/conftest.py | 158 +++++++++++++++++++++++++++++ pmoves/tests/test_hirag_gateway.py | 58 +++++++++++ pmoves/tests/test_langextract.py | 20 ++++ pmoves/tests/test_pmoves_yt.py | 68 +++++++++++++ 7 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 pmoves/tests/__init__.py create mode 100644 pmoves/tests/conftest.py create mode 100644 pmoves/tests/test_hirag_gateway.py create mode 100644 pmoves/tests/test_langextract.py create mode 100644 pmoves/tests/test_pmoves_yt.py diff --git a/pmoves/.github/workflows/ci.yml b/pmoves/.github/workflows/ci.yml index b166f75fa0..fb132a639a 100644 --- a/pmoves/.github/workflows/ci.yml +++ b/pmoves/.github/workflows/ci.yml @@ -1,3 +1,23 @@ name: pmoves-ci -on: {push: {branches: [main, dev]}, pull_request: {}} -jobs: {noop: {runs-on: ubuntu-latest, steps: [{uses: actions/checkout@v4}]}} + +on: + push: + branches: + - main + - dev + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install fastapi==0.114.2 httpx==0.27.2 pytest==8.3.3 requests==2.32.3 lxml==4.9.4 + - name: Run smoke tests + run: pytest pmoves/tests -q diff --git a/pmoves/README.md b/pmoves/README.md index dda6edab6a..76bd6fff4c 100644 --- a/pmoves/README.md +++ b/pmoves/README.md @@ -37,6 +37,10 @@ Notes - Legacy `hi-rag-gateway` remains available. Use `make up-legacy` to start it with `retrieval-eval` targeting the legacy gateway. - Compose snippets for services are already merged in `docker-compose.yml` for ease-of-use. +### Tests + +- Smoke tests stub external dependencies and can run offline: `pytest pmoves/tests`. + Agents Profile - Start: `docker compose --profile agents up -d nats agent-zero archon` - Defaults: both use `NATS_URL=nats://nats:4222`; change via `.env` if external broker is used. diff --git a/pmoves/tests/__init__.py b/pmoves/tests/__init__.py new file mode 100644 index 0000000000..adebc5de48 --- /dev/null +++ b/pmoves/tests/__init__.py @@ -0,0 +1 @@ +# Pytest package marker for shared fixtures. diff --git a/pmoves/tests/conftest.py b/pmoves/tests/conftest.py new file mode 100644 index 0000000000..bc2d9ed0a6 --- /dev/null +++ b/pmoves/tests/conftest.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Callable, Dict + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def _ensure_repo_on_path() -> None: + """Ensure the repository root is importable during tests.""" + root = Path(__file__).resolve().parents[2] + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + + +def _install_module(name: str, module: ModuleType) -> None: + sys.modules.setdefault(name, module) + + +@pytest.fixture(scope="session", autouse=True) +def stub_external_modules() -> None: + """Provide lightweight stand-ins for optional heavy dependencies.""" + # qdrant client + http models + if "qdrant_client" not in sys.modules: + qdrant_module = ModuleType("qdrant_client") + + class _FakeQdrantClient: + def __init__(self, *args, **kwargs) -> None: # pragma: no cover - trivial + self.args = args + self.kwargs = kwargs + + qdrant_module.QdrantClient = _FakeQdrantClient # type: ignore[attr-defined] + _install_module("qdrant_client", qdrant_module) + + models_module = ModuleType("qdrant_client.http.models") + + class _FakeFilter: + def __init__(self, *args, **kwargs) -> None: # pragma: no cover - trivial + self.args = args + self.kwargs = kwargs + + class _FakeFieldCondition(_FakeFilter): + pass + + class _FakeMatchValue(_FakeFilter): + pass + + models_module.Filter = _FakeFilter # type: ignore[attr-defined] + models_module.FieldCondition = _FakeFieldCondition # type: ignore[attr-defined] + models_module.MatchValue = _FakeMatchValue # type: ignore[attr-defined] + http_module = ModuleType("qdrant_client.http") + http_module.models = models_module # type: ignore[attr-defined] + _install_module("qdrant_client.http", http_module) + _install_module("qdrant_client.http.models", models_module) + + # sentence-transformers + if "sentence_transformers" not in sys.modules: + st_module = ModuleType("sentence_transformers") + + class _FakeSentenceTransformer: + def __init__(self, *args, **kwargs) -> None: # pragma: no cover - trivial + self.args = args + self.kwargs = kwargs + + def encode(self, texts, normalize_embeddings: bool = True): # pragma: no cover - trivial + return [[0.0, 0.0, 0.0] for _ in texts] + + st_module.SentenceTransformer = _FakeSentenceTransformer # type: ignore[attr-defined] + _install_module("sentence_transformers", st_module) + + # rapidfuzz + if "rapidfuzz" not in sys.modules: + rapidfuzz_module = ModuleType("rapidfuzz") + rapidfuzz_module.fuzz = SimpleNamespace(token_set_ratio=lambda a, b: 100.0) # type: ignore[attr-defined] + _install_module("rapidfuzz", rapidfuzz_module) + + # neo4j + if "neo4j" not in sys.modules: + neo4j_module = ModuleType("neo4j") + + class _FakeGraphDatabase: + @staticmethod + def driver(*args, **kwargs): # pragma: no cover - trivial + raise RuntimeError("neo4j driver unavailable in tests") + + neo4j_module.GraphDatabase = _FakeGraphDatabase # type: ignore[attr-defined] + _install_module("neo4j", neo4j_module) + + # yt_dlp (overridden per-test with richer behaviour) + if "yt_dlp" not in sys.modules: + yt_module = ModuleType("yt_dlp") + + class _PlaceholderYDL: # pragma: no cover - simple stub + def __init__(self, *args, **kwargs) -> None: + raise RuntimeError("yt_dlp stub used without monkeypatch") + + yt_module.YoutubeDL = _PlaceholderYDL # type: ignore[attr-defined] + _install_module("yt_dlp", yt_module) + + # boto3 client stub; upload_file is patched in tests + if "boto3" not in sys.modules: + boto3_module = ModuleType("boto3") + + class _FakeS3Client: + def upload_file(self, *args, **kwargs) -> None: # pragma: no cover - trivial + return None + + def _fake_client(*args, **kwargs): # pragma: no cover - trivial + return _FakeS3Client() + + boto3_module.client = _fake_client # type: ignore[attr-defined] + _install_module("boto3", boto3_module) + + # nats-py + if "nats" not in sys.modules: + nats_module = ModuleType("nats") + aio_module = ModuleType("nats.aio") + client_module = ModuleType("nats.aio.client") + + class _FakeNATS: + async def connect(self, *args, **kwargs): # pragma: no cover - trivial + return None + + async def publish(self, *args, **kwargs): # pragma: no cover - trivial + return None + + async def close(self): # pragma: no cover - trivial + return None + + client_module.Client = _FakeNATS # type: ignore[attr-defined] + _install_module("nats", nats_module) + _install_module("nats.aio", aio_module) + _install_module("nats.aio.client", client_module) + + +@pytest.fixture(scope="session") +def load_service_module() -> Callable[[str, str], ModuleType]: + """Helper to import service modules by file path once per session.""" + cache: Dict[str, ModuleType] = {} + base = Path(__file__).resolve().parents[1] + + def _load(name: str, relative_path: str) -> ModuleType: + if name in cache: + return cache[name] + module_path = base / relative_path + spec = importlib.util.spec_from_file_location(name, module_path) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load module {name} from {module_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + cache[name] = module + return module + + return _load diff --git a/pmoves/tests/test_hirag_gateway.py b/pmoves/tests/test_hirag_gateway.py new file mode 100644 index 0000000000..87af6bfb97 --- /dev/null +++ b/pmoves/tests/test_hirag_gateway.py @@ -0,0 +1,58 @@ +from fastapi.testclient import TestClient + + +class _FakePoint: + def __init__(self, payload, score): + self.payload = payload + self.score = score + + +def test_hirag_query_returns_stubbed_results(load_service_module, monkeypatch): + gateway = load_service_module("hirag_gateway", "services/hi-rag-gateway/gateway.py") + + fake_results = [ + _FakePoint( + payload={ + "doc_id": "doc-1", + "section_id": "sec-1", + "chunk_id": "chunk-1", + "text": "example chunk", + "namespace": "demo", + }, + score=0.9, + ) + ] + + class _FakeQdrant: + def __init__(self): + self.calls = [] + + def search(self, collection, query_vector, limit, query_filter, with_payload, with_vectors): + self.calls.append( + { + "collection": collection, + "query_vector": query_vector, + "limit": limit, + "query_filter": query_filter, + "with_payload": with_payload, + "with_vectors": with_vectors, + } + ) + return fake_results + + gateway.qdrant = _FakeQdrant() + monkeypatch.setattr(gateway, "embed_query", lambda q: [0.1, 0.2, 0.3]) + monkeypatch.setattr(gateway, "driver", None) + + client = TestClient(gateway.app) + resp = client.post( + "/hirag/query", + json={"query": "hello world", "namespace": "demo", "k": 4, "alpha": 0.5}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["query"] == "hello world" + assert body["results"] and body["results"][0]["doc_id"] == "doc-1" + assert gateway.qdrant.calls, "qdrant.search should be invoked" + assert gateway.qdrant.calls[0]["limit"] == 16 diff --git a/pmoves/tests/test_langextract.py b/pmoves/tests/test_langextract.py new file mode 100644 index 0000000000..0afcdc8903 --- /dev/null +++ b/pmoves/tests/test_langextract.py @@ -0,0 +1,20 @@ +from fastapi.testclient import TestClient + + +def test_extract_text_smoke(load_service_module, monkeypatch): + api = load_service_module("langextract_api", "services/langextract/api.py") + + published = [] + monkeypatch.setattr(api, "_maybe_publish", lambda payload: published.append(payload)) + + client = TestClient(api.app) + resp = client.post( + "/extract/text", + json={"text": "Hello world?\n\nAnother paragraph.", "namespace": "demo", "doc_id": "doc-42"}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["count"] >= 1 + assert body["chunks"][0]["namespace"] == "demo" + assert published and published[0]["count"] == body["count"] diff --git a/pmoves/tests/test_pmoves_yt.py b/pmoves/tests/test_pmoves_yt.py new file mode 100644 index 0000000000..a66dbf89cc --- /dev/null +++ b/pmoves/tests/test_pmoves_yt.py @@ -0,0 +1,68 @@ +from pathlib import Path + +from fastapi.testclient import TestClient + + +def test_yt_download_uses_stubs(load_service_module, monkeypatch): + yt = load_service_module("pmoves_yt", "services/pmoves-yt/yt.py") + + uploads = [] + published = [] + inserts = [] + + def fake_upload(path: str, bucket: str, key: str) -> str: + uploads.append((Path(path).name, bucket, key)) + return f"https://local/{bucket}/{key}" + + def fake_publish(topic: str, payload): + published.append((topic, payload)) + + def fake_insert(table: str, row): + inserts.append((table, row)) + return [{"id": "stub"}] + + monkeypatch.setattr(yt, "upload_to_s3", fake_upload) + monkeypatch.setattr(yt, "_publish_event", fake_publish) + monkeypatch.setattr(yt, "supa_insert", fake_insert) + + class DummyYDL: + def __init__(self, opts): + self.opts = opts + self._filename: str | None = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def extract_info(self, url, download): + outtmpl = self.opts["outtmpl"] + video_path = Path(outtmpl.replace("%(id)s", "abc123").replace("%(ext)s", "mp4")) + video_path.parent.mkdir(parents=True, exist_ok=True) + video_path.write_bytes(b"demo") + self._filename = str(video_path) + return { + "id": "abc123", + "title": "Demo Title", + "requested_downloads": [{"_filename": self._filename}], + } + + def prepare_filename(self, info): + return self._filename or "" + + monkeypatch.setattr(yt.yt_dlp, "YoutubeDL", DummyYDL) + + client = TestClient(yt.app) + resp = client.post( + "/yt/download", + json={"url": "https://youtu.be/example", "bucket": "assets", "namespace": "demo"}, + ) + + assert resp.status_code == 200 + payload = resp.json() + assert payload["ok"] is True + assert payload["video_id"] == "abc123" + assert uploads and uploads[0][1] == "assets" + assert any(topic == "ingest.file.added.v1" for topic, _ in published) + assert {table for table, _ in inserts} == {"studio_board", "videos"}