Add pytest smoke tests for gateway, yt, and langextract services

pmoves/.github/workflows/ci.yml

@@ -1,3 +1,23 @@
 name: pmoves-ci
-on: {push: {branches: [main, dev]}, pull_request: {}}
-jobs: {noop: {runs-on: ubuntu-latest, steps: [{uses: actions/checkout@v4}]}}
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+  pull_request:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install fastapi==0.114.2 httpx==0.27.2 pytest==8.3.3 requests==2.32.3 lxml==4.9.4
+      - name: Run smoke tests
+        run: pytest pmoves/tests -q

pmoves/README.md

@@ -37,6 +37,10 @@ Notes
 - Legacy `hi-rag-gateway` remains available. Use `make up-legacy` to start it with `retrieval-eval` targeting the legacy gateway.
 - Compose snippets for services are already merged in `docker-compose.yml` for ease-of-use.
 
+### Tests
+
+- Smoke tests stub external dependencies and can run offline: `pytest pmoves/tests`.
+
 Agents Profile
 - Start: `docker compose --profile agents up -d nats agent-zero archon`
 - Defaults: both use `NATS_URL=nats://nats:4222`; change via `.env` if external broker is used.

pmoves/tests/__init__.py

@@ -0,0 +1 @@
+# Pytest package marker for shared fixtures.

pmoves/tests/conftest.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+from typing import Callable, Dict
+
+import pytest
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _ensure_repo_on_path() -> None:
+    """Ensure the repository root is importable during tests."""
+    root = Path(__file__).resolve().parents[2]
+    if str(root) not in sys.path:
+        sys.path.insert(0, str(root))
+
+
+def _install_module(name: str, module: ModuleType) -> None:
+    sys.modules.setdefault(name, module)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def stub_external_modules() -> None:
+    """Provide lightweight stand-ins for optional heavy dependencies."""
+    # qdrant client + http models
+    if "qdrant_client" not in sys.modules:
+        qdrant_module = ModuleType("qdrant_client")
+
+        class _FakeQdrantClient:
+            def __init__(self, *args, **kwargs) -> None:  # pragma: no cover - trivial
+                self.args = args
+                self.kwargs = kwargs
+
+        qdrant_module.QdrantClient = _FakeQdrantClient  # type: ignore[attr-defined]
+        _install_module("qdrant_client", qdrant_module)
+
+        models_module = ModuleType("qdrant_client.http.models")
+
+        class _FakeFilter:
+            def __init__(self, *args, **kwargs) -> None:  # pragma: no cover - trivial
+                self.args = args
+                self.kwargs = kwargs
+
+        class _FakeFieldCondition(_FakeFilter):
+            pass
+
+        class _FakeMatchValue(_FakeFilter):
+            pass
+
+        models_module.Filter = _FakeFilter  # type: ignore[attr-defined]
+        models_module.FieldCondition = _FakeFieldCondition  # type: ignore[attr-defined]
+        models_module.MatchValue = _FakeMatchValue  # type: ignore[attr-defined]
+        http_module = ModuleType("qdrant_client.http")
+        http_module.models = models_module  # type: ignore[attr-defined]
+        _install_module("qdrant_client.http", http_module)
+        _install_module("qdrant_client.http.models", models_module)
+
+    # sentence-transformers
+    if "sentence_transformers" not in sys.modules:
+        st_module = ModuleType("sentence_transformers")
+
+        class _FakeSentenceTransformer:
+            def __init__(self, *args, **kwargs) -> None:  # pragma: no cover - trivial
+                self.args = args
+                self.kwargs = kwargs
+
+            def encode(self, texts, normalize_embeddings: bool = True):  # pragma: no cover - trivial
+                return [[0.0, 0.0, 0.0] for _ in texts]
+
+        st_module.SentenceTransformer = _FakeSentenceTransformer  # type: ignore[attr-defined]
+        _install_module("sentence_transformers", st_module)
+
+    # rapidfuzz
+    if "rapidfuzz" not in sys.modules:
+        rapidfuzz_module = ModuleType("rapidfuzz")
+        rapidfuzz_module.fuzz = SimpleNamespace(token_set_ratio=lambda a, b: 100.0)  # type: ignore[attr-defined]
+        _install_module("rapidfuzz", rapidfuzz_module)
+
+    # neo4j
+    if "neo4j" not in sys.modules:
+        neo4j_module = ModuleType("neo4j")
+
+        class _FakeGraphDatabase:
+            @staticmethod
+            def driver(*args, **kwargs):  # pragma: no cover - trivial
+                raise RuntimeError("neo4j driver unavailable in tests")
+
+        neo4j_module.GraphDatabase = _FakeGraphDatabase  # type: ignore[attr-defined]
+        _install_module("neo4j", neo4j_module)
+
+    # yt_dlp (overridden per-test with richer behaviour)
+    if "yt_dlp" not in sys.modules:
+        yt_module = ModuleType("yt_dlp")
+
+        class _PlaceholderYDL:  # pragma: no cover - simple stub
+            def __init__(self, *args, **kwargs) -> None:
+                raise RuntimeError("yt_dlp stub used without monkeypatch")
+
+        yt_module.YoutubeDL = _PlaceholderYDL  # type: ignore[attr-defined]
+        _install_module("yt_dlp", yt_module)
+
+    # boto3 client stub; upload_file is patched in tests
+    if "boto3" not in sys.modules:
+        boto3_module = ModuleType("boto3")
+
+        class _FakeS3Client:
+            def upload_file(self, *args, **kwargs) -> None:  # pragma: no cover - trivial
+                return None
+
+        def _fake_client(*args, **kwargs):  # pragma: no cover - trivial
+            return _FakeS3Client()
+
+        boto3_module.client = _fake_client  # type: ignore[attr-defined]
+        _install_module("boto3", boto3_module)
+
+    # nats-py
+    if "nats" not in sys.modules:
+        nats_module = ModuleType("nats")
+        aio_module = ModuleType("nats.aio")
+        client_module = ModuleType("nats.aio.client")
+
+        class _FakeNATS:
+            async def connect(self, *args, **kwargs):  # pragma: no cover - trivial
+                return None
+
+            async def publish(self, *args, **kwargs):  # pragma: no cover - trivial
+                return None
+
+            async def close(self):  # pragma: no cover - trivial
+                return None
+
+        client_module.Client = _FakeNATS  # type: ignore[attr-defined]
+        _install_module("nats", nats_module)
+        _install_module("nats.aio", aio_module)
+        _install_module("nats.aio.client", client_module)
+
+
+@pytest.fixture(scope="session")
+def load_service_module() -> Callable[[str, str], ModuleType]:
+    """Helper to import service modules by file path once per session."""
+    cache: Dict[str, ModuleType] = {}
+    base = Path(__file__).resolve().parents[1]
+
+    def _load(name: str, relative_path: str) -> ModuleType:
+        if name in cache:
+            return cache[name]
+        module_path = base / relative_path
+        spec = importlib.util.spec_from_file_location(name, module_path)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Cannot load module {name} from {module_path}")
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        cache[name] = module
+        return module
+
+    return _load

pmoves/tests/test_hirag_gateway.py

@@ -0,0 +1,58 @@
+from fastapi.testclient import TestClient
+
+
+class _FakePoint:
+    def __init__(self, payload, score):
+        self.payload = payload
+        self.score = score
+
+
+def test_hirag_query_returns_stubbed_results(load_service_module, monkeypatch):
+    gateway = load_service_module("hirag_gateway", "services/hi-rag-gateway/gateway.py")
+
+    fake_results = [
+        _FakePoint(
+            payload={
+                "doc_id": "doc-1",
+                "section_id": "sec-1",
+                "chunk_id": "chunk-1",
+                "text": "example chunk",
+                "namespace": "demo",
+            },
+            score=0.9,
+        )
+    ]
+
+    class _FakeQdrant:
+        def __init__(self):
+            self.calls = []
+
+        def search(self, collection, query_vector, limit, query_filter, with_payload, with_vectors):
+            self.calls.append(
+                {
+                    "collection": collection,
+                    "query_vector": query_vector,
+                    "limit": limit,
+                    "query_filter": query_filter,
+                    "with_payload": with_payload,
+                    "with_vectors": with_vectors,
+                }
+            )
+            return fake_results
+
+    gateway.qdrant = _FakeQdrant()
+    monkeypatch.setattr(gateway, "embed_query", lambda q: [0.1, 0.2, 0.3])
+    monkeypatch.setattr(gateway, "driver", None)
+
+    client = TestClient(gateway.app)
+    resp = client.post(
+        "/hirag/query",
+        json={"query": "hello world", "namespace": "demo", "k": 4, "alpha": 0.5},
+    )
+
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["query"] == "hello world"
+    assert body["results"] and body["results"][0]["doc_id"] == "doc-1"
+    assert gateway.qdrant.calls, "qdrant.search should be invoked"
+    assert gateway.qdrant.calls[0]["limit"] == 16

pmoves/tests/test_langextract.py

@@ -0,0 +1,20 @@
+from fastapi.testclient import TestClient
+
+
+def test_extract_text_smoke(load_service_module, monkeypatch):
+    api = load_service_module("langextract_api", "services/langextract/api.py")
+
+    published = []
+    monkeypatch.setattr(api, "_maybe_publish", lambda payload: published.append(payload))
+
+    client = TestClient(api.app)
+    resp = client.post(
+        "/extract/text",
+        json={"text": "Hello world?\n\nAnother paragraph.", "namespace": "demo", "doc_id": "doc-42"},
+    )
+
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["count"] >= 1
+    assert body["chunks"][0]["namespace"] == "demo"
+    assert published and published[0]["count"] == body["count"]

pmoves/tests/test_pmoves_yt.py

@@ -0,0 +1,68 @@
+from pathlib import Path
+
+from fastapi.testclient import TestClient
+
+
+def test_yt_download_uses_stubs(load_service_module, monkeypatch):
+    yt = load_service_module("pmoves_yt", "services/pmoves-yt/yt.py")
+
+    uploads = []
+    published = []
+    inserts = []
+
+    def fake_upload(path: str, bucket: str, key: str) -> str:
+        uploads.append((Path(path).name, bucket, key))
+        return f"https://local/{bucket}/{key}"
+
+    def fake_publish(topic: str, payload):
+        published.append((topic, payload))
+
+    def fake_insert(table: str, row):
+        inserts.append((table, row))
+        return [{"id": "stub"}]
+
+    monkeypatch.setattr(yt, "upload_to_s3", fake_upload)
+    monkeypatch.setattr(yt, "_publish_event", fake_publish)
+    monkeypatch.setattr(yt, "supa_insert", fake_insert)
+
+    class DummyYDL:
+        def __init__(self, opts):
+            self.opts = opts
+            self._filename: str | None = None
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc, tb):
+            return False
+
+        def extract_info(self, url, download):
+            outtmpl = self.opts["outtmpl"]
+            video_path = Path(outtmpl.replace("%(id)s", "abc123").replace("%(ext)s", "mp4"))
+            video_path.parent.mkdir(parents=True, exist_ok=True)
+            video_path.write_bytes(b"demo")
+            self._filename = str(video_path)
+            return {
+                "id": "abc123",
+                "title": "Demo Title",
+                "requested_downloads": [{"_filename": self._filename}],
+            }
+
+        def prepare_filename(self, info):
+            return self._filename or ""
+
+    monkeypatch.setattr(yt.yt_dlp, "YoutubeDL", DummyYDL)
+
+    client = TestClient(yt.app)
+    resp = client.post(
+        "/yt/download",
+        json={"url": "https://youtu.be/example", "bucket": "assets", "namespace": "demo"},
+    )
+
+    assert resp.status_code == 200
+    payload = resp.json()
+    assert payload["ok"] is True
+    assert payload["video_id"] == "abc123"
+    assert uploads and uploads[0][1] == "assets"
+    assert any(topic == "ingest.file.added.v1" for topic, _ in published)
+    assert {table for table, _ in inserts} == {"studio_board", "videos"}