diff --git a/.github/workflows/api_tests.yml b/.github/workflows/api_tests.yml index 2ec5342be..74e9b3ab7 100644 --- a/.github/workflows/api_tests.yml +++ b/.github/workflows/api_tests.yml @@ -31,7 +31,7 @@ jobs: pip install pytest httpx pytest-timeout - name: Build and start services - working-directory: tests/api_tests/api_run + working-directory: tests/integration/api/api_run run: | docker compose build docker compose up -d @@ -71,12 +71,12 @@ jobs: OPENRAG_API_URL: http://localhost:8080 AUTH_TOKEN: test-admin-token run: | - cd tests/api_tests + cd tests/integration/api OPENRAG_API_URL=http://localhost:8080 python3 -m pytest . -v --timeout=120 --tb=short - name: Show logs on failure if: failure() - working-directory: tests/api_tests/api_run + working-directory: tests/integration/api/api_run run: | echo "=== OpenRAG Logs ===" docker compose logs openrag --tail=100 @@ -87,6 +87,6 @@ jobs: - name: Cleanup if: always() - working-directory: tests/api_tests/api_run + working-directory: tests/integration/api/api_run run: | - docker compose down -v \ No newline at end of file + docker compose down -v diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac96bf0ba..86525c40a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,6 +59,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: infra/docker/api.Dockerfile platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} @@ -106,7 +107,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - file: Dockerfile.ray + file: infra/docker/ray.Dockerfile platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/build_dev.yml b/.github/workflows/build_dev.yml index f5423fee6..777256288 100644 --- a/.github/workflows/build_dev.yml +++ b/.github/workflows/build_dev.yml @@ -63,6 +63,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: infra/docker/api.Dockerfile platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} @@ -198,7 +199,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - file: Dockerfile.ray + file: infra/docker/ray.Dockerfile platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml index f2e7d299b..f74646965 100644 --- a/.github/workflows/helm.yaml +++ b/.github/workflows/helm.yaml @@ -6,7 +6,7 @@ on: - main - dev paths: - - "charts/**" + - "infra/charts/**" workflow_dispatch: env: @@ -31,7 +31,7 @@ jobs: - name: Set chart version run: | - BASE_VERSION=$(yq '.version' charts/openrag-stack/Chart.yaml) + BASE_VERSION=$(yq '.version' infra/charts/openrag-stack/Chart.yaml) if [[ "${GITHUB_REF_NAME}" == "main" ]]; then # Keep version as-is for stable @@ -39,17 +39,17 @@ jobs: else # Append branch for dev builds CHART_VERSION="${BASE_VERSION}-dev" - yq -i ".version = \"${CHART_VERSION}\"" charts/openrag-stack/Chart.yaml + yq -i ".version = \"${CHART_VERSION}\"" infra/charts/openrag-stack/Chart.yaml fi echo "CHART_VERSION=$CHART_VERSION" >> $GITHUB_ENV echo "Using chart version: $CHART_VERSION" - name: Helm dependency update - run: helm dependency update charts/openrag-stack + run: helm dependency update infra/charts/openrag-stack - name: Package chart - run: helm package charts/openrag-stack --destination . + run: helm package infra/charts/openrag-stack --destination . - name: Login to GHCR run: | diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index e71a7c7dc..b00153c8c 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -46,7 +46,7 @@ jobs: run: uv sync - name: Start Milvus stack - working-directory: tests/integration + working-directory: tests/integration/repos run: docker compose up -d --wait --wait-timeout 180 - name: Verify Milvus reachable from host @@ -65,11 +65,11 @@ jobs: env: OPENRAG_TEST_VDB_HOST: localhost OPENRAG_TEST_VDB_PORT: "19530" - run: uv run pytest tests/integration/ -m integration -v --tb=short + run: uv run pytest tests/integration/repos/ -m integration -v --tb=short - name: Show logs on failure if: failure() - working-directory: tests/integration + working-directory: tests/integration/repos run: | echo "=== Milvus Logs ===" docker compose logs milvus --tail=200 @@ -80,5 +80,5 @@ jobs: - name: Cleanup if: always() - working-directory: tests/integration + working-directory: tests/integration/repos run: docker compose down -v diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 51e86c64a..edb5de096 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -46,4 +46,4 @@ jobs: - name: Run tests run: | - uv run pytest + uv run pytest tests/unit/ diff --git a/.gitignore b/.gitignore index ebab7f115..00b6d2cf1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,8 @@ **/*chainlit.md .venv/ qdrant_storage/ -vdb/volumes +vdb/ +db/ # other files and folders misc @@ -67,7 +68,7 @@ volumes/* *.pkl #helm -charts/openrag-stack/charts/*.tgz +infra/charts/openrag-stack/charts/*.tgz # Planning .planning/ @@ -85,6 +86,12 @@ docs/plans/ # Build artifacts *.egg-info/ +# Test / coverage artifacts +.coverage +.coverage.* +htmlcov/ +.pytest_cache/ + # Local developer overrides (not committed) docker-compose.override.yaml docker-compose.override.yml diff --git a/CLAUDE.md b/CLAUDE.md index b40c41591..59c05b575 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,29 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co OpenRag is a modular Retrieval-Augmented Generation (RAG) framework built with FastAPI, Ray for distributed computing, and Milvus as the vector database. It provides document ingestion, chunking, embedding, and retrieval capabilities with an OpenAI-compatible API. +## Project Layout + +```text +openrag/ # Python package (application code only): core/ services/ api/ di/ prompts/ +conf/ # YAML configuration +infra/ # All deployment infrastructure + docker/ # api.Dockerfile, ray.Dockerfile (build from repo root) + compose/ # docker-compose.yaml + service configs (grafana, prometheus, milvus, .env.example) + scripts/ # entrypoint.sh and other deployment scripts + ansible/ # Ansible playbooks + charts/ # Helm charts (openrag-stack) + quick_start/ # Getting-started compose + cluster.yaml # Ray cluster config +scripts/ # Developer/operational CLI tools (check_layer_imports.py, data_indexer.py, postgres-init/) +tests/ # Integration tests (api_tests/, integration/) +docs/ # Documentation (Astro site + refactoring docs) +ui -> extern/indexer-ui # Symlink to the admin frontend submodule +extern/ # Git submodules + compose service includes +``` + +Prompt templates ship inside the package at `openrag/prompts/templates/*.txt` and are +loaded into `DEFAULT_SEEDS` by `openrag/prompts/__init__.py`. + ## Common Commands ### Development @@ -14,9 +37,11 @@ OpenRag is a modular Retrieval-Augmented Generation (RAG) framework built with F # Install dependencies uv sync -# Run the application locally (requires Docker services) -docker compose up -d # GPU deployment -docker compose --profile cpu up -d # CPU deployment +# Run the application locally (requires Docker services). +# The compose stack and its service configs live under infra/compose/. +cd infra/compose +docker compose up -d # GPU deployment +docker compose --profile cpu up -d # CPU deployment # Run with rebuild for development docker compose up --build -d @@ -25,14 +50,18 @@ docker compose up --build -d ### Testing ```bash -# Run all unit tests -uv run pytest +# Run all unit tests (fast, no infra needed) +uv run pytest tests/unit/ # Run a single test file -uv run pytest openrag/components/indexer/chunker/test_chunking.py +uv run pytest tests/unit/core/models/test_chunk.py # Run tests matching a pattern uv run pytest -k "test_chunk" + +# Integration tests (need running services) / load tests +uv run pytest tests/integration/ +uv run pytest tests/load/ ``` ### Linting @@ -53,19 +82,19 @@ npm run dev # Start dev server at http://localhost:4321/openrag ### Core Components -The main application entry point is `openrag/main.py` which creates a FastAPI app with Ray initialization. +The main application entry point is `openrag/api/main.py` which creates a FastAPI app with Ray initialization. **Ray Actors** (distributed components): -- `Indexer` (`openrag/components/indexer/indexer.py`) - Handles document ingestion, chunking, and insertion into vector DB -- `TaskStateManager` (`openrag/components/indexer/indexer.py`) - Tracks async task states: QUEUED → SERIALIZING → CHUNKING → INSERTING → COMPLETED (or FAILED or CANCELLED) -- `Vectordb` / `MilvusDB` (`openrag/components/indexer/vectordb/vectordb.py`) - Vector database operations with hybrid search (dense + BM25 sparse) -- `DocSerializer` - Serializes files to Document objects using appropriate loaders -- `MarkerPool` / `MarkerWorker` - Pool of workers for PDF processing with Marker +- `Indexer` (`openrag/services/workers/indexer_pool.py`) - Handles document ingestion, chunking, and insertion into vector DB +- `TaskStateManager` (`openrag/services/workers/task_state.py`) - Tracks async task states: QUEUED → SERIALIZING → CHUNKING → INSERTING → COMPLETED (or FAILED or CANCELLED) +- `Vectordb` / `MilvusDB` (`openrag/services/storage/milvus_store.py`) - Vector database operations with hybrid search (dense + BM25 sparse) +- `DocSerializer` (`openrag/services/workers/parsers/doc_serializer.py`) - Serializes files to Document objects using appropriate loaders +- `MarkerPool` / `MarkerWorker` (`openrag/services/workers/parsers/marker_workers.py`) - Pool of workers for PDF processing with Marker **Pipeline Classes**: -- `RagPipeline` (`openrag/components/pipeline.py`) - Orchestrates retrieval and LLM generation -- `RetrieverPipeline` - Handles document retrieval and reranking -- `RAGMapReduce` (`openrag/components/map_reduce.py`) - Map-reduce for processing large document sets +- `RagPipeline` (`openrag/services/orchestrators/query_service.py`) - Orchestrates retrieval and LLM generation +- `RetrieverPipeline` (`openrag/core/retrieval/pipeline.py`) - Handles document retrieval and reranking +- `RAGMapReduce` (`openrag/services/orchestrators/query_service.py`) - Map-reduce for processing large document sets ### Document Processing Flow @@ -75,10 +104,10 @@ The main application entry point is `openrag/main.py` which creates a FastAPI ap 4. Embedder generates vectors via VLLM (OpenAI-compatible API) 5. Chunks inserted into Milvus with partition-based organization -### File Loaders (`openrag/components/indexer/loaders/`) +### File Loaders (`openrag/services/workers/parsers/legacy_loaders/`) Each file type has a dedicated loader that converts to markdown: -- `MarkerLoader` (default for PDF, in `pdf_loaders/`) - Supports OCR, complex layouts, tables +- `MarkerLoader` (default for PDF, in `pdf_loaders/marker.py`) - Supports OCR, complex layouts, tables - `DocxLoader`, `PPTXLoader`, `DocLoader` - Office formats (uses MarkItDown library) - `ImageLoader` - VLM-powered image captioning - `VideoAudioLoader` - Audio transcription via Whisper @@ -102,30 +131,30 @@ Each file type has a dedicated loader that converts to markdown: The RAG pipeline filters out false-positive sources by having the LLM self-report which sources it actually used: -1. `format_context()` (`openrag/components/utils.py`) numbers each source (`[Source 1]`, `[Source 2]`, ...) in the context and returns `(formatted_text, included_indices)` — the indices track which docs fit within the token budget -2. Prompt templates (`prompts/example1/*.txt`) instruct the LLM to append `[Sources: 1, 3, 5]` at the end of its response -3. `extract_and_strip_sources_block()` strips this tag from the response before sending to the client -4. `filter_sources_by_citations()` filters the source metadata to only include cited sources (falls back to all sources if none match) +1. `format_context()` (`openrag/core/prompts/chat_prompt_builder.py`) numbers each source (`[Source 1]`, `[Source 2]`, ...) in the context and returns `(formatted_text, included_indices)` — the indices track which docs fit within the token budget +2. Prompt templates (`openrag/prompts/templates/*.txt`) instruct the LLM to append `[Sources: 1, 3, 5]` at the end of its response +3. `extract_and_strip_sources_block()` (`openrag/core/utils/source_filtering.py`) strips this tag from the response before sending to the client +4. `filter_sources_by_citations()` (`openrag/core/utils/source_filtering.py`) filters the source metadata to only include cited sources (falls back to all sources if none match) 5. For streaming, the OpenAI router buffers the last 100 chars to catch the sources tag before it reaches the client The `extra` field in API responses is a JSON string: `{"sources": [filtered_source_list]}`. -### API Routers (`openrag/routers/`) +### API Routers (`openrag/api/routers/`) -- `openai.py` - OpenAI-compatible `/v1/chat/completions` endpoint -- `indexer.py` - Document ingestion endpoints -- `search.py` - Semantic search endpoints -- `partition.py` - Partition management (multi-tenant document collections) -- `users.py` - User and membership management -- `queue.py` - Task queue monitoring -- `workspaces.py` - Workspace CRUD and file management -- `tools.py` - Tools like `extractText` at `/v1/tools/execute` (tool param requires JSON: `{"name": "extractText"}`) +- `user/chat.py` - OpenAI-compatible `/v1/chat/completions` endpoint +- `admin/indexing.py` - Document ingestion endpoints +- `user/search.py` - Semantic search endpoints +- `admin/partitions.py` - Partition management (multi-tenant document collections) +- `admin/users.py` - User and membership management +- `admin/jobs.py` - Task queue monitoring +- `admin/workspaces.py` - Workspace CRUD and file management +- `admin/tools.py` - Tools like `extractText` at `/v1/tools/execute` (tool param requires JSON: `{"name": "extractText"}`) ### User Management & Authentication The system uses token-based authentication with role-based access control (RBAC) for multi-tenant partition access. -**Database Schema** (PostgreSQL with SQLAlchemy, in `openrag/components/indexer/vectordb/utils.py`): +**Database Schema** (PostgreSQL with SQLAlchemy, in `openrag/services/persistence/schema.py`): - `users` - User accounts with `id`, `external_user_id`, `display_name`, `token` (SHA-256 hashed), `is_admin`, `file_quota`, `file_count` - `files` - File records with `file_id`, `partition_name`, `file_metadata`, `created_by` (FK to users), `relationship_id`, `parent_id` - `partition_memberships` - Join table linking users to partitions with roles (`owner`, `editor`, `viewer`) @@ -133,19 +162,19 @@ The system uses token-based authentication with role-based access control (RBAC) - `workspaces` - Named file subsets within a partition for scoped search/chat - `workspace_files` - Join table linking workspaces to files -**Authentication Flow** (`AuthMiddleware` from `openrag/components/auth/middleware.py`, registered in `openrag/main.py`): +**Authentication Flow** (`AuthMiddleware` from `openrag/api/middleware/auth.py`, registered in `openrag/api/main.py`): 1. Token extracted from `Authorization: Bearer ` header (or `?token=` query param for `/static` routes) 2. Token hashed with SHA-256, looked up in database 3. User info and accessible partitions set on `request.state.user` and `request.state.user_partitions` 4. Bypassed for: `/docs`, `/openapi.json`, `/redoc`, `/health_check`, `/version`, `/chainlit/*` 5. If `AUTH_TOKEN` env var is not set, defaults to admin user (id=1) for all requests -**Role Hierarchy** (`openrag/routers/utils.py`): +**Role Hierarchy** (`openrag/services/orchestrators/auth_service.py`): ```python ROLE_HIERARCHY = {"viewer": 1, "editor": 2, "owner": 3} ``` -**Permission Dependencies** (`openrag/routers/utils.py`): +**Permission Dependencies** (`openrag/api/dependencies/auth.py`): - `require_admin` - User must have `is_admin=True` - `require_partition_viewer` / `require_partition_editor` / `require_partition_owner` - Check partition membership role - `SUPER_ADMIN_MODE=true` env var allows admin users (`is_admin=True`) to bypass partition checks; regular users remain restricted to their partition memberships @@ -168,7 +197,7 @@ ROLE_HIERARCHY = {"viewer": 1, "editor": 2, "owner": 3} | `/partition/{partition}/users/{user_id}` | DELETE | Owner | Remove user | | `/partition/{partition}/users/{user_id}` | PATCH | Owner | Update user role | -**Core Implementation** (`PartitionFileManager` in `openrag/components/indexer/vectordb/utils.py`): +**Core Implementation** (`PartitionFileManager` in `openrag/services/persistence/partition_repo.py`): ```python # User operations (called via MilvusDB Ray actor) await vectordb.create_user.remote(display_name="Name", is_admin=False) @@ -196,7 +225,7 @@ await vectordb.list_partition_members.remote(partition) Optional web search augmentation via the Staan API, allowing the LLM to combine RAG document context with live web results. -**Configuration** (`.hydra_config/config.yaml` → `websearch:` block, env vars): +**Configuration** (`conf/config.yaml` → `websearch:` block, env vars): - `WEBSEARCH_API_TOKEN` — provider API token; if unset, web search is silently disabled - `WEBSEARCH_BASE_URL` — provider endpoint (default: Staan API) - `WEBSEARCH_TOP_K` — number of web results (default: 5) @@ -209,10 +238,10 @@ Optional web search augmentation via the Staan API, allowing the LLM to combine - Source entries include `source_type: "document"` or `source_type: "web"` in the `extra.sources` response **Key files:** -- `openrag/components/websearch/` — `WebSearchService`, `BaseWebSearchProvider`, `StaanProvider` -- `openrag/components/utils.py` — `format_web_context()` formats web results as numbered source blocks -- `openrag/components/pipeline.py` — `_prepare_for_web_only()`, web search logic in `_prepare_for_chat_completion()` -- `openrag/routers/openai.py` — `__prepare_sources()` merges document and web sources +- `openrag/services/websearch/` — `WebSearchService` (`service.py`), `BaseWebSearchProvider` (`base.py`), `StaanProvider` (`providers/staan.py`) +- `openrag/core/prompts/chat_prompt_builder.py` — `format_web_context()` formats web results as numbered source blocks +- `openrag/services/orchestrators/query_service.py` — `_prepare_for_web_only()`, web search logic in `_prepare_for_chat_completion()` +- `openrag/api/routers/user/chat.py` — `__prepare_sources()` merges document and web sources ### File Quota System @@ -223,7 +252,7 @@ Per-user file quota enforcement tracked via the `file_count` and `file_quota` co - `users.file_count` is incremented/decremented in application code (in `PartitionFileManager`) — no SQL triggers - Decrements use `func.greatest(file_count - N, 0)` to prevent negative values from race conditions - `delete_partition` queries per-uploader counts before cascade delete, then bulk decrements -- Quota check (`check_user_file_quota` in `openrag/routers/utils.py`) runs on upload, considering both indexed files and pending tasks +- Quota check (`check_user_file_quota` in `openrag/api/dependencies/auth.py`) runs on upload, considering both indexed files and pending tasks **Quota logic (`file_quota` column):** - `None` → use global default (`DEFAULT_FILE_QUOTA` env var, default `-1`) @@ -236,30 +265,32 @@ Per-user file quota enforcement tracked via the `file_count` and `file_quota` co - `created_by` uses `ondelete="SET NULL"` so deleting a user doesn't cascade-delete their files - `Indexer.delete_file` and `MilvusDB.delete_file/delete_partition` don't need a `user_id` parameter — the uploader is looked up from `files.created_by` -**Migration:** `openrag/scripts/migrations/alembic/versions/c224d4befe71_add_file_count_and_file_quota.py` +**Migration:** `openrag/services/persistence/migrations/alembic/versions/c224d4befe71_add_file_count_and_file_quota.py` ### Alembic Migration Idempotency -`Base.metadata.create_all()` runs at app startup (`PartitionFileManager.__init__` in `openrag/components/indexer/vectordb/utils.py`), so a freshly bootstrapped database already contains the full current-model schema before alembic ever touches it. Migrations must therefore be **idempotent** — re-applying an `ADD COLUMN` / `CREATE TABLE` / `CREATE INDEX` against an already-existing object would raise `DuplicateColumn` / `DuplicateTable`. +`Base.metadata.create_all()` runs at app startup (`PartitionFileManager.__init__` in `openrag/services/persistence/partition_repo.py`), so a freshly bootstrapped database already contains the full current-model schema before alembic ever touches it. Migrations must therefore be **idempotent** — re-applying an `ADD COLUMN` / `CREATE TABLE` / `CREATE INDEX` against an already-existing object would raise `DuplicateColumn` / `DuplicateTable`. Guard every schema-mutating op with an inspector-based existence check (`table_exists`, `column_exists`, `index_exists`, `fk_exists`), in both `upgrade()` and `downgrade()`. For migrations that convert a column type, also short-circuit if the column is already the target type. ### Configuration -Configuration uses Hydra with YAML files in `.hydra_config/`: -- Main config: `.hydra_config/config.yaml` -- Chunker configs: `.hydra_config/chunker/` -- Retriever configs: `.hydra_config/retriever/` -- RAG mode configs: `.hydra_config/rag/` +Configuration is a single YAML file validated with Pydantic models: +- Main config: `conf/config.yaml` +- Loaded by `openrag/core/config/loader.py` (`load_config()` exposed from `openrag/core/config/__init__.py`) +- Pydantic config classes live in `openrag/core/config/` (`root.py`, `auth.py`, `chunking.py`, `retrieval.py`, `indexation.py`, `endpoints.py`, `mcp.py`, `infrastructure.py`, `base.py`) -Environment variables override config values (see `.env.example`). +Environment variables override config values (see `infra/compose/.env.example`). ### Testing Structure -- Unit tests: `openrag/components/**/test_*.py` (pytest) -- API integration tests: `tests/api_tests/*.py` (pytest, requires running server) -- Robot Framework tests: `tests/api/*.robot` -- Test config in `pytest.ini` sets `CONFIG_PATH` and `PROMPTS_DIR` +All tests live in a separate `tests/` tree (zero test files inside the `openrag/` package): +- Unit tests: `tests/unit/**/test_*.py` (pytest, mirrors the package structure; no external services needed) +- Integration tests: `tests/integration/api/*.py` (HTTP endpoint tests, requires running server) and `tests/integration/repos/*.py` (repo/store tests) +- Robot Framework tests: `tests/integration/robot/api/*.robot` +- Load/benchmark tests: `tests/load/` +- Shared fixtures: `tests/unit/conftest.py` (mock ports), `tests/unit/api/conftest.py` (ASGI client), plus per-suite conftests +- Test config lives in `pyproject.toml` (`[tool.pytest.ini_options]`): `testpaths = ["tests"]`, `pythonpath = ["./openrag"]`, and the `env` block sets `PROMPTS_DIR=./openrag/prompts/templates` and `LOG_DIR` **Running integration tests locally with act:** ```bash @@ -288,7 +319,7 @@ await vectordb.async_search.remote(query=query, partition=partition) Use the centralized utility for calling Ray actors with proper timeout and cancellation handling: ```python -from components.ray_utils import call_ray_actor_with_timeout +from services.workers.ray_utils import call_ray_actor_with_timeout result = await call_ray_actor_with_timeout( future=actor.method.remote(args), @@ -304,7 +335,7 @@ This handles: ### Custom Exceptions -All custom exceptions inherit from `OpenRAGError` (`openrag/utils/exceptions/`): +All custom exceptions inherit from `OpenRAGError` (`openrag/core/utils/exceptions.py`): - `VDBError` subclasses for vector database errors - `EmbeddingError` for embedding failures @@ -322,9 +353,9 @@ logger.bind(file_id=file_id, partition=partition).info("Message") Use absolute imports from the `openrag/` directory (which is the Python path root): ```python # Correct - absolute imports -from components.ray_utils import call_ray_actor_with_timeout +from services.workers.ray_utils import call_ray_actor_with_timeout from core.utils.logging import get_logger -from config import load_config +from core.config import load_config # Avoid relative imports across packages # from .ray_utils import ... # Only within same package diff --git a/README.md b/README.md index cbd6608fe..4335720a8 100644 --- a/README.md +++ b/README.md @@ -137,10 +137,10 @@ cd openrag git checkout main # or a given release ``` #### 2. Create a `.env` File -Create a `.env` file at the root of the project, mirroring the structure of `.env.example`, to configure your environment and supply blank environment variables. +Create a `.env` file under `infra/compose/`, mirroring the structure of `infra/compose/.env.example`, to configure your environment and supply blank environment variables. ```bash -cp .env.example .env +cp infra/compose/.env.example infra/compose/.env ``` #### 3. File Parser configuration All supported file format parsers are pre-configured. For PDF processing, **[MarkerLoader](https://github.com/datalab-to/marker)** serves as the default parser, offering comprehensive support for OCR-scanned documents, complex layouts, tables, and embedded images. MarkerLoader operates efficiently on both GPU and CPU environments. @@ -160,10 +160,10 @@ For CPU-only deployments or lightweight testing scenarios, you can consider swit * **Simple and quick** launch for testing >[!IMPORTANT] - > For a **simple `quick deployment`** using only the docker-compose file, only the [quick_start **folder**](./quick_start/) is required. Follow these steps to launch the application: + > For a **simple `quick deployment`** using only the docker-compose file, only the [quick_start **folder**](./infra/quick_start/) is required. Follow these steps to launch the application: - 1. Navigate to the **`quick_start`** directory or download only that folder - 2. Place your **`.env`** file inside the **`quick_start`** directory + 1. Navigate to the **`infra/quick_start`** directory or download only that folder + 2. Place your **`.env`** file inside the **`infra/quick_start`** directory 3. Run the appropriate command for your system: ```bash @@ -177,7 +177,7 @@ For CPU-only deployments or lightweight testing scenarios, you can consider swit ``` * **Development Environment**: For development builds, use the **`--build`** flag to rebuild images: >[!NOTE] - > Execute these commands from the project root directory + > The full stack and its service configs live under `infra/compose/`. Execute these commands from there (`cd infra/compose`). ```bash # GPU deployment with rebuild (recommended for optimal performance) @@ -215,7 +215,7 @@ OpenRag supports two authentication modes: - **Token Mode** (`AUTH_MODE=token`, default): Bearer token authentication via `Authorization: Bearer ` header. Suitable for development and programmatic access. - **OIDC Mode** (`AUTH_MODE=oidc`): OpenID Connect flow with an external identity provider (Keycloak, LemonLDAP::NG, etc.). Users authenticate via browser redirect to the IdP. -To enable OIDC, set `AUTH_MODE=oidc` and configure the required OIDC variables (see [`.env.example`](./.env.example) for the full list). +To enable OIDC, set `AUTH_MODE=oidc` and configure the required OIDC variables (see [`infra/compose/.env.example`](./infra/compose/.env.example) for the full list). For comprehensive OIDC setup and configuration, see the [OIDC Authentication Guide](./docs/oidc.md). diff --git a/conf/config.yaml b/conf/config.yaml index 4f058accb..d30607a2d 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -107,7 +107,9 @@ llm_context: # --- Paths --- # Env: PROMPTS_DIR, DATA_DIR, DB_DIR, LOG_DIR paths: - prompts_dir: ../prompts/example1 + # prompts_dir defaults to the templates bundled in the package + # (openrag/prompts/templates). Override via PROMPTS_DIR to load templates + # from a custom directory. data_dir: ../data db_dir: /app/db log_dir: /app/logs diff --git a/docs/assets/env_example.env b/docs/assets/env_example.env index f5c7cb782..74b42cc50 100644 --- a/docs/assets/env_example.env +++ b/docs/assets/env_example.env @@ -33,8 +33,9 @@ EMBEDDER_MODEL_NAME=jinaai/jina-embeddings-v3 # or other embedder from huggingfa RERANKER_ENABLED=true # deactivate the reranker if your CPU is not powerful enough RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base # or jinaai/jina-reranker-v2-base-multilingual -# Prompts -PROMPTS_DIR=../prompts/example3_en # you can change it to ../prompts/example3 for french prompts +# Prompts (templates ship inside the package at openrag/prompts/templates; +# set PROMPTS_DIR only to override with a custom template directory) +# PROMPTS_DIR=/path/to/custom/templates # Ray RAY_DEDUP_LOGS=0 # turns off ray log deduplication that appear across multiple processes diff --git a/docs/assets/env_linux_gpu.env b/docs/assets/env_linux_gpu.env index 33ff43a92..099114125 100644 --- a/docs/assets/env_linux_gpu.env +++ b/docs/assets/env_linux_gpu.env @@ -29,8 +29,9 @@ EMBEDDER_MODEL_NAME=jinaai/jina-embeddings-v3 # or any other embedder from huggi RERANKER_ENABLED=true RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base # or jinaai/jina-reranker-v2-base-multilingual -# Prompts -PROMPTS_DIR=../prompts/example1 +# Prompts (templates ship inside the package at openrag/prompts/templates; +# set PROMPTS_DIR only to override with a custom template directory) +# PROMPTS_DIR=/path/to/custom/templates # Ray RAY_DEDUP_LOGS=0 # turns off ray log deduplication that appear across multiple processes diff --git a/docs/content/docs/documentation/API.mdx b/docs/content/docs/documentation/API.mdx index dc2407f55..1e9c74607 100644 --- a/docs/content/docs/documentation/API.mdx +++ b/docs/content/docs/documentation/API.mdx @@ -576,7 +576,7 @@ curl -X POST http://localhost:8080/v1/tools/execute \ #### Bulk File Indexing -For indexing multiple files programmatically, you can use this script [`data_indexer.py`](http://github.com/linagora/openrag/blob/main/utility/data_indexer.py) utility script in the [`📁 utility`](https://github.com/linagora/openrag/tree/main/utility) folder or simply use **`indexer ui`**. +For indexing multiple files programmatically, you can use the [`data_indexer.py`](http://github.com/linagora/openrag/blob/main/scripts/data_indexer.py) utility script in the [`scripts/`](https://github.com/linagora/openrag/tree/main/scripts) folder or simply use **`indexer ui`**. #### Example OpenAI Client Usage diff --git a/docs/content/docs/documentation/backup_restore.md b/docs/content/docs/documentation/backup_restore.md index 5f138c4e4..74388a078 100644 --- a/docs/content/docs/documentation/backup_restore.md +++ b/docs/content/docs/documentation/backup_restore.md @@ -15,7 +15,7 @@ docker compose --env-file .env \ --build \ --rm \ -v /my-backup-dir/:/backup:rw \ - --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh ${PARTITION_NAME}" \ + --entrypoint "bash /app/scripts/entrypoint-backup.sh ${PARTITION_NAME}" \ openrag-cpu ``` @@ -28,7 +28,7 @@ docker compose --env-file .env \ --build \ --rm \ -v /my-backup-dir/:/backup:rw \ - --entrypoint "bash /app/openrag/scripts/entrypoint-backup-mt.sh ${PARTITION_NAME}" \ + --entrypoint "bash /app/scripts/entrypoint-backup-mt.sh ${PARTITION_NAME}" \ openrag-cpu ``` ::: @@ -40,7 +40,7 @@ docker compose --env-file .env \ docker compose --env-file .env \ run --build --rm \ -v ~/backup:/backup:rw \ - --entrypoint "uv run /app/openrag/scripts/backup.py -o /backup/test.openrag" \ + --entrypoint "uv run /app/scripts/backup.py -o /backup/test.openrag" \ openrag # Use --include-only to specify the partitions to back up. @@ -57,7 +57,7 @@ docker compose --env-file .env \ --build \ --rm \ -v /my-backup-dir/:/backup:ro \ - --entrypoint "bash /app/openrag/scripts/entrypoint-restore-dry-run.sh backup-file-without-path parition-name" \ + --entrypoint "bash /app/scripts/entrypoint-restore-dry-run.sh backup-file-without-path parition-name" \ openrag-cpu ``` Backup files are expected to be in `/my-backup-dir/`. If the dry run is successful, run the following script to insert the data : @@ -68,7 +68,7 @@ docker compose --env-file .env \ --build \ --rm \ -v /my-backup-dir/:/backup:ro \ - --entrypoint "bash /app/openrag/scripts/entrypoint-restore.sh backup-file-without-path parition-name" \ + --entrypoint "bash /app/scripts/entrypoint-restore.sh backup-file-without-path parition-name" \ openrag-cpu ``` @@ -78,7 +78,7 @@ docker compose --env-file .env \ docker compose --env-file .env \ run --build --rm \ -v ~/backup:/backup:rw \ - --entrypoint "uv run /app/openrag/scripts/restore.py /backup/test.openrag"\ + --entrypoint "uv run /app/scripts/restore.py /backup/test.openrag"\ openrag # Use --include-only to specify the partitions to restore. diff --git a/docs/content/docs/documentation/deploy_ray_cluster.md b/docs/content/docs/documentation/deploy_ray_cluster.md index 52194b617..c5ad01f7e 100644 --- a/docs/content/docs/documentation/deploy_ray_cluster.md +++ b/docs/content/docs/documentation/deploy_ray_cluster.md @@ -84,7 +84,7 @@ We recommend using **GlusterFS** for this. ## 🚀 3. Start the Ray Cluster -First, prepare your `cluster.yaml` file. Here's an example for a **local provider**: +First, prepare your `cluster.yaml` file (a ready-to-edit example ships at `infra/cluster.yaml`). Here's an example for a **local provider**: ```yaml // cluster.yaml @@ -118,7 +118,7 @@ worker_start_ray_commands: - uv run ray start --address ${HEAD_NODE_IP:-10.0.0.1}:6379 ``` -> 🛠️ The base image (`ghcr.io/linagora/openrag-ray`) must be built from `Dockerfile.ray` and pushed to a container registry before use. +> 🛠️ The base image (`ghcr.io/linagora/openrag-ray`) must be built from the repository root with `infra/docker/ray.Dockerfile` and pushed to a container registry before use. ### ⬆️ Launch the cluster @@ -151,4 +151,4 @@ If you encounter errors like `Permission denied` when Ray or Docker tries to acc ```bash sudo chmod -R 777 /ray_mount -``` \ No newline at end of file +``` diff --git a/docs/content/docs/documentation/env_vars.md b/docs/content/docs/documentation/env_vars.md index 113dabdc3..398c89644 100644 --- a/docs/content/docs/documentation/env_vars.md +++ b/docs/content/docs/documentation/env_vars.md @@ -18,7 +18,7 @@ Openrag loads all files into a pivot markdown file format before proceeding to c | Variable | Type | Default | Description | |----------|------|---------|-------------| -| `IMAGE_CAPTIONING` | `bool` | `true` | If `true`, an LLM is used to describe images and convert them into text using a [specific prompt](https://github.com/linagora/openrag/blob/main/prompts/example1/image_captioning_tmpl.txt). The image in files are replaced by their descriptions | +| `IMAGE_CAPTIONING` | `bool` | `true` | If `true`, an LLM is used to describe images and convert them into text using a [specific prompt](https://github.com/linagora/openrag/blob/main/openrag/prompts/templates/image_captioning_tmpl.txt). The image in files are replaced by their descriptions | | `IMAGE_CAPTIONING_URL` | `bool` | `true` | If `true`, HTTP/HTTPS image URLs in markdown files are fetched and described by the VLM. | | `SAVE_MARKDOWN` | `bool` | `false` | If `true`, the pivot-format markdown produced during parsing is saved. Useful for debugging and verifying the correctness of the generated markdown. | |`SAVE_UPLOADED_FILES`|`bool`|`false`| When `true`, uploaded files are stored on disk. You must enable this option if you want Chainlit to show sources while chatting.| @@ -44,7 +44,7 @@ The `MarkerLoader` is the default PDF parsing engine. It can be configured using :::note[Page chunking with `MARKER_CHUNK_SIZE`] Enabling page chunking allows processing large PDFs **significantly faster** by dispatching page ranges to all available workers in parallel rather than sending the entire file to a single worker. The main benefit is the ability to safely scale `MARKER_MAX_PROCESSES` without risking OOM. -It also **reduces per-worker GPU memory spikes** on large files. With a reasonable chunk size (around 10 pages), spikes are shorter and lower, making it safer to run more concurrent workers. See `benchmarks/marker/marker_page_chunking.md` for measured results. +It also **reduces per-worker GPU memory spikes** on large files. With a reasonable chunk size (around 10 pages), spikes are shorter and lower, making it safer to run more concurrent workers. See `tests/load/marker/marker_page_chunking.md` for measured results. **NB:** Consider increasing `MARKER_MAX_TASKS_PER_CHILD` when using page chunking, as worker utilization increases significantly and you may observe frequent subprocess restarts with the default value. ::: @@ -265,7 +265,7 @@ The reranker enhances search quality by re-scoring and reordering retrieved docu ## Extra ### Prompts -The RAG pipeline comes with preconfigured prompts **`./prompts/example1`**. Here are available Prompt Templates in that folder. +The RAG pipeline ships with preconfigured prompts bundled inside the package at **`openrag/prompts/templates`**. Here are the available Prompt Templates in that folder. | Template File | Purpose | |---------------|---------| @@ -278,19 +278,19 @@ The RAG pipeline comes with preconfigured prompts **`./prompts/example1`**. Here | `multi_query_pmpt_tmpl.txt` | Template for generating multiple query variations | To customize prompt: -1. **Duplicate the example folder**: Copy the `example1` folder from `./prompts/` +1. **Copy the bundled templates**: Copy `openrag/prompts/templates` to a folder of your choice 2. **Create your custom folder**: Rename it to something meaningful, e.g., `my_prompt` 3. **Modify the prompts**: Edit any prompt templates within your new folder -4. **Update configuration**: Point to your custom prompts directory +4. **Update configuration**: Point `PROMPTS_DIR` at your custom prompts directory ```bash //.env # Use custom prompts - export PROMPTS_DIR=../prompts/my_prompt + export PROMPTS_DIR=/path/to/my_prompt ``` | Variable | Type | Default | Description | |----------|------|---------|-------------| -| `PROMPTS_DIR` | str | ../prompts/example1 | Path to the directory containing your prompt templates | +| `PROMPTS_DIR` | str | (bundled `openrag/prompts/templates`) | Path to a directory of prompt templates. Unset uses the templates bundled in the package; set it only to override with a custom directory. | ### Logging Our application uses Loguru with custom formatting. Log messages appear in two places: diff --git a/docs/content/docs/documentation/milvus_migration.mdx b/docs/content/docs/documentation/milvus_migration.mdx index cf73e3ce1..9ae27c581 100644 --- a/docs/content/docs/documentation/milvus_migration.mdx +++ b/docs/content/docs/documentation/milvus_migration.mdx @@ -59,10 +59,10 @@ During this step, keep your current version of OpenRAG (< 1.1.7) running. Onl Milvus requires an intermediate upgrade to **v2.5.16** before jumping to 2.6.x. This step must be done manually **before** updating OpenRAG. -Temporarily edit `vdb/milvus.yaml` to set the intermediate Milvus image: +Temporarily edit `infra/compose/milvus/milvus.yaml` to set the intermediate Milvus image: ```diff lang=yaml -// vdb/milvus.yaml +// infra/compose/milvus/milvus.yaml milvus: - image: milvusdb/milvus:v2.5.4 + image: milvusdb/milvus:v2.5.16 @@ -84,7 +84,7 @@ docker inspect milvus-standalone --format '{{ .Config.Image }}' ### Step 2 — Update OpenRAG -Once Milvus 2.5.16 is healthy, stop all services and update OpenRAG to the new version. The updated `vdb/milvus.yaml` already includes Milvus 2.6.11 and the required MinIO and etcd upgrades. +Once Milvus 2.5.16 is healthy, stop all services and update OpenRAG to the new version. The updated `infra/compose/milvus/milvus.yaml` already includes Milvus 2.6.11 and the required MinIO and etcd upgrades. ```bash docker compose down diff --git a/docs/content/docs/documentation/update_embeddings.md b/docs/content/docs/documentation/update_embeddings.md index b539657f7..d141ca586 100644 --- a/docs/content/docs/documentation/update_embeddings.md +++ b/docs/content/docs/documentation/update_embeddings.md @@ -8,7 +8,7 @@ A command-line utility for generating text embeddings using any **OpenAI-compati It supports adaptive batching for optimal performance and handles both plain and `.xz` compressed [backup files](/openrag/documentation/backup_restore/#backup-dump-format). ```bash -python3 openrag/scripts/embed.py \ +python3 scripts/embed.py \ -u http://openai-compatible-endpoint/v1 \ -m Qwen/Qwen3-Embedding-0.6B \ -k sk-... \ @@ -17,7 +17,7 @@ python3 openrag/scripts/embed.py \ -o output-file.openrag(.xz) ## The following version if you're using uv -# uv run python3 openrag/scripts/embed.py \ +# uv run python3 scripts/embed.py \ # -u http://openai-compatible-endpoint/v1 \ # -m Qwen/Qwen3-Embedding-0.6B \ # -k sk-... \ diff --git a/ansible/README.md b/infra/ansible/README.md similarity index 99% rename from ansible/README.md rename to infra/ansible/README.md index 7576b103a..3533ab361 100644 --- a/ansible/README.md +++ b/infra/ansible/README.md @@ -27,7 +27,7 @@ These scripts are designed for installation on fresh production servers. ### 2. Local Deployment (Easiest) ```bash -cd ansible/ +cd infra/ansible/ ./deploy.sh # Choose option 1: "Deploy to local machine" # Select CPU-only or GPU-enabled deployment when prompted diff --git a/ansible/ansible.cfg b/infra/ansible/ansible.cfg similarity index 100% rename from ansible/ansible.cfg rename to infra/ansible/ansible.cfg diff --git a/ansible/deploy.sh b/infra/ansible/deploy.sh similarity index 100% rename from ansible/deploy.sh rename to infra/ansible/deploy.sh diff --git a/ansible/inventory.ini.example b/infra/ansible/inventory.ini.example similarity index 100% rename from ansible/inventory.ini.example rename to infra/ansible/inventory.ini.example diff --git a/ansible/playbooks/docker.yml b/infra/ansible/playbooks/docker.yml similarity index 100% rename from ansible/playbooks/docker.yml rename to infra/ansible/playbooks/docker.yml diff --git a/ansible/playbooks/main-playbook.yml b/infra/ansible/playbooks/main-playbook.yml similarity index 100% rename from ansible/playbooks/main-playbook.yml rename to infra/ansible/playbooks/main-playbook.yml diff --git a/ansible/playbooks/nvidia-drivers-toolkit.yml b/infra/ansible/playbooks/nvidia-drivers-toolkit.yml similarity index 100% rename from ansible/playbooks/nvidia-drivers-toolkit.yml rename to infra/ansible/playbooks/nvidia-drivers-toolkit.yml diff --git a/ansible/playbooks/openrag.yml b/infra/ansible/playbooks/openrag.yml similarity index 100% rename from ansible/playbooks/openrag.yml rename to infra/ansible/playbooks/openrag.yml diff --git a/charts/openrag-stack/.helmignore b/infra/charts/openrag-stack/.helmignore similarity index 100% rename from charts/openrag-stack/.helmignore rename to infra/charts/openrag-stack/.helmignore diff --git a/charts/openrag-stack/Chart.lock b/infra/charts/openrag-stack/Chart.lock similarity index 100% rename from charts/openrag-stack/Chart.lock rename to infra/charts/openrag-stack/Chart.lock diff --git a/charts/openrag-stack/Chart.yaml b/infra/charts/openrag-stack/Chart.yaml similarity index 100% rename from charts/openrag-stack/Chart.yaml rename to infra/charts/openrag-stack/Chart.yaml diff --git a/charts/openrag-stack/templates/_helpers.tpl b/infra/charts/openrag-stack/templates/_helpers.tpl similarity index 100% rename from charts/openrag-stack/templates/_helpers.tpl rename to infra/charts/openrag-stack/templates/_helpers.tpl diff --git a/charts/openrag-stack/templates/configmap-env.yaml b/infra/charts/openrag-stack/templates/configmap-env.yaml similarity index 100% rename from charts/openrag-stack/templates/configmap-env.yaml rename to infra/charts/openrag-stack/templates/configmap-env.yaml diff --git a/charts/openrag-stack/templates/indexer-ui.yaml b/infra/charts/openrag-stack/templates/indexer-ui.yaml similarity index 100% rename from charts/openrag-stack/templates/indexer-ui.yaml rename to infra/charts/openrag-stack/templates/indexer-ui.yaml diff --git a/charts/openrag-stack/templates/infinity.yaml b/infra/charts/openrag-stack/templates/infinity.yaml similarity index 100% rename from charts/openrag-stack/templates/infinity.yaml rename to infra/charts/openrag-stack/templates/infinity.yaml diff --git a/charts/openrag-stack/templates/ingress.yaml b/infra/charts/openrag-stack/templates/ingress.yaml similarity index 100% rename from charts/openrag-stack/templates/ingress.yaml rename to infra/charts/openrag-stack/templates/ingress.yaml diff --git a/charts/openrag-stack/templates/openrag.yaml b/infra/charts/openrag-stack/templates/openrag.yaml similarity index 100% rename from charts/openrag-stack/templates/openrag.yaml rename to infra/charts/openrag-stack/templates/openrag.yaml diff --git a/charts/openrag-stack/templates/pvc.yaml b/infra/charts/openrag-stack/templates/pvc.yaml similarity index 100% rename from charts/openrag-stack/templates/pvc.yaml rename to infra/charts/openrag-stack/templates/pvc.yaml diff --git a/charts/openrag-stack/templates/raycluster.yaml b/infra/charts/openrag-stack/templates/raycluster.yaml similarity index 100% rename from charts/openrag-stack/templates/raycluster.yaml rename to infra/charts/openrag-stack/templates/raycluster.yaml diff --git a/charts/openrag-stack/templates/secrets-env.yaml b/infra/charts/openrag-stack/templates/secrets-env.yaml similarity index 100% rename from charts/openrag-stack/templates/secrets-env.yaml rename to infra/charts/openrag-stack/templates/secrets-env.yaml diff --git a/charts/openrag-stack/values.yaml b/infra/charts/openrag-stack/values.yaml similarity index 98% rename from charts/openrag-stack/values.yaml rename to infra/charts/openrag-stack/values.yaml index 197b9c98a..92b38eebb 100644 --- a/charts/openrag-stack/values.yaml +++ b/infra/charts/openrag-stack/values.yaml @@ -300,8 +300,8 @@ env: RERANKER_BASE_URL: "http://{{ .Release.Name }}-reranker:{{ .Values.reranker.servicePort }}" RERANKER_MODEL_TYPE: "infinity" - # Prompts - PROMPTS_DIR: "../prompts/example1" + # Prompts: templates ship inside the package (openrag/prompts/templates); + # set PROMPTS_DIR only to override with a custom template directory. # Loaders PDFLoader: "MarkerLoader" diff --git a/cluster.yaml b/infra/cluster.yaml similarity index 100% rename from cluster.yaml rename to infra/cluster.yaml diff --git a/.env.example b/infra/compose/.env.example similarity index 97% rename from .env.example rename to infra/compose/.env.example index 71f64d116..0156ef8f8 100644 --- a/.env.example +++ b/infra/compose/.env.example @@ -52,8 +52,9 @@ RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base # or jinaai/jina-reran # API accepts natively. Override to restrict (e.g. to ".wav" only for vLLM deployments). # TRANSCRIBER_DIRECT_UPLOAD_SUFFIXES=.wav|.flac|.ogg|.mp3|.mp4|.m4a|.webm|.mpeg|.mpga -# Prompts -PROMPTS_DIR=../prompts/example1 +# Prompts (templates ship inside the package at openrag/prompts/templates; +# set PROMPTS_DIR only to override with a custom template directory) +# PROMPTS_DIR=/path/to/custom/templates # Ray RAY_DEDUP_LOGS=0 # turns off ray log deduplication that appear across multiple processes diff --git a/.env.ollama b/infra/compose/.env.ollama similarity index 100% rename from .env.ollama rename to infra/compose/.env.ollama diff --git a/docker-compose.yaml b/infra/compose/docker-compose.yaml similarity index 86% rename from docker-compose.yaml rename to infra/compose/docker-compose.yaml index 8c8fb6d45..6890ccff5 100644 --- a/docker-compose.yaml +++ b/infra/compose/docker-compose.yaml @@ -1,14 +1,16 @@ include: - - vdb/milvus.yaml - - ${CHAINLIT_DATALAYER_COMPOSE:-extern/dummy.yaml} - - extern/reranker/${RERANKER_PROVIDER:-infinity}.yaml - - ${TRANSCRIBER_COMPOSE:-extern/dummy.yaml} + - milvus/milvus.yaml + - ${CHAINLIT_DATALAYER_COMPOSE:-../../extern/dummy.yaml} + - ../../extern/reranker/${RERANKER_PROVIDER:-infinity}.yaml + - ${TRANSCRIBER_COMPOSE:-../../extern/dummy.yaml} x-openrag: &openrag_template image: linagoraai/openrag:latest build: - context: . - dockerfile: Dockerfile + # Build context is the repo root so the Dockerfile can COPY pyproject.toml, + # uv.lock, openrag/, conf/ and infra/scripts/entrypoint.sh directly. + context: ../.. + dockerfile: infra/docker/api.Dockerfile extra_hosts: # Let containers reach services running on the Docker host via # ``host.docker.internal`` (Linux Docker >= 20.10). Useful when an OIDC @@ -19,12 +21,12 @@ x-openrag: &openrag_template # shadowed for other users. - "host.docker.internal:host-gateway" volumes: - - ${DATA_VOLUME:-./data}:/app/data - - ./i8n:/app/openrag/.chainlit/translations + - ${DATA_VOLUME:-../../data}:/app/data + - ../../i8n:/app/openrag/.chainlit/translations - ${MODEL_WEIGHTS_VOLUME:-~/.cache/huggingface}:/app/model_weights # Model weights for RAG - - ./openrag:/app/openrag # For dev mode + - ../../openrag:/app/openrag # For dev mode - /$SHARED_ENV:/ray_mount/.env # Shared environment variables - - ./logs:/app/logs # For dev mode + - ../../logs:/app/logs # For dev mode ports: - ${APP_PORT:-8080}:${APP_iPORT:-8080} - ${RAY_DASHBOARD_PORT:-8265}:8265 # Disable when in cluster mode @@ -74,7 +76,7 @@ services: indexer-ui: image: linagoraai/indexer-ui:latest build: - context: ./extern/indexer-ui + context: ../../extern/indexer-ui dockerfile: Dockerfile args: # Mount indexer-ui under a subpath (e.g. /indexerui/) on a shared @@ -135,8 +137,8 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-root_password} - POSTGRES_USER=${POSTGRES_USER:-root} volumes: - - ${DB_VOLUME:-./db}:/var/lib/postgresql/data - - ./scripts/postgres-init:/docker-entrypoint-initdb.d:ro + - ${DB_VOLUME:-../../db}:/var/lib/postgresql/data + - ../../scripts/postgres-init:/docker-entrypoint-initdb.d:ro expose: - 5432 @@ -161,7 +163,7 @@ services: vllm-cpu: <<: *vllm_template build: - context: extern/vllm + context: ../../extern/vllm dockerfile: Dockerfile.cpu target: vllm-openai image: openrag-vllm-openai-cpu diff --git a/openrag_metrics/grafana/dashboards/gpu-metrics.json b/infra/compose/grafana/dashboards/gpu-metrics.json similarity index 100% rename from openrag_metrics/grafana/dashboards/gpu-metrics.json rename to infra/compose/grafana/dashboards/gpu-metrics.json diff --git a/openrag_metrics/grafana/dashboards/openrag-http.json b/infra/compose/grafana/dashboards/openrag-http.json similarity index 100% rename from openrag_metrics/grafana/dashboards/openrag-http.json rename to infra/compose/grafana/dashboards/openrag-http.json diff --git a/openrag_metrics/grafana/dashboards/system-overview.json b/infra/compose/grafana/dashboards/system-overview.json similarity index 100% rename from openrag_metrics/grafana/dashboards/system-overview.json rename to infra/compose/grafana/dashboards/system-overview.json diff --git a/openrag_metrics/grafana/provisioning/dashboards/dashboard.yml b/infra/compose/grafana/provisioning/dashboards/dashboard.yml similarity index 100% rename from openrag_metrics/grafana/provisioning/dashboards/dashboard.yml rename to infra/compose/grafana/provisioning/dashboards/dashboard.yml diff --git a/openrag_metrics/grafana/provisioning/datasources/datasource.yml b/infra/compose/grafana/provisioning/datasources/datasource.yml similarity index 100% rename from openrag_metrics/grafana/provisioning/datasources/datasource.yml rename to infra/compose/grafana/provisioning/datasources/datasource.yml diff --git a/vdb/milvus.yaml b/infra/compose/milvus/milvus.yaml similarity index 100% rename from vdb/milvus.yaml rename to infra/compose/milvus/milvus.yaml diff --git a/openrag_metrics/docker-compose.yaml b/infra/compose/monitoring.docker-compose.yaml similarity index 100% rename from openrag_metrics/docker-compose.yaml rename to infra/compose/monitoring.docker-compose.yaml diff --git a/openrag_metrics/prometheus/prometheus.yml b/infra/compose/prometheus/prometheus.yml similarity index 100% rename from openrag_metrics/prometheus/prometheus.yml rename to infra/compose/prometheus/prometheus.yml diff --git a/Dockerfile b/infra/docker/api.Dockerfile similarity index 83% rename from Dockerfile rename to infra/docker/api.Dockerfile index 01e35d3d4..28d68bcc6 100644 --- a/Dockerfile +++ b/infra/docker/api.Dockerfile @@ -1,3 +1,4 @@ +# Build from repo root: docker build -f infra/docker/api.Dockerfile . FROM python:3.12-slim # Installer curl @@ -33,7 +34,7 @@ RUN pip3 install uv && \ uv python install 3.12.7 && \ uv python pin 3.12.7 # && \ uv sync --no-dev -COPY entrypoint.sh /app/entrypoint.sh +COPY infra/scripts/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Set workdir for source code WORKDIR /app/openrag @@ -41,8 +42,8 @@ WORKDIR /app/openrag # Copy source code COPY openrag/ . -# Copy assets and config -COPY prompts/ /app/prompts/ +# Copy assets and config (prompt templates ship inside the package under openrag/prompts/) +COPY scripts/ /app/scripts/ COPY conf/ /app/conf/ ENV PYTHONPATH=/app/openrag/ ENV APP_iPORT=${APP_iPORT:-8080} diff --git a/Dockerfile.ray b/infra/docker/ray.Dockerfile similarity index 80% rename from Dockerfile.ray rename to infra/docker/ray.Dockerfile index f56ea6370..250a04c68 100644 --- a/Dockerfile.ray +++ b/infra/docker/ray.Dockerfile @@ -1,3 +1,4 @@ +# Build from repo root: docker build -f infra/docker/ray.Dockerfile . FROM python:3.12-slim # Installer curl @@ -37,7 +38,7 @@ RUN pip3 install uv && \ uv python install 3.12.7 && \ uv python pin 3.12.7 # && \uv sync --no-dev -COPY entrypoint.sh /app/entrypoint.sh +COPY infra/scripts/entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh # Set workdir for source code @@ -46,10 +47,10 @@ WORKDIR /app/openrag # Copy source code COPY openrag/ . -# Copy assets and config -COPY prompts/ /app/prompts/ +# Copy assets and config (prompt templates ship inside the package under openrag/prompts/) +COPY scripts/ /app/scripts/ COPY conf/ /app/conf/ RUN ln -s /app/.venv/bin/ray /usr/local/bin/ray -ENV PYTHONPATH=/app/openrag/ \ No newline at end of file +ENV PYTHONPATH=/app/openrag/ diff --git a/quick_start/docker-compose.yaml b/infra/quick_start/docker-compose.yaml similarity index 100% rename from quick_start/docker-compose.yaml rename to infra/quick_start/docker-compose.yaml diff --git a/quick_start/extern/infinity.yaml b/infra/quick_start/extern/infinity.yaml similarity index 100% rename from quick_start/extern/infinity.yaml rename to infra/quick_start/extern/infinity.yaml diff --git a/quick_start/extern/vllm/Dockerfile.cpu b/infra/quick_start/extern/vllm/Dockerfile.cpu similarity index 100% rename from quick_start/extern/vllm/Dockerfile.cpu rename to infra/quick_start/extern/vllm/Dockerfile.cpu diff --git a/quick_start/vdb/milvus.yaml b/infra/quick_start/vdb/milvus.yaml similarity index 100% rename from quick_start/vdb/milvus.yaml rename to infra/quick_start/vdb/milvus.yaml diff --git a/entrypoint.sh b/infra/scripts/entrypoint.sh similarity index 100% rename from entrypoint.sh rename to infra/scripts/entrypoint.sh diff --git a/openrag/api/middleware/auth.py b/openrag/api/middleware/auth.py index 6d5deaa8d..316d4482e 100644 --- a/openrag/api/middleware/auth.py +++ b/openrag/api/middleware/auth.py @@ -23,7 +23,7 @@ Phase 10G migrated the last three legacy importers (``chainlit_api.py``, the ``components/auth`` test module, and the -``tests/api_tests/test_oidc_lifecycle.py`` fixture) onto this module +``tests/integration/api/test_oidc_lifecycle.py`` fixture) onto this module and deleted the ``components/auth/middleware.py`` shim. """ diff --git a/openrag/core/config/infrastructure.py b/openrag/core/config/infrastructure.py index cad5f996c..ee4e9e897 100644 --- a/openrag/core/config/infrastructure.py +++ b/openrag/core/config/infrastructure.py @@ -94,8 +94,13 @@ class RayConfig(ConfigMixin): # --------------------------------------------------------------------------- +# Bundled prompt templates: openrag/prompts/templates/ (this file is at +# openrag/core/config/infrastructure.py, three levels under the package root). +_DEFAULT_PROMPTS_DIR = Path(__file__).resolve().parent.parent.parent / "prompts" / "templates" + + class PathsConfig(ConfigMixin): - prompts_dir: Path = Path("../prompts/example1") + prompts_dir: Path = _DEFAULT_PROMPTS_DIR data_dir: Path = Path("../data") db_dir: Path = Path("/app/db") log_dir: Path = Path("/app/logs") diff --git a/openrag/core/prompts/query_rewriter.py b/openrag/core/prompts/query_rewriter.py index a0b449ad7..d7f6ef9c4 100644 --- a/openrag/core/prompts/query_rewriter.py +++ b/openrag/core/prompts/query_rewriter.py @@ -1,8 +1,8 @@ """Query-rewriting prompt builders for HyDe and Multi-Query retrieval. -Templates live on disk under ``prompts//`` and are loaded via -``template_loader``. These functions are pure: they take a template string + -substitution variables and return the formatted prompt. +Templates ship with the package under ``openrag/prompts/templates/`` and are +loaded via ``template_loader``. These functions are pure: they take a template +string + substitution variables and return the formatted prompt. Template variables expected: HyDe template: ``{question}`` diff --git a/openrag/prompts/__init__.py b/openrag/prompts/__init__.py new file mode 100644 index 000000000..28c897051 --- /dev/null +++ b/openrag/prompts/__init__.py @@ -0,0 +1,26 @@ +"""Bundled default prompt templates. + +Plain-text prompt templates live in ``templates/`` and are read into the +``DEFAULT_SEEDS`` mapping at import time, keyed by filename stem (e.g. +``"sys_prompt_tmpl"`` for ``templates/sys_prompt_tmpl.txt``). + +These are the first-boot defaults. At runtime the application resolves prompts +DB-first — per-partition overrides stored in Postgres take precedence — and +falls back to ``DEFAULT_SEEDS`` when no override exists. +""" + +from __future__ import annotations + +from pathlib import Path + +TEMPLATES_DIR = Path(__file__).resolve().parent / "templates" + + +def _load_seeds() -> dict[str, str]: + """Read every ``templates/*.txt`` file into a stem -> contents mapping.""" + return {path.stem: path.read_text(encoding="utf-8") for path in sorted(TEMPLATES_DIR.glob("*.txt"))} + + +DEFAULT_SEEDS: dict[str, str] = _load_seeds() + +__all__ = ["DEFAULT_SEEDS", "TEMPLATES_DIR"] diff --git a/prompts/example1/chunk_contextualizer_tmpl.txt b/openrag/prompts/templates/chunk_contextualizer_tmpl.txt similarity index 100% rename from prompts/example1/chunk_contextualizer_tmpl.txt rename to openrag/prompts/templates/chunk_contextualizer_tmpl.txt diff --git a/prompts/example1/hyde.txt b/openrag/prompts/templates/hyde.txt similarity index 100% rename from prompts/example1/hyde.txt rename to openrag/prompts/templates/hyde.txt diff --git a/prompts/example1/image_captioning_tmpl.txt b/openrag/prompts/templates/image_captioning_tmpl.txt similarity index 100% rename from prompts/example1/image_captioning_tmpl.txt rename to openrag/prompts/templates/image_captioning_tmpl.txt diff --git a/prompts/example1/multi_query_pmpt_tmpl.txt b/openrag/prompts/templates/multi_query_pmpt_tmpl.txt similarity index 100% rename from prompts/example1/multi_query_pmpt_tmpl.txt rename to openrag/prompts/templates/multi_query_pmpt_tmpl.txt diff --git a/prompts/example1/query_contextualizer_tmpl.txt b/openrag/prompts/templates/query_contextualizer_tmpl.txt similarity index 100% rename from prompts/example1/query_contextualizer_tmpl.txt rename to openrag/prompts/templates/query_contextualizer_tmpl.txt diff --git a/prompts/example1/spoken_style_answer_tmpl.txt b/openrag/prompts/templates/spoken_style_answer_tmpl.txt similarity index 100% rename from prompts/example1/spoken_style_answer_tmpl.txt rename to openrag/prompts/templates/spoken_style_answer_tmpl.txt diff --git a/prompts/example1/sys_prompt_tmpl.txt b/openrag/prompts/templates/sys_prompt_tmpl.txt similarity index 100% rename from prompts/example1/sys_prompt_tmpl.txt rename to openrag/prompts/templates/sys_prompt_tmpl.txt diff --git a/openrag/scripts/entrypoint-backup.sh b/openrag/scripts/entrypoint-backup.sh deleted file mode 100755 index 27eac9907..000000000 --- a/openrag/scripts/entrypoint-backup.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -PARTITION_NAME=$1 -OUTPUT_FILE=/backup/${PARTITION_NAME}.openrag - -uv run /app/openrag/scripts/backup.py --include-only=${PARTITION_NAME} --output ${OUTPUT_FILE} - diff --git a/openrag/scripts/entrypoint-restore-dry-run.sh b/openrag/scripts/entrypoint-restore-dry-run.sh deleted file mode 100644 index 7bb4c126e..000000000 --- a/openrag/scripts/entrypoint-restore-dry-run.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -PARTITION_NAME=$1 -BACKUP_FILE=$2 - -uv run /app/openrag/scripts/restore.py --include-only=${PARTITION_NAME} --batch-size 8192 --dry-run ${BACKUP_FILE} - diff --git a/openrag/scripts/entrypoint-restore.sh b/openrag/scripts/entrypoint-restore.sh deleted file mode 100644 index bf0975bde..000000000 --- a/openrag/scripts/entrypoint-restore.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -PARTITION_NAME=$1 -BACKUP_FILE=$2 - -uv run /app/openrag/scripts/restore.py --include-only=${PARTITION_NAME} --batch-size 8192 ${BACKUP_FILE} - diff --git a/openrag/tests/__init__.py b/openrag/tests/__init__.py deleted file mode 100644 index a26504824..000000000 --- a/openrag/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Integration tests package diff --git a/pyproject.toml b/pyproject.toml index 78488c3ab..aea654cc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,10 @@ build-backend = "setuptools.build_meta" where = ["."] include = ["openrag*"] +# Ship the bundled prompt templates (read at import time by openrag/prompts) +# inside the wheel — they are non-.py data files setuptools would otherwise skip. +[tool.setuptools.package-data] +"openrag.prompts" = ["templates/*.txt"] [tool.ruff] @@ -96,7 +100,9 @@ exclude = [ "logs", ".astro", "openrag.egg-info", - "extern" + "extern", + "infra", + "tests/load", ] [tool.ruff.lint] @@ -120,4 +126,20 @@ ignore = [ known-first-party = ["openrag"] [tool.uv.workspace] -exclude = ["benchmarks/prompt_eval"] +exclude = ["tests/load/prompt_eval"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["./openrag"] +python_files = ["test_*.py"] +addopts = "--strict-markers --tb=short" +asyncio_mode = "auto" +env = [ + "PROMPTS_DIR=./openrag/prompts/templates", + "LOG_DIR=./logs", +] +markers = [ + "integration: Integration tests requiring running OpenRAG server", + "unit: Unit tests that run without external dependencies", + "slow: Slow tests and load/performance benchmarks", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 517ec6d42..000000000 --- a/pytest.ini +++ /dev/null @@ -1,15 +0,0 @@ -[pytest] -testpaths = - openrag -pythonpath = ./openrag -python_files = - test_*.py - -env = - PROMPTS_DIR=./prompts/example1 - LOG_DIR=./logs - -# Markers for test categorization -markers = - integration: Integration tests requiring running OpenRAG server - unit: Unit tests that run without external dependencies diff --git a/scripts/_bootstrap.py b/scripts/_bootstrap.py new file mode 100644 index 000000000..4cc9f16a6 --- /dev/null +++ b/scripts/_bootstrap.py @@ -0,0 +1,20 @@ +"""Import bootstrap for top-level developer scripts. + +The application still uses bare imports such as ``from core...`` until the +future package-wide ``openrag.*`` import migration. Top-level scripts run from +outside the package, so they add the source package directory explicitly. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def ensure_openrag_source_path() -> None: + repo_root = Path(__file__).resolve().parents[1] + package_root = repo_root / "openrag" + for path in (repo_root, package_root): + value = str(path) + if value not in sys.path: + sys.path.insert(0, value) diff --git a/openrag/scripts/backup.py b/scripts/backup.py similarity index 98% rename from openrag/scripts/backup.py rename to scripts/backup.py index 3dcc16c34..f25e5c53d 100644 --- a/openrag/scripts/backup.py +++ b/scripts/backup.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 +# ruff: noqa: E402 import json import os import sys from typing import IO, Any +from _bootstrap import ensure_openrag_source_path + +ensure_openrag_source_path() + from core.utils.logging import get_logger from pymilvus import Collection, connections from services.persistence.schema import files as files_table diff --git a/openrag/scripts/backup.sh.example b/scripts/backup.sh.example similarity index 65% rename from openrag/scripts/backup.sh.example rename to scripts/backup.sh.example index ac6662fa0..87c90c29e 100644 --- a/openrag/scripts/backup.sh.example +++ b/scripts/backup.sh.example @@ -8,6 +8,5 @@ docker compose --env-file .env \ --build \ --rm \ -v ${OUTPUT_DIR}:/backup:rw \ - --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh ${PARTITION_NAME}" \ + --entrypoint "bash /app/scripts/entrypoint-backup.sh ${PARTITION_NAME}" \ openrag-cpu - diff --git a/openrag/scripts/check_file_counts.py b/scripts/check_file_counts.py similarity index 97% rename from openrag/scripts/check_file_counts.py rename to scripts/check_file_counts.py index 55b047c23..57afabfd0 100644 --- a/openrag/scripts/check_file_counts.py +++ b/scripts/check_file_counts.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# ruff: noqa: E402,I001 """Check that each user's materialized file_count matches the actual count in the files table.""" import argparse @@ -7,7 +8,10 @@ from sqlalchemy import create_engine, func, select, update -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from _bootstrap import ensure_openrag_source_path + +ensure_openrag_source_path() + from services.persistence.schema import files, users diff --git a/utility/data_indexer.py b/scripts/data_indexer.py similarity index 97% rename from utility/data_indexer.py rename to scripts/data_indexer.py index e9fdd24d5..ee34fb739 100644 --- a/utility/data_indexer.py +++ b/scripts/data_indexer.py @@ -90,4 +90,4 @@ def __check_file_exists(base_url, partition, file_name, headers): # How to run this code: -# uv run python utility/data_indexer.py -d /path/to/your/documents -p your_partition_name -u http://localhost:8080 -a your_auth_key +# uv run python scripts/data_indexer.py -d /path/to/your/documents -p your_partition_name -u http://localhost:8080 -a your_auth_key diff --git a/openrag/scripts/embed.py b/scripts/embed.py similarity index 100% rename from openrag/scripts/embed.py rename to scripts/embed.py diff --git a/openrag/scripts/entrypoint-backup-mt.sh b/scripts/entrypoint-backup-mt.sh similarity index 59% rename from openrag/scripts/entrypoint-backup-mt.sh rename to scripts/entrypoint-backup-mt.sh index 01dd81344..43362ac02 100755 --- a/openrag/scripts/entrypoint-backup-mt.sh +++ b/scripts/entrypoint-backup-mt.sh @@ -8,5 +8,4 @@ if [ -f ${OUTPUT_FILE} ]; then exit 1 fi -uv run /app/openrag/scripts/backup.py --include-only=${PARTITION_NAME} --output - | xz -9ec -T 0 --memlimit=20% > ${OUTPUT_FILE} - +uv run /app/scripts/backup.py --include-only=${PARTITION_NAME} --output - | xz -9ec -T 0 --memlimit=20% > ${OUTPUT_FILE} diff --git a/scripts/entrypoint-backup.sh b/scripts/entrypoint-backup.sh new file mode 100755 index 000000000..2156c0863 --- /dev/null +++ b/scripts/entrypoint-backup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +PARTITION_NAME=$1 +OUTPUT_FILE=/backup/${PARTITION_NAME}.openrag + +uv run /app/scripts/backup.py --include-only=${PARTITION_NAME} --output ${OUTPUT_FILE} diff --git a/scripts/entrypoint-restore-dry-run.sh b/scripts/entrypoint-restore-dry-run.sh new file mode 100644 index 000000000..88079bf6b --- /dev/null +++ b/scripts/entrypoint-restore-dry-run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +PARTITION_NAME=$1 +BACKUP_FILE=$2 + +uv run /app/scripts/restore.py --include-only=${PARTITION_NAME} --batch-size 8192 --dry-run ${BACKUP_FILE} diff --git a/scripts/entrypoint-restore.sh b/scripts/entrypoint-restore.sh new file mode 100644 index 000000000..18dc94339 --- /dev/null +++ b/scripts/entrypoint-restore.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +PARTITION_NAME=$1 +BACKUP_FILE=$2 + +uv run /app/scripts/restore.py --include-only=${PARTITION_NAME} --batch-size 8192 ${BACKUP_FILE} diff --git a/openrag/scripts/filter-logs.py b/scripts/filter_logs.py similarity index 100% rename from openrag/scripts/filter-logs.py rename to scripts/filter_logs.py diff --git a/openrag/scripts/restore.py b/scripts/restore.py similarity index 99% rename from openrag/scripts/restore.py rename to scripts/restore.py index aa622d725..2130a81ab 100644 --- a/openrag/scripts/restore.py +++ b/scripts/restore.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 +# ruff: noqa: E402 import json import sys import time from typing import IO, Any +from _bootstrap import ensure_openrag_source_path + +ensure_openrag_source_path() + from core.utils.logging import get_logger from pymilvus import MilvusClient from services.persistence.schema import files as files_table diff --git a/openrag/scripts/restore.sh.example b/scripts/restore.sh.example similarity index 63% rename from openrag/scripts/restore.sh.example rename to scripts/restore.sh.example index 872ae5ffe..f9d600bd6 100644 --- a/openrag/scripts/restore.sh.example +++ b/scripts/restore.sh.example @@ -9,6 +9,5 @@ docker compose --env-file .env \ --build \ --rm \ -v ${BACKUP_DIR}:/backup:ro \ - --entrypoint "bash /app/openrag/scripts/entrypoint-restore.sh ${BACKUP_FILE} ${PARTITION_NAME}" \ + --entrypoint "bash /app/scripts/entrypoint-restore.sh ${BACKUP_FILE} ${PARTITION_NAME}" \ openrag-cpu - diff --git a/tests/api_tests/OIDC_TEST_COVERAGE.md b/tests/api_tests/OIDC_TEST_COVERAGE.md deleted file mode 100644 index c84e02389..000000000 --- a/tests/api_tests/OIDC_TEST_COVERAGE.md +++ /dev/null @@ -1,28 +0,0 @@ -# OIDC Test Coverage Matrix - -Maps every acceptance criterion from `.omc/plans/oidc-auth/plan.md` §7 to the -test(s) that cover it. - -| AC | Description | Test file : function | -|----|-------------|----------------------| -| AC1 | `AUTH_MODE=token` behaviour strictly unchanged | `openrag/components/auth/test_middleware.py::TestTokenModeLegacy::test_bearer_valid_returns_200`, `::test_bearer_invalid_returns_403`, `::test_missing_token_returns_403` | -| AC2 | `AUTH_MODE=oidc` — UI path without cookie → 302 to `/auth/login` | `openrag/components/auth/test_middleware.py::TestOIDCMode::test_no_creds_root_path_returns_302`, `::test_no_creds_static_path_returns_302` | -| AC3 | `AUTH_MODE=oidc` — API path without cookie/Bearer → 401 JSON | `openrag/components/auth/test_middleware.py::TestOIDCMode::test_no_creds_api_path_returns_401`, `::test_no_creds_v1_chat_returns_401` | -| AC4 | `GET /auth/login` → 302 with `state`, `nonce`, `code_challenge` (PKCE S256), `scope openid email` | `openrag/routers/test_auth_router.py::test_login_redirects_to_idp_with_pkce`, `tests/api_tests/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 1) | -| AC5 | `GET /auth/callback` with valid code+state → sets `openrag_session` cookie + 302 to `next_url` | `openrag/routers/test_auth_router.py::test_callback_success_by_external_id`, `tests/api_tests/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 3) | -| AC6 | Callback with unknown `sub` → 403, no session created (no email fallback) | `openrag/routers/test_auth_router.py::test_callback_user_not_registered` | -| AC7 | Callback with invalid/mismatched `state` → 400 | `openrag/routers/test_auth_router.py::test_callback_state_mismatch`, `::test_callback_missing_state_cookie` | -| AC8 | Callback with nonce mismatch in ID token → 400 | `openrag/components/auth/test_oidc_client.py` (exchange_code nonce validation tests) | -| AC9 | Request with valid session cookie → `request.state.user` populated, normal flow | `openrag/components/auth/test_middleware.py::TestOIDCMode::test_cookie_valid_and_access_token_fresh_no_refresh`, `tests/api_tests/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 5) | -| AC10 | Cookie with near-expiry access_token + refresh_token → transparent refresh, new tokens in DB | `openrag/components/auth/test_middleware.py::TestOIDCMode::test_cookie_near_expiry_triggers_refresh`, `::TestRefreshHelper::test_no_refresh_when_token_fresh` | -| AC11 | Cookie with expired access_token, no refresh_token → session revoked, 302 to `/auth/login` | `openrag/components/auth/test_middleware.py::TestOIDCMode::test_cookie_refresh_fails_session_revoked_and_302`, `::TestRefreshHelper::test_expired_no_refresh_token_returns_none` | -| AC12 | `POST /auth/backchannel-logout` with valid signed `logout_token` (sid match) → 200, sessions revoked | `openrag/routers/test_auth_router.py::test_backchannel_logout_revokes_by_sid`, `tests/api_tests/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 6) | -| AC13 | `POST /auth/backchannel-logout` with invalid signature → 400, no revocation | `openrag/routers/test_auth_router.py::test_backchannel_logout_rejects_invalid_token` | -| AC14 | After backchannel-logout, old cookie rejected (401/302) | `tests/api_tests/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 7) | -| AC15 | Bearer `users.token` accepted in `AUTH_MODE=oidc` for programmatic access | `openrag/components/auth/test_middleware.py::TestOIDCMode::test_bearer_fallback_accepted_in_oidc_mode` | -| AC16 | `access_token` and `refresh_token` stored encrypted (Fernet), unreadable without key | `openrag/components/auth/test_session_tokens.py::TestEncryptDecrypt::test_round_trip`, `::test_wrong_key_raises_value_error` | -| AC17 | Alembic migration upgrade/downgrade idempotent | Manual: `alembic upgrade head && alembic downgrade -1 && alembic upgrade head` | -| AC18 | `OIDC_TOKEN_ENCRYPTION_KEY` missing in oidc mode → clear startup error | `openrag/components/auth/test_oidc_client.py` (startup/config validation tests) | -| AC19 | `OIDC_CLAIM_MAPPING` unset → callback does NOT update user row | `openrag/routers/test_auth_router.py::test_callback_skips_mapping_when_unset` | -| AC20 | `OIDC_CLAIM_MAPPING` set + `OIDC_CLAIM_SOURCE=id_token` → user row updated from ID token claims | `openrag/routers/test_auth_router.py::test_callback_applies_claim_mapping_from_id_token` | -| AC21 | `OIDC_CLAIM_MAPPING` set + `OIDC_CLAIM_SOURCE=userinfo` → `/userinfo` fetched, user row updated | `openrag/routers/test_auth_router.py::test_callback_applies_claim_mapping_from_userinfo` | diff --git a/tests/api_tests/__init__.py b/tests/api_tests/__init__.py deleted file mode 100644 index ffe0808cd..000000000 --- a/tests/api_tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# API Tests Package diff --git a/openrag/conftest.py b/tests/conftest.py similarity index 52% rename from openrag/conftest.py rename to tests/conftest.py index 19f368e97..49ce4236b 100644 --- a/openrag/conftest.py +++ b/tests/conftest.py @@ -1,19 +1,22 @@ """Pytest import guards for top-level imports. During collection, pytest can prepend nested test directories such as -``openrag/routers`` to ``sys.path``, where local modules could otherwise shadow +``tests/unit/api/routers`` to ``sys.path``, where local modules could otherwise shadow third-party imports such as ``openai``. """ from __future__ import annotations -import importlib +import importlib.util import sys from pathlib import Path -OPENRAG_ROOT = Path(__file__).resolve().parent +OPENRAG_ROOT = Path(__file__).resolve().parents[1] / "openrag" root = str(OPENRAG_ROOT) if root not in sys.path: sys.path.insert(0, root) -sys.modules.setdefault("openai", importlib.import_module("openai")) +if importlib.util.find_spec("openai") is not None: + import openai + + sys.modules.setdefault("openai", openai) diff --git a/tests/integration/api/OIDC_TEST_COVERAGE.md b/tests/integration/api/OIDC_TEST_COVERAGE.md new file mode 100644 index 000000000..308077bc0 --- /dev/null +++ b/tests/integration/api/OIDC_TEST_COVERAGE.md @@ -0,0 +1,28 @@ +# OIDC Test Coverage Matrix + +Maps every acceptance criterion from `.omc/plans/oidc-auth/plan.md` §7 to the +test(s) that cover it. + +| AC | Description | Test file : function | +|----|-------------|----------------------| +| AC1 | `AUTH_MODE=token` behaviour strictly unchanged | `tests/unit/api/middleware/test_auth.py::TestTokenModeLegacy::test_bearer_valid_returns_200`, `::test_bearer_invalid_returns_403`, `::test_missing_token_returns_403` | +| AC2 | `AUTH_MODE=oidc` — UI path without cookie → 302 to `/auth/login` | `tests/unit/api/middleware/test_auth.py::TestOIDCMode::test_no_creds_root_path_returns_302`, `::test_no_creds_static_path_returns_302` | +| AC3 | `AUTH_MODE=oidc` — API path without cookie/Bearer → 401 JSON | `tests/unit/api/middleware/test_auth.py::TestOIDCMode::test_no_creds_api_path_returns_401`, `::test_no_creds_v1_chat_returns_401` | +| AC4 | `GET /auth/login` → 302 with `state`, `nonce`, `code_challenge` (PKCE S256), `scope openid email` | `tests/unit/test_auth_router.py::test_login_redirects_to_idp_with_pkce`, `tests/integration/api/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 1) | +| AC5 | `GET /auth/callback` with valid code+state → sets `openrag_session` cookie + 302 to `next_url` | `tests/unit/test_auth_router.py::test_callback_success_by_external_id`, `tests/integration/api/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 3) | +| AC6 | Callback with unknown `sub` → 403, no session created (no email fallback) | `tests/unit/test_auth_router.py::test_callback_user_not_registered` | +| AC7 | Callback with invalid/mismatched `state` → 400 | `tests/unit/test_auth_router.py::test_callback_state_mismatch`, `::test_callback_missing_state_cookie` | +| AC8 | Callback with nonce mismatch in ID token → 400 | `openrag/components/auth/test_oidc_client.py` (exchange_code nonce validation tests) | +| AC9 | Request with valid session cookie → `request.state.user` populated, normal flow | `tests/unit/api/middleware/test_auth.py::TestOIDCMode::test_cookie_valid_and_access_token_fresh_no_refresh`, `tests/integration/api/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 5) | +| AC10 | Cookie with near-expiry access_token + refresh_token → transparent refresh, new tokens in DB | `tests/unit/api/middleware/test_auth.py::TestOIDCMode::test_cookie_near_expiry_triggers_refresh`, `::TestRefreshHelper::test_no_refresh_when_token_fresh` | +| AC11 | Cookie with expired access_token, no refresh_token → session revoked, 302 to `/auth/login` | `tests/unit/api/middleware/test_auth.py::TestOIDCMode::test_cookie_refresh_fails_session_revoked_and_302`, `::TestRefreshHelper::test_expired_no_refresh_token_returns_none` | +| AC12 | `POST /auth/backchannel-logout` with valid signed `logout_token` (sid match) → 200, sessions revoked | `tests/unit/test_auth_router.py::test_backchannel_logout_revokes_by_sid`, `tests/integration/api/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 6) | +| AC13 | `POST /auth/backchannel-logout` with invalid signature → 400, no revocation | `tests/unit/test_auth_router.py::test_backchannel_logout_rejects_invalid_token` | +| AC14 | After backchannel-logout, old cookie rejected (401/302) | `tests/integration/api/test_oidc_lifecycle.py::test_full_oidc_lifecycle` (step 7) | +| AC15 | Bearer `users.token` accepted in `AUTH_MODE=oidc` for programmatic access | `tests/unit/api/middleware/test_auth.py::TestOIDCMode::test_bearer_fallback_accepted_in_oidc_mode` | +| AC16 | `access_token` and `refresh_token` stored encrypted (Fernet), unreadable without key | `openrag/components/auth/test_session_tokens.py::TestEncryptDecrypt::test_round_trip`, `::test_wrong_key_raises_value_error` | +| AC17 | Alembic migration upgrade/downgrade idempotent | Manual: `alembic upgrade head && alembic downgrade -1 && alembic upgrade head` | +| AC18 | `OIDC_TOKEN_ENCRYPTION_KEY` missing in oidc mode → clear startup error | `openrag/components/auth/test_oidc_client.py` (startup/config validation tests) | +| AC19 | `OIDC_CLAIM_MAPPING` unset → callback does NOT update user row | `tests/unit/test_auth_router.py::test_callback_skips_mapping_when_unset` | +| AC20 | `OIDC_CLAIM_MAPPING` set + `OIDC_CLAIM_SOURCE=id_token` → user row updated from ID token claims | `tests/unit/test_auth_router.py::test_callback_applies_claim_mapping_from_id_token` | +| AC21 | `OIDC_CLAIM_MAPPING` set + `OIDC_CLAIM_SOURCE=userinfo` → `/userinfo` fetched, user row updated | `tests/unit/test_auth_router.py::test_callback_applies_claim_mapping_from_userinfo` | diff --git a/tests/api_tests/api_run/Dockerfile.mock-vllm b/tests/integration/api/api_run/Dockerfile.mock-vllm similarity index 100% rename from tests/api_tests/api_run/Dockerfile.mock-vllm rename to tests/integration/api/api_run/Dockerfile.mock-vllm diff --git a/tests/api_tests/api_run/docker-compose.yaml b/tests/integration/api/api_run/docker-compose.yaml similarity index 96% rename from tests/api_tests/api_run/docker-compose.yaml rename to tests/integration/api/api_run/docker-compose.yaml index f3085533a..86974b069 100644 --- a/tests/api_tests/api_run/docker-compose.yaml +++ b/tests/integration/api/api_run/docker-compose.yaml @@ -75,8 +75,8 @@ services: openrag: build: - context: ../../.. - dockerfile: Dockerfile + context: ../../../.. + dockerfile: infra/docker/api.Dockerfile environment: - EMBEDDER_MODEL_NAME=mock-embedding-model - EMBEDDER_BASE_URL=http://vllm:8000/v1/ @@ -100,7 +100,6 @@ services: - RAY_POOL_SIZE=1 - RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 - RAY_memory_monitor_refresh_ms=0 - - PROMPTS_DIR=../prompts/example1 - DEFAULT_FILE_QUOTA=10 - AUTH_TOKEN=${AUTH_TOKEN:-test-admin-token} - SUPER_ADMIN_MODE=${SUPER_ADMIN_MODE:-true} @@ -123,4 +122,4 @@ services: start_period: 60s volumes: - test_data: \ No newline at end of file + test_data: diff --git a/tests/api_tests/api_run/fixture/sample.txt b/tests/integration/api/api_run/fixture/sample.txt similarity index 100% rename from tests/api_tests/api_run/fixture/sample.txt rename to tests/integration/api/api_run/fixture/sample.txt diff --git a/tests/api_tests/api_run/mock_vllm.py b/tests/integration/api/api_run/mock_vllm.py similarity index 100% rename from tests/api_tests/api_run/mock_vllm.py rename to tests/integration/api/api_run/mock_vllm.py diff --git a/tests/api_tests/api_run/scripts/run_api_tests_local.sh b/tests/integration/api/api_run/scripts/run_api_tests_local.sh similarity index 100% rename from tests/api_tests/api_run/scripts/run_api_tests_local.sh rename to tests/integration/api/api_run/scripts/run_api_tests_local.sh diff --git a/tests/api_tests/conftest.py b/tests/integration/api/conftest.py similarity index 100% rename from tests/api_tests/conftest.py rename to tests/integration/api/conftest.py diff --git a/tests/api_tests/email_test_file.json b/tests/integration/api/email_test_file.json similarity index 100% rename from tests/api_tests/email_test_file.json rename to tests/integration/api/email_test_file.json diff --git a/tests/api_tests/test_actors.py b/tests/integration/api/test_actors.py similarity index 100% rename from tests/api_tests/test_actors.py rename to tests/integration/api/test_actors.py diff --git a/tests/api_tests/test_extract.py b/tests/integration/api/test_extract.py similarity index 100% rename from tests/api_tests/test_extract.py rename to tests/integration/api/test_extract.py diff --git a/tests/api_tests/test_health.py b/tests/integration/api/test_health.py similarity index 100% rename from tests/api_tests/test_health.py rename to tests/integration/api/test_health.py diff --git a/tests/api_tests/test_indexer.py b/tests/integration/api/test_indexer.py similarity index 99% rename from tests/api_tests/test_indexer.py rename to tests/integration/api/test_indexer.py index 1cc86f855..05bb8f815 100644 --- a/tests/api_tests/test_indexer.py +++ b/tests/integration/api/test_indexer.py @@ -5,13 +5,12 @@ from pathlib import Path import pytest - -from .conftest import TASK_TIMEOUT, wait_for_task +from conftest import TASK_TIMEOUT, wait_for_task # Check if image captioning is enabled (disabled in CI) IMAGE_CAPTIONING_ENABLED = os.environ.get("IMAGE_CAPTIONING", "").lower() not in ("false", "0", "") -RESOURCES_DIR = Path(__file__).parent.parent / "resources" +RESOURCES_DIR = Path(__file__).parents[2] / "resources" PDF_FILE = RESOURCES_DIR / "test_file.pdf" diff --git a/tests/api_tests/test_oidc_lifecycle.py b/tests/integration/api/test_oidc_lifecycle.py similarity index 99% rename from tests/api_tests/test_oidc_lifecycle.py rename to tests/integration/api/test_oidc_lifecycle.py index e426a2014..b5cf31a75 100644 --- a/tests/api_tests/test_oidc_lifecycle.py +++ b/tests/integration/api/test_oidc_lifecycle.py @@ -286,11 +286,11 @@ def _logger(): # resolve AuthService/UserService from the DI providers, so we import the # provider symbols here (di.providers is not popped, so these are the same # function objects the routers close over) to key dependency_overrides. -from components.auth import OIDCClient # noqa: E402 -from components.auth.session_tokens import hash_session_token # noqa: E402 from core.config.auth import OIDCConfig # noqa: E402 from core.models.user import OIDCSession, User # noqa: E402 from di.providers import get_auth_service, get_user_service # noqa: E402 +from services.auth import OIDCClient # noqa: E402 +from services.auth.session_tokens import hash_session_token # noqa: E402 from services.orchestrators.auth_service import AuthService # noqa: E402 from services.orchestrators.user_service import UserService # noqa: E402 diff --git a/tests/api_tests/test_openai_compat.py b/tests/integration/api/test_openai_compat.py similarity index 99% rename from tests/api_tests/test_openai_compat.py rename to tests/integration/api/test_openai_compat.py index b0446da3a..f2ade7e2d 100644 --- a/tests/api_tests/test_openai_compat.py +++ b/tests/integration/api/test_openai_compat.py @@ -4,10 +4,9 @@ import time import uuid +import conftest import pytest - -from . import conftest -from .conftest import wait_for_indexing +from conftest import wait_for_indexing class TestOpenAICompatibleAPI: diff --git a/tests/api_tests/test_partition.py b/tests/integration/api/test_partition.py similarity index 100% rename from tests/api_tests/test_partition.py rename to tests/integration/api/test_partition.py diff --git a/tests/api_tests/test_queue.py b/tests/integration/api/test_queue.py similarity index 100% rename from tests/api_tests/test_queue.py rename to tests/integration/api/test_queue.py diff --git a/tests/api_tests/test_search.py b/tests/integration/api/test_search.py similarity index 99% rename from tests/api_tests/test_search.py rename to tests/integration/api/test_search.py index 57af79439..1cf090290 100644 --- a/tests/api_tests/test_search.py +++ b/tests/integration/api/test_search.py @@ -5,10 +5,9 @@ import uuid from datetime import datetime +import conftest import pytest -from . import conftest - class TestSemanticSearch: """Test semantic search functionality.""" diff --git a/tests/api_tests/test_tools.py b/tests/integration/api/test_tools.py similarity index 99% rename from tests/api_tests/test_tools.py rename to tests/integration/api/test_tools.py index fea4f51f2..17e0158b5 100644 --- a/tests/api_tests/test_tools.py +++ b/tests/integration/api/test_tools.py @@ -4,7 +4,7 @@ import pytest -RESOURCES_DIR = Path(__file__).parent.parent / "resources" +RESOURCES_DIR = Path(__file__).parents[2] / "resources" PDF_FILE = RESOURCES_DIR / "test_file.pdf" TEXT_FILE = RESOURCES_DIR / "test_file.txt" diff --git a/tests/api_tests/test_users.py b/tests/integration/api/test_users.py similarity index 100% rename from tests/api_tests/test_users.py rename to tests/integration/api/test_users.py diff --git a/tests/api_tests/test_workspaces.py b/tests/integration/api/test_workspaces.py similarity index 99% rename from tests/api_tests/test_workspaces.py rename to tests/integration/api/test_workspaces.py index 96cd63e69..e90246576 100644 --- a/tests/api_tests/test_workspaces.py +++ b/tests/integration/api/test_workspaces.py @@ -4,8 +4,7 @@ import uuid import pytest - -from .conftest import wait_for_indexing +from conftest import wait_for_indexing pytestmark = pytest.mark.integration diff --git a/tests/integration/__init__.py b/tests/integration/repos/__init__.py similarity index 100% rename from tests/integration/__init__.py rename to tests/integration/repos/__init__.py diff --git a/tests/integration/conftest.py b/tests/integration/repos/conftest.py similarity index 93% rename from tests/integration/conftest.py rename to tests/integration/repos/conftest.py index b90d0e57d..a0e9dc155 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/repos/conftest.py @@ -9,10 +9,10 @@ between tests so each one starts clean without paying for a full drop/migrate cycle per case. -The integration tests live under ``tests/integration/`` rather than +The repository integration tests live under ``tests/integration/repos/`` rather than alongside the repository code because they need a real database — pytest's -``testpaths`` keeps them out of the default unit run; invoke explicitly with -``uv run pytest tests/integration``. +``testpaths`` discovers them with the rest of the top-level test tree; invoke explicitly with +``uv run pytest tests/integration/repos``. """ from __future__ import annotations @@ -25,10 +25,10 @@ import pytest import pytest_asyncio -# tests/integration is outside the pytest.ini `pythonpath = ./openrag` +# tests/integration/repos is outside the pytest.ini `pythonpath = ./openrag` # search path, so we have to add it ourselves before importing project # modules. Mirrors how scripts/migrations/env.py wires its path. -_OPENRAG = Path(__file__).resolve().parents[2] / "openrag" +_OPENRAG = Path(__file__).resolve().parents[3] / "openrag" if str(_OPENRAG) not in sys.path: sys.path.insert(0, str(_OPENRAG)) diff --git a/tests/integration/docker-compose.yaml b/tests/integration/repos/docker-compose.yaml similarity index 100% rename from tests/integration/docker-compose.yaml rename to tests/integration/repos/docker-compose.yaml diff --git a/tests/integration/test_document_repo.py b/tests/integration/repos/test_document_repo.py similarity index 100% rename from tests/integration/test_document_repo.py rename to tests/integration/repos/test_document_repo.py diff --git a/tests/integration/test_milvus_store_integration.py b/tests/integration/repos/test_milvus_store_integration.py similarity index 100% rename from tests/integration/test_milvus_store_integration.py rename to tests/integration/repos/test_milvus_store_integration.py diff --git a/tests/integration/test_oidc_session_repo.py b/tests/integration/repos/test_oidc_session_repo.py similarity index 100% rename from tests/integration/test_oidc_session_repo.py rename to tests/integration/repos/test_oidc_session_repo.py diff --git a/tests/integration/test_partition_membership_repo.py b/tests/integration/repos/test_partition_membership_repo.py similarity index 100% rename from tests/integration/test_partition_membership_repo.py rename to tests/integration/repos/test_partition_membership_repo.py diff --git a/tests/integration/test_partition_repo.py b/tests/integration/repos/test_partition_repo.py similarity index 100% rename from tests/integration/test_partition_repo.py rename to tests/integration/repos/test_partition_repo.py diff --git a/tests/integration/test_postgres_store.py b/tests/integration/repos/test_postgres_store.py similarity index 100% rename from tests/integration/test_postgres_store.py rename to tests/integration/repos/test_postgres_store.py diff --git a/openrag/tests/test_relationships_integration.py b/tests/integration/repos/test_relationships.py similarity index 100% rename from openrag/tests/test_relationships_integration.py rename to tests/integration/repos/test_relationships.py diff --git a/tests/integration/test_stores.py b/tests/integration/repos/test_stores.py similarity index 100% rename from tests/integration/test_stores.py rename to tests/integration/repos/test_stores.py diff --git a/tests/integration/test_user_repo.py b/tests/integration/repos/test_user_repo.py similarity index 100% rename from tests/integration/test_user_repo.py rename to tests/integration/repos/test_user_repo.py diff --git a/tests/integration/test_workspace_repo.py b/tests/integration/repos/test_workspace_repo.py similarity index 100% rename from tests/integration/test_workspace_repo.py rename to tests/integration/repos/test_workspace_repo.py diff --git a/tests/api/__init__.robot b/tests/integration/robot/api/__init__.robot similarity index 100% rename from tests/api/__init__.robot rename to tests/integration/robot/api/__init__.robot diff --git a/tests/api/actors.robot b/tests/integration/robot/api/actors.robot similarity index 100% rename from tests/api/actors.robot rename to tests/integration/robot/api/actors.robot diff --git a/tests/api/indexer.robot b/tests/integration/robot/api/indexer.robot similarity index 100% rename from tests/api/indexer.robot rename to tests/integration/robot/api/indexer.robot diff --git a/tests/api/keywords.robot b/tests/integration/robot/api/keywords.robot similarity index 100% rename from tests/api/keywords.robot rename to tests/integration/robot/api/keywords.robot diff --git a/tests/api/main.robot b/tests/integration/robot/api/main.robot similarity index 100% rename from tests/api/main.robot rename to tests/integration/robot/api/main.robot diff --git a/tests/api/openai.robot b/tests/integration/robot/api/openai.robot similarity index 100% rename from tests/api/openai.robot rename to tests/integration/robot/api/openai.robot diff --git a/tests/api/partition.robot b/tests/integration/robot/api/partition.robot similarity index 100% rename from tests/api/partition.robot rename to tests/integration/robot/api/partition.robot diff --git a/tests/api/queue.robot b/tests/integration/robot/api/queue.robot similarity index 100% rename from tests/api/queue.robot rename to tests/integration/robot/api/queue.robot diff --git a/tests/api/search.robot b/tests/integration/robot/api/search.robot similarity index 100% rename from tests/api/search.robot rename to tests/integration/robot/api/search.robot diff --git a/tests/api/users.robot b/tests/integration/robot/api/users.robot similarity index 100% rename from tests/api/users.robot rename to tests/integration/robot/api/users.robot diff --git a/automatic-evaluation-pipeline/README.md b/tests/load/automatic-evaluation-pipeline/README.md similarity index 100% rename from automatic-evaluation-pipeline/README.md rename to tests/load/automatic-evaluation-pipeline/README.md diff --git a/automatic-evaluation-pipeline/assets/Hit_rate.png b/tests/load/automatic-evaluation-pipeline/assets/Hit_rate.png similarity index 100% rename from automatic-evaluation-pipeline/assets/Hit_rate.png rename to tests/load/automatic-evaluation-pipeline/assets/Hit_rate.png diff --git a/automatic-evaluation-pipeline/assets/Recall.png b/tests/load/automatic-evaluation-pipeline/assets/Recall.png similarity index 100% rename from automatic-evaluation-pipeline/assets/Recall.png rename to tests/load/automatic-evaluation-pipeline/assets/Recall.png diff --git a/automatic-evaluation-pipeline/assets/documents_ranking.png b/tests/load/automatic-evaluation-pipeline/assets/documents_ranking.png similarity index 100% rename from automatic-evaluation-pipeline/assets/documents_ranking.png rename to tests/load/automatic-evaluation-pipeline/assets/documents_ranking.png diff --git a/automatic-evaluation-pipeline/assets/image-3.png b/tests/load/automatic-evaluation-pipeline/assets/image-3.png similarity index 100% rename from automatic-evaluation-pipeline/assets/image-3.png rename to tests/load/automatic-evaluation-pipeline/assets/image-3.png diff --git a/automatic-evaluation-pipeline/assets/image.png b/tests/load/automatic-evaluation-pipeline/assets/image.png similarity index 100% rename from automatic-evaluation-pipeline/assets/image.png rename to tests/load/automatic-evaluation-pipeline/assets/image.png diff --git a/automatic-evaluation-pipeline/assets/nDCG.png b/tests/load/automatic-evaluation-pipeline/assets/nDCG.png similarity index 100% rename from automatic-evaluation-pipeline/assets/nDCG.png rename to tests/load/automatic-evaluation-pipeline/assets/nDCG.png diff --git a/automatic-evaluation-pipeline/assets/nDCG_formula.webp b/tests/load/automatic-evaluation-pipeline/assets/nDCG_formula.webp similarity index 100% rename from automatic-evaluation-pipeline/assets/nDCG_formula.webp rename to tests/load/automatic-evaluation-pipeline/assets/nDCG_formula.webp diff --git a/automatic-evaluation-pipeline/benchmark.py b/tests/load/automatic-evaluation-pipeline/benchmark.py similarity index 100% rename from automatic-evaluation-pipeline/benchmark.py rename to tests/load/automatic-evaluation-pipeline/benchmark.py diff --git a/automatic-evaluation-pipeline/generate_questions.py b/tests/load/automatic-evaluation-pipeline/generate_questions.py similarity index 100% rename from automatic-evaluation-pipeline/generate_questions.py rename to tests/load/automatic-evaluation-pipeline/generate_questions.py diff --git a/automatic-evaluation-pipeline/upload_files.py b/tests/load/automatic-evaluation-pipeline/upload_files.py similarity index 100% rename from automatic-evaluation-pipeline/upload_files.py rename to tests/load/automatic-evaluation-pipeline/upload_files.py diff --git a/benchmarks/marker/marker_page_chunking.md b/tests/load/marker/marker_page_chunking.md similarity index 100% rename from benchmarks/marker/marker_page_chunking.md rename to tests/load/marker/marker_page_chunking.md diff --git a/benchmarks/prompt_eval/.env.example b/tests/load/prompt_eval/.env.example similarity index 100% rename from benchmarks/prompt_eval/.env.example rename to tests/load/prompt_eval/.env.example diff --git a/benchmarks/prompt_eval/Report_query_decomposition.md b/tests/load/prompt_eval/Report_query_decomposition.md similarity index 100% rename from benchmarks/prompt_eval/Report_query_decomposition.md rename to tests/load/prompt_eval/Report_query_decomposition.md diff --git a/benchmarks/prompt_eval/Report_temporal_filter.md b/tests/load/prompt_eval/Report_temporal_filter.md similarity index 100% rename from benchmarks/prompt_eval/Report_temporal_filter.md rename to tests/load/prompt_eval/Report_temporal_filter.md diff --git a/benchmarks/prompt_eval/datasets/query_decomposition.json b/tests/load/prompt_eval/datasets/query_decomposition.json similarity index 100% rename from benchmarks/prompt_eval/datasets/query_decomposition.json rename to tests/load/prompt_eval/datasets/query_decomposition.json diff --git a/benchmarks/prompt_eval/datasets/temporal_filter.json b/tests/load/prompt_eval/datasets/temporal_filter.json similarity index 100% rename from benchmarks/prompt_eval/datasets/temporal_filter.json rename to tests/load/prompt_eval/datasets/temporal_filter.json diff --git a/benchmarks/prompt_eval/eval_query_decomposition.py b/tests/load/prompt_eval/eval_query_decomposition.py similarity index 100% rename from benchmarks/prompt_eval/eval_query_decomposition.py rename to tests/load/prompt_eval/eval_query_decomposition.py diff --git a/benchmarks/prompt_eval/eval_temporal_filter_generation.py b/tests/load/prompt_eval/eval_temporal_filter_generation.py similarity index 100% rename from benchmarks/prompt_eval/eval_temporal_filter_generation.py rename to tests/load/prompt_eval/eval_temporal_filter_generation.py diff --git a/benchmarks/prompt_eval/prompts/query_contextualizer_tmpl_v0.txt b/tests/load/prompt_eval/prompts/query_contextualizer_tmpl_v0.txt similarity index 100% rename from benchmarks/prompt_eval/prompts/query_contextualizer_tmpl_v0.txt rename to tests/load/prompt_eval/prompts/query_contextualizer_tmpl_v0.txt diff --git a/benchmarks/prompt_eval/prompts/query_contextualizer_tmpl_v1.txt b/tests/load/prompt_eval/prompts/query_contextualizer_tmpl_v1.txt similarity index 100% rename from benchmarks/prompt_eval/prompts/query_contextualizer_tmpl_v1.txt rename to tests/load/prompt_eval/prompts/query_contextualizer_tmpl_v1.txt diff --git a/benchmarks/prompt_eval/pyproject.toml b/tests/load/prompt_eval/pyproject.toml similarity index 100% rename from benchmarks/prompt_eval/pyproject.toml rename to tests/load/prompt_eval/pyproject.toml diff --git a/benchmarks/prompt_eval/results/result_filter_generation.json b/tests/load/prompt_eval/results/result_filter_generation.json similarity index 100% rename from benchmarks/prompt_eval/results/result_filter_generation.json rename to tests/load/prompt_eval/results/result_filter_generation.json diff --git a/benchmarks/prompt_eval/results/result_query_decomposition.json b/tests/load/prompt_eval/results/result_query_decomposition.json similarity index 100% rename from benchmarks/prompt_eval/results/result_query_decomposition.json rename to tests/load/prompt_eval/results/result_query_decomposition.json diff --git a/benchmarks/prompt_eval/uv.lock b/tests/load/prompt_eval/uv.lock similarity index 100% rename from benchmarks/prompt_eval/uv.lock rename to tests/load/prompt_eval/uv.lock diff --git a/benchmarks/workspace/README.md b/tests/load/workspace/README.md similarity index 98% rename from benchmarks/workspace/README.md rename to tests/load/workspace/README.md index 0ca9ca707..545650d3f 100644 --- a/benchmarks/workspace/README.md +++ b/tests/load/workspace/README.md @@ -5,7 +5,7 @@ Performance benchmarks for OpenRAG. Each benchmark is a standalone script that s ## Quick Start ```bash -cd benchmarks +cd tests/load/workspace docker compose up -d # Milvus standalone + PostgreSQL pip install -r requirements.txt ``` diff --git a/benchmarks/workspace/docker-compose.yml b/tests/load/workspace/docker-compose.yml similarity index 100% rename from benchmarks/workspace/docker-compose.yml rename to tests/load/workspace/docker-compose.yml diff --git a/benchmarks/workspace/requirements.txt b/tests/load/workspace/requirements.txt similarity index 100% rename from benchmarks/workspace/requirements.txt rename to tests/load/workspace/requirements.txt diff --git a/benchmarks/workspace/results_workspace.md b/tests/load/workspace/results_workspace.md similarity index 100% rename from benchmarks/workspace/results_workspace.md rename to tests/load/workspace/results_workspace.md diff --git a/benchmarks/workspace/workspace.py b/tests/load/workspace/workspace.py similarity index 91% rename from benchmarks/workspace/workspace.py rename to tests/load/workspace/workspace.py index 7dc2d3eb5..e47e113f6 100644 --- a/benchmarks/workspace/workspace.py +++ b/tests/load/workspace/workspace.py @@ -6,25 +6,25 @@ B) PostgreSQL resolution → file_id in [...] with batching for >1000 files Usage: - cd benchmarks && docker compose up -d + cd tests/load/workspace && docker compose up -d pip install -r requirements.txt python workspace.py # print to stdout python workspace.py -o results_workspace.md # also write to file """ import argparse -import time import statistics +import time from concurrent.futures import ThreadPoolExecutor, as_completed import numpy as np from pymilvus import ( + AnnSearchRequest, Collection, DataType, Function, FunctionType, MilvusClient, - AnnSearchRequest, RRFRanker, connections, ) @@ -247,11 +247,7 @@ def setup_postgres(engine): ) """) ) - conn.execute( - text( - "CREATE INDEX ix_partition_file ON files(partition_name, file_id)" - ) - ) + conn.execute(text("CREATE INDEX ix_partition_file ON files(partition_name, file_id)")) conn.execute( text(""" CREATE TABLE file_workspaces ( @@ -263,11 +259,7 @@ def setup_postgres(engine): ) """) ) - conn.execute( - text( - "CREATE INDEX ix_ws_partition ON file_workspaces(workspace_id, partition_name)" - ) - ) + conn.execute(text("CREATE INDEX ix_ws_partition ON file_workspaces(workspace_id, partition_name)")) print(" PostgreSQL tables created.") @@ -328,9 +320,7 @@ def insert_data( # files table for fid in all_file_ids: conn.execute( - text( - "INSERT INTO files (file_id, partition_name) VALUES (:fid, :p)" - ), + text("INSERT INTO files (file_id, partition_name) VALUES (:fid, :p)"), {"fid": fid, "p": PARTITION_NAME}, ) @@ -338,10 +328,7 @@ def insert_data( for ws_id, file_ids in assignments.items(): for fid in file_ids: conn.execute( - text( - "INSERT INTO file_workspaces (file_id, workspace_id, partition_name) " - "VALUES (:fid, :ws, :p)" - ), + text("INSERT INTO file_workspaces (file_id, workspace_id, partition_name) VALUES (:fid, :ws, :p)"), {"fid": fid, "ws": ws_id, "p": PARTITION_NAME}, ) print(" PostgreSQL inserts complete.") @@ -411,9 +398,7 @@ def search_approach_a( ) ) else: # hybrid - t, _ = timed( - lambda: _hybrid_search(col, query_vector, query_text, filter_expr) - ) + t, _ = timed(lambda: _hybrid_search(col, query_vector, query_text, filter_expr)) return t @@ -427,10 +412,7 @@ def pg_resolve_workspace(engine, workspace_id: str) -> tuple[float, list[str]]: def _query(): with engine.connect() as conn: rows = conn.execute( - text( - "SELECT file_id FROM file_workspaces " - "WHERE workspace_id = :ws AND partition_name = :p" - ), + text("SELECT file_id FROM file_workspaces WHERE workspace_id = :ws AND partition_name = :p"), {"ws": workspace_id, "p": PARTITION_NAME}, ).fetchall() return [r[0] for r in rows] @@ -467,9 +449,7 @@ def search_approach_b_single( ) ) else: # hybrid - t, _ = timed( - lambda: _hybrid_search(col, query_vector, query_text, filter_expr) - ) + t, _ = timed(lambda: _hybrid_search(col, query_vector, query_text, filter_expr)) return t @@ -487,9 +467,7 @@ def search_approach_b_batched( Approach B step 2+3 (batched, >batch_size files): multiple Milvus searches + merge. Returns (search_time, merge_time) in seconds. """ - batches = [ - file_ids[i : i + batch_size] for i in range(0, len(file_ids), batch_size) - ] + batches = [file_ids[i : i + batch_size] for i in range(0, len(file_ids), batch_size)] # Step 2: search batches in parallel using threads all_results = [] @@ -652,9 +630,7 @@ def _write(): ws_ids = [w for w in row["workspace_ids"] if w != new_ws] cleanup.append({"_id": row["_id"], "workspace_ids": ws_ids}) if cleanup: - client.upsert( - collection_name=COLLECTION, data=cleanup, partial_update=True - ) + client.upsert(collection_name=COLLECTION, data=cleanup, partial_update=True) return times @@ -686,9 +662,7 @@ def _write(): # Cleanup with engine.begin() as conn: conn.execute( - text( - "DELETE FROM file_workspaces WHERE file_id = :fid AND workspace_id = :ws" - ), + text("DELETE FROM file_workspaces WHERE file_id = :fid AND workspace_id = :ws"), {"fid": target_file, "ws": new_ws}, ) @@ -719,9 +693,7 @@ def main(output_file: str | None = None): if count == expected_entities: # Also check PG with engine.connect() as conn: - pg_count = conn.execute( - text("SELECT count(*) FROM file_workspaces") - ).scalar() + pg_count = conn.execute(text("SELECT count(*) FROM file_workspaces")).scalar() if pg_count and pg_count > 0: data_ready = True print(f"\n[2/6] Skipping setup — data already present ({count:,} chunks, {pg_count:,} PG rows).") @@ -811,13 +783,16 @@ def main(output_file: str | None = None): times_b_pg.append(t_pg) if len(file_ids) <= effective_batch: - t_search = search_approach_b_single( - client, col, file_ids, qv, qt, search_type - ) + t_search = search_approach_b_single(client, col, file_ids, qv, qt, search_type) t_merge = 0.0 else: t_search, t_merge = search_approach_b_batched( - client, col, file_ids, qv, qt, search_type, + client, + col, + file_ids, + qv, + qt, + search_type, batch_size=effective_batch, ) @@ -838,11 +813,7 @@ def main(output_file: str | None = None): f" merge={sb_merge['median']}ms)" ) - batches = ( - max(1, (num_files + effective_batch - 1) // effective_batch) - if num_files > effective_batch - else 0 - ) + batches = max(1, (num_files + effective_batch - 1) // effective_batch) if num_files > effective_batch else 0 results_table.append( { "Scenario": scenario_name, @@ -911,9 +882,7 @@ def main(output_file: str | None = None): f"A={a:7.2f}ms B={b:7.2f}ms -> {winner} wins by {diff:.2f}ms ({ratio:.1f}x)" ) summary_lines.append( - f"\n Write: A(full)={wa_full['median']:.2f}ms" - f" A(partial)={wa_partial['median']:.2f}ms" - f" B={wb['median']:.2f}ms" + f"\n Write: A(full)={wa_full['median']:.2f}ms A(partial)={wa_partial['median']:.2f}ms B={wb['median']:.2f}ms" ) # Print to stdout @@ -936,18 +905,24 @@ def main(output_file: str | None = None): with open(output_file, "w") as f: f.write("# Workspace Search Benchmark Results\n\n") f.write("## Approaches\n\n") - f.write("- **Approach A — Milvus ARRAY_CONTAINS**: Each chunk stores a " - "`workspace_ids` ARRAY field. Search filters with " - "`ARRAY_CONTAINS(workspace_ids, \"ws\")` using an INVERTED index. " - "Writes require upserting every chunk to append a workspace ID.\n") - f.write("- **Approach B — PostgreSQL resolution + file_id IN**: Workspace " - "membership is stored in PostgreSQL. At search time, file IDs are " - "resolved via a SQL query, then passed to Milvus as " - "`file_id in [...]`. For >1,000 files, searches are batched in " - "parallel and results merged. Writes are a single SQL INSERT.\n\n") + f.write( + "- **Approach A — Milvus ARRAY_CONTAINS**: Each chunk stores a " + "`workspace_ids` ARRAY field. Search filters with " + '`ARRAY_CONTAINS(workspace_ids, "ws")` using an INVERTED index. ' + "Writes require upserting every chunk to append a workspace ID.\n" + ) + f.write( + "- **Approach B — PostgreSQL resolution + file_id IN**: Workspace " + "membership is stored in PostgreSQL. At search time, file IDs are " + "resolved via a SQL query, then passed to Milvus as " + "`file_id in [...]`. For >1,000 files, searches are batched in " + "parallel and results merged. Writes are a single SQL INSERT.\n\n" + ) f.write("## Setup\n\n") - f.write(f"- **Data**: {FILES:,} files x {CHUNKS_PER_FILE} chunks = " - f"{FILES * CHUNKS_PER_FILE:,} chunks in Milvus\n") + f.write( + f"- **Data**: {FILES:,} files x {CHUNKS_PER_FILE} chunks = " + f"{FILES * CHUNKS_PER_FILE:,} chunks in Milvus\n" + ) f.write(f"- **Vectors**: {VECTOR_DIM}-dim float32 ({VECTOR_DIM * 4:,} bytes each)\n") f.write(f"- **Search params**: top_k={TOP_K}, ef={SEARCH_EF}\n") f.write(f"- **Runs**: {WARMUP_RUNS} warmup + {MEASURED_RUNS} measured\n\n") diff --git a/tests/smoke_test_data/README.md b/tests/smoke_test_data/README.md index 8ca0b694a..701cbea67 100644 --- a/tests/smoke_test_data/README.md +++ b/tests/smoke_test_data/README.md @@ -39,7 +39,7 @@ Indexes 500 SimplWiki documents and waits for completion. **What it does:** 1. Checks API health at `/health_check` -2. Indexes 500 documents to partition `simplewiki-500` using `utility/data_indexer.py` +2. Indexes 500 documents to partition `simplewiki-500` using `scripts/data_indexer.py` 3. Waits for all indexing tasks to complete via `/queue/info` **Prerequisites:** diff --git a/tests/smoke_test_data/run_backup_restore_test.sh b/tests/smoke_test_data/run_backup_restore_test.sh index 02c07ac05..c062e102f 100755 --- a/tests/smoke_test_data/run_backup_restore_test.sh +++ b/tests/smoke_test_data/run_backup_restore_test.sh @@ -39,7 +39,7 @@ mkdir -p "${BACKUP_DIR}" echo "Step 1: Creating backup of partition '${PARTITION}'..." docker compose --env-file "${ENV_FILE}" run --rm \ -v "${BACKUP_DIR}:/backup:rw" \ - --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh ${PARTITION}" \ + --entrypoint "bash /app/scripts/entrypoint-backup.sh ${PARTITION}" \ openrag-cpu BACKUP_FILE="${BACKUP_DIR}/${PARTITION}.openrag" @@ -63,7 +63,7 @@ echo "" echo "Step 3: Restoring backup to new partition '${PARTITION_RESTORED}'..." docker compose --env-file "${ENV_FILE}" run --rm \ -v "${BACKUP_DIR}:/backup:ro" \ - --entrypoint "bash /app/openrag/scripts/entrypoint-restore.sh ${PARTITION_RESTORED} /backup/modified.openrag" \ + --entrypoint "bash /app/scripts/entrypoint-restore.sh ${PARTITION_RESTORED} /backup/modified.openrag" \ openrag-cpu echo "Restore completed" @@ -73,7 +73,7 @@ echo "" echo "Step 4: Creating backup of restored partition '${PARTITION_RESTORED}'..." docker compose --env-file "${ENV_FILE}" run --rm \ -v "${BACKUP_DIR}:/backup:rw" \ - --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh ${PARTITION_RESTORED}" \ + --entrypoint "bash /app/scripts/entrypoint-backup.sh ${PARTITION_RESTORED}" \ openrag-cpu RESTORED_BACKUP="${BACKUP_DIR}/${PARTITION_RESTORED}.openrag" diff --git a/tests/smoke_test_data/run_smoke_test.sh b/tests/smoke_test_data/run_smoke_test.sh index 04fcbbb0d..c5114f44e 100755 --- a/tests/smoke_test_data/run_smoke_test.sh +++ b/tests/smoke_test_data/run_smoke_test.sh @@ -28,7 +28,7 @@ echo "Step 1: Checking API health..." # Step 2: Index documents echo "" echo "Step 2: Indexing 500 documents..." -python3 "${PROJECT_ROOT}/utility/data_indexer.py" \ +python3 "${PROJECT_ROOT}/scripts/data_indexer.py" \ -u "${URL}" \ -d "${DATA_DIR}" \ -p "${PARTITION}" diff --git a/tests/unit/api/conftest.py b/tests/unit/api/conftest.py new file mode 100644 index 000000000..e22dbe0f3 --- /dev/null +++ b/tests/unit/api/conftest.py @@ -0,0 +1,45 @@ +"""API unit-test fixtures: an ASGI HTTPX client factory for exercising +FastAPI apps in-process without a running server. + +Most router/middleware unit tests build a small purpose-built ``FastAPI`` app +(mounting only the router under test) and override dependencies with mocks. +``async_client_factory`` wraps any such app in an ``httpx.AsyncClient`` backed +by ``ASGITransport``; ``async_client`` is a ready-to-use client over a bare app +for tests that supply their own routes via the returned app. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Callable + +import httpx +import pytest +import pytest_asyncio +from fastapi import FastAPI + + +@pytest.fixture() +def async_client_factory() -> Callable[[FastAPI], httpx.AsyncClient]: + """Return a builder that wraps a FastAPI app in an ASGI-backed client. + + Apply ``app.dependency_overrides[...] = ...`` before issuing requests to + swap real services for the mocks from ``tests/unit/conftest.py``. + """ + + def _build(app: FastAPI) -> httpx.AsyncClient: + transport = httpx.ASGITransport(app=app) + return httpx.AsyncClient(transport=transport, base_url="http://testserver") + + return _build + + +@pytest_asyncio.fixture() +async def async_client( + async_client_factory: Callable[[FastAPI], httpx.AsyncClient], +) -> AsyncIterator[httpx.AsyncClient]: + """An ASGI client over a bare app; mount routes on ``client._transport.app`` + or prefer ``async_client_factory`` when you need a pre-built app.""" + app = FastAPI() + client = async_client_factory(app) + async with client: + yield client diff --git a/openrag/api/dependencies/test_auth.py b/tests/unit/api/dependencies/test_auth.py similarity index 100% rename from openrag/api/dependencies/test_auth.py rename to tests/unit/api/dependencies/test_auth.py diff --git a/openrag/api/dependencies/test_files.py b/tests/unit/api/dependencies/test_files.py similarity index 100% rename from openrag/api/dependencies/test_files.py rename to tests/unit/api/dependencies/test_files.py diff --git a/openrag/api/mcp/test_server.py b/tests/unit/api/mcp/test_server.py similarity index 100% rename from openrag/api/mcp/test_server.py rename to tests/unit/api/mcp/test_server.py diff --git a/openrag/api/middleware/test_bypass_config.py b/tests/unit/api/middleware/test_bypass_config.py similarity index 100% rename from openrag/api/middleware/test_bypass_config.py rename to tests/unit/api/middleware/test_bypass_config.py diff --git a/openrag/api/middleware/test_instrumentation.py b/tests/unit/api/middleware/test_instrumentation.py similarity index 100% rename from openrag/api/middleware/test_instrumentation.py rename to tests/unit/api/middleware/test_instrumentation.py diff --git a/openrag/api/middleware/test_request_id.py b/tests/unit/api/middleware/test_request_id.py similarity index 100% rename from openrag/api/middleware/test_request_id.py rename to tests/unit/api/middleware/test_request_id.py diff --git a/openrag/api/middleware/test_request_timeout.py b/tests/unit/api/middleware/test_request_timeout.py similarity index 100% rename from openrag/api/middleware/test_request_timeout.py rename to tests/unit/api/middleware/test_request_timeout.py diff --git a/openrag/api/routers/admin/test_workspaces.py b/tests/unit/api/routers/admin/test_admin_workspaces.py similarity index 100% rename from openrag/api/routers/admin/test_workspaces.py rename to tests/unit/api/routers/admin/test_admin_workspaces.py diff --git a/openrag/api/routers/admin/test_task_logs.py b/tests/unit/api/routers/admin/test_task_logs.py similarity index 100% rename from openrag/api/routers/admin/test_task_logs.py rename to tests/unit/api/routers/admin/test_task_logs.py diff --git a/openrag/api/routers/admin/test_task_logs_legacy_cases.py b/tests/unit/api/routers/admin/test_task_logs_legacy_cases.py similarity index 100% rename from openrag/api/routers/admin/test_task_logs_legacy_cases.py rename to tests/unit/api/routers/admin/test_task_logs_legacy_cases.py diff --git a/openrag/api/routers/test_router_imports.py b/tests/unit/api/routers/test_router_imports.py similarity index 100% rename from openrag/api/routers/test_router_imports.py rename to tests/unit/api/routers/test_router_imports.py diff --git a/openrag/api/routers/user/test_source_links.py b/tests/unit/api/routers/user/test_source_links.py similarity index 100% rename from openrag/api/routers/user/test_source_links.py rename to tests/unit/api/routers/user/test_source_links.py diff --git a/openrag/api/schemas/test_api_schema_imports.py b/tests/unit/api/schemas/test_api_schema_imports.py similarity index 100% rename from openrag/api/schemas/test_api_schema_imports.py rename to tests/unit/api/schemas/test_api_schema_imports.py diff --git a/openrag/api/test_error_handlers.py b/tests/unit/api/test_error_handlers.py similarity index 100% rename from openrag/api/test_error_handlers.py rename to tests/unit/api/test_error_handlers.py diff --git a/openrag/api/test_main_proxy_headers.py b/tests/unit/api/test_main_proxy_headers.py similarity index 96% rename from openrag/api/test_main_proxy_headers.py rename to tests/unit/api/test_main_proxy_headers.py index 4a09fa232..4b1d8725a 100644 --- a/openrag/api/test_main_proxy_headers.py +++ b/tests/unit/api/test_main_proxy_headers.py @@ -11,11 +11,11 @@ import ast import importlib -import os import sys +from pathlib import Path from types import ModuleType -_MAIN_PATH = os.path.dirname(__file__) + "/main.py" +_MAIN_PATH = Path(__file__).resolve().parents[3] / "openrag" / "api" / "main.py" def test_uvicorn_run_passes_forwarded_allow_ips(): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 000000000..cc27f097c --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,113 @@ +"""Shared unit-test fixtures: dependency-free mock implementations of the +core ports so unit tests can exercise services without real infrastructure. + +The mocks subclass the actual port ABCs (``core.embeddings.embedder.Embedder``, +``core.vector_stores.vector_store.VectorStore``) so they stay in sync — adding a +new abstract method makes the mock fail to instantiate until it is implemented. +``CatalogStore`` has too many repository accessors to hand-write, so it is +provided as a ``MagicMock`` spec'd against the real class. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +import pytest +from core.embeddings.embedder import Embedder +from core.ports.catalog_store import CatalogStore +from core.vector_stores.vector_store import VectorStore + + +class MockEmbedder(Embedder): + """Deterministic in-memory embedder. Returns fixed-dimension vectors.""" + + def __init__(self, dimension: int = 8) -> None: + self._dimension = dimension + self.embed_calls: list[list[str]] = [] + + async def embed(self, texts: list[str]) -> list[list[float]]: + self.embed_calls.append(list(texts)) + return [self._vector(t) for t in texts] + + async def embed_single(self, text: str) -> list[float]: + return self._vector(text) + + @property + def dimension(self) -> int: + return self._dimension + + def _vector(self, text: str) -> list[float]: + # Stable pseudo-embedding derived from the text length so equal inputs + # yield equal vectors without any external service. + seed = float(len(text) % 7) + return [seed + i for i in range(self._dimension)] + + +class MockVectorStore(VectorStore): + """In-memory vector store keyed by collection name.""" + + def __init__(self) -> None: + self.collections: dict[str, dict[str, Any]] = {} + self.search_results: list[dict[str, Any]] = [] + + async def upsert(self, chunks: list[Any], collection: str = "default") -> int: + store = self.collections.setdefault(collection, {}) + for chunk in chunks: + store[getattr(chunk, "id", id(chunk))] = chunk + return len(chunks) + + async def search( + self, + embedding: list[float], + query_text: str | None = None, + top_k: int = 10, + collection: str = "default", + filters: dict[str, Any] | None = None, + similarity_threshold: float | None = None, + ) -> list[dict[str, Any]]: + return self.search_results[:top_k] + + async def delete(self, ids: list[str], collection: str = "default") -> int: + store = self.collections.get(collection, {}) + removed = 0 + for cid in ids: + if store.pop(cid, None) is not None: + removed += 1 + return removed + + async def ensure_collection(self, name: str, dimension: int, **kwargs: Any) -> None: + self.collections.setdefault(name, {}) + + async def drop_collection(self, name: str) -> None: + self.collections.pop(name, None) + + async def collection_exists(self, name: str) -> bool: + return name in self.collections + + async def query_ids_by_filter(self, collection: str, filters: dict[str, Any]) -> list[str]: + return list(self.collections.get(collection, {}).keys()) + + async def query_chunks_by_filter( + self, + collection: str, + filters: dict[str, Any], + output_fields: list[str] | None = None, + ) -> list[dict[str, Any]]: + return [] + + +@pytest.fixture() +def mock_embedder() -> MockEmbedder: + return MockEmbedder() + + +@pytest.fixture() +def mock_vector_store() -> MockVectorStore: + return MockVectorStore() + + +@pytest.fixture() +def mock_catalog_store() -> MagicMock: + """A ``CatalogStore`` whose repository accessors return autospec'd mocks.""" + return MagicMock(spec=CatalogStore) diff --git a/openrag/core/chunking/test_markdown_utils.py b/tests/unit/core/chunking/test_markdown_utils.py similarity index 100% rename from openrag/core/chunking/test_markdown_utils.py rename to tests/unit/core/chunking/test_markdown_utils.py diff --git a/openrag/core/chunking/test_recursive.py b/tests/unit/core/chunking/test_recursive.py similarity index 100% rename from openrag/core/chunking/test_recursive.py rename to tests/unit/core/chunking/test_recursive.py diff --git a/openrag/core/config/test_auth_env_validation.py b/tests/unit/core/config/test_auth_env_validation.py similarity index 100% rename from openrag/core/config/test_auth_env_validation.py rename to tests/unit/core/config/test_auth_env_validation.py diff --git a/openrag/core/config/test_indexation.py b/tests/unit/core/config/test_indexation.py similarity index 100% rename from openrag/core/config/test_indexation.py rename to tests/unit/core/config/test_indexation.py diff --git a/openrag/core/indexing/parsers/pdf/test_docling.py b/tests/unit/core/indexing/parsers/pdf/test_docling.py similarity index 100% rename from openrag/core/indexing/parsers/pdf/test_docling.py rename to tests/unit/core/indexing/parsers/pdf/test_docling.py diff --git a/openrag/core/indexing/parsers/test_doc_parser.py b/tests/unit/core/indexing/parsers/test_doc_parser.py similarity index 96% rename from openrag/core/indexing/parsers/test_doc_parser.py rename to tests/unit/core/indexing/parsers/test_doc_parser.py index 454de181f..febf57bca 100644 --- a/openrag/core/indexing/parsers/test_doc_parser.py +++ b/tests/unit/core/indexing/parsers/test_doc_parser.py @@ -6,9 +6,8 @@ from unittest.mock import AsyncMock, MagicMock import pytest - -from ...models.document import Document, DocumentType, ProcessedDocument, TextBlock -from .doc_parser import DocParser +from core.indexing.parsers.doc_parser import DocParser +from core.models.document import Document, DocumentType, ProcessedDocument, TextBlock @pytest.fixture diff --git a/openrag/core/indexing/parsers/test_docx_parser.py b/tests/unit/core/indexing/parsers/test_docx_parser.py similarity index 97% rename from openrag/core/indexing/parsers/test_docx_parser.py rename to tests/unit/core/indexing/parsers/test_docx_parser.py index cf612b069..3c6b22458 100644 --- a/openrag/core/indexing/parsers/test_docx_parser.py +++ b/tests/unit/core/indexing/parsers/test_docx_parser.py @@ -8,11 +8,10 @@ from pathlib import Path import pytest +from core.indexing.parsers.docx_parser import DocxParser, _image_ref +from core.models.document import Document, DocumentType from PIL import Image -from ...models.document import Document, DocumentType -from .docx_parser import DocxParser, _image_ref - def _png_bytes(color: str = "red") -> bytes: img = Image.new("RGBA", (10, 10), color) diff --git a/openrag/core/indexing/parsers/test_pptx_parser.py b/tests/unit/core/indexing/parsers/test_pptx_parser.py similarity index 100% rename from openrag/core/indexing/parsers/test_pptx_parser.py rename to tests/unit/core/indexing/parsers/test_pptx_parser.py diff --git a/openrag/core/indexing/test_image_preprocessor.py b/tests/unit/core/indexing/test_image_preprocessor.py similarity index 98% rename from openrag/core/indexing/test_image_preprocessor.py rename to tests/unit/core/indexing/test_image_preprocessor.py index 29ae9ec5d..08bbc3dbb 100644 --- a/openrag/core/indexing/test_image_preprocessor.py +++ b/tests/unit/core/indexing/test_image_preprocessor.py @@ -4,9 +4,7 @@ import base64 -from PIL import Image - -from .image_preprocessor import ( +from core.indexing.image_preprocessor import ( MIN_IMAGE_PIXELS, decode_data_uri, ensure_png_compatible_mode, @@ -14,6 +12,7 @@ mime_from_data_uri, pil_to_png_bytes, ) +from PIL import Image class TestEnsurePngCompatibleMode: diff --git a/openrag/core/indexing/test_validators.py b/tests/unit/core/indexing/test_validators.py similarity index 95% rename from openrag/core/indexing/test_validators.py rename to tests/unit/core/indexing/test_validators.py index 930896cec..ce70c4065 100644 --- a/openrag/core/indexing/test_validators.py +++ b/tests/unit/core/indexing/test_validators.py @@ -3,9 +3,8 @@ from __future__ import annotations import pytest - -from ..utils.exceptions import ValidationError -from .validators import parse_metadata, validate_file_format, validate_file_id +from core.indexing.validators import parse_metadata, validate_file_format, validate_file_id +from core.utils.exceptions import ValidationError class TestParseMetadata: diff --git a/openrag/core/models/test_chunk.py b/tests/unit/core/models/test_chunk.py similarity index 100% rename from openrag/core/models/test_chunk.py rename to tests/unit/core/models/test_chunk.py diff --git a/openrag/core/prompts/test_chat_prompt_builder.py b/tests/unit/core/prompts/test_chat_prompt_builder.py similarity index 100% rename from openrag/core/prompts/test_chat_prompt_builder.py rename to tests/unit/core/prompts/test_chat_prompt_builder.py diff --git a/openrag/core/prompts/test_contextualization_builder.py b/tests/unit/core/prompts/test_contextualization_builder.py similarity index 100% rename from openrag/core/prompts/test_contextualization_builder.py rename to tests/unit/core/prompts/test_contextualization_builder.py diff --git a/openrag/core/prompts/test_map_reduce_builder.py b/tests/unit/core/prompts/test_map_reduce_builder.py similarity index 100% rename from openrag/core/prompts/test_map_reduce_builder.py rename to tests/unit/core/prompts/test_map_reduce_builder.py diff --git a/openrag/core/prompts/test_query_rewriter.py b/tests/unit/core/prompts/test_query_rewriter.py similarity index 100% rename from openrag/core/prompts/test_query_rewriter.py rename to tests/unit/core/prompts/test_query_rewriter.py diff --git a/openrag/core/prompts/test_template_loader.py b/tests/unit/core/prompts/test_template_loader.py similarity index 100% rename from openrag/core/prompts/test_template_loader.py rename to tests/unit/core/prompts/test_template_loader.py diff --git a/openrag/core/prompts/test_vlm_prompt_builder.py b/tests/unit/core/prompts/test_vlm_prompt_builder.py similarity index 100% rename from openrag/core/prompts/test_vlm_prompt_builder.py rename to tests/unit/core/prompts/test_vlm_prompt_builder.py diff --git a/openrag/core/retrieval/test_pipeline.py b/tests/unit/core/retrieval/test_pipeline.py similarity index 100% rename from openrag/core/retrieval/test_pipeline.py rename to tests/unit/core/retrieval/test_pipeline.py diff --git a/openrag/core/retrieval/test_retriever.py b/tests/unit/core/retrieval/test_retriever.py similarity index 100% rename from openrag/core/retrieval/test_retriever.py rename to tests/unit/core/retrieval/test_retriever.py diff --git a/openrag/core/retrieval/test_rrf.py b/tests/unit/core/retrieval/test_rrf.py similarity index 100% rename from openrag/core/retrieval/test_rrf.py rename to tests/unit/core/retrieval/test_rrf.py diff --git a/openrag/core/utils/test_external_errors.py b/tests/unit/core/utils/test_external_errors.py similarity index 100% rename from openrag/core/utils/test_external_errors.py rename to tests/unit/core/utils/test_external_errors.py diff --git a/openrag/core/utils/test_log_tail.py b/tests/unit/core/utils/test_log_tail.py similarity index 100% rename from openrag/core/utils/test_log_tail.py rename to tests/unit/core/utils/test_log_tail.py diff --git a/openrag/core/utils/test_logging.py b/tests/unit/core/utils/test_logging.py similarity index 100% rename from openrag/core/utils/test_logging.py rename to tests/unit/core/utils/test_logging.py diff --git a/openrag/core/utils/test_source_filtering.py b/tests/unit/core/utils/test_source_filtering.py similarity index 100% rename from openrag/core/utils/test_source_filtering.py rename to tests/unit/core/utils/test_source_filtering.py diff --git a/openrag/core/utils/test_url_safety.py b/tests/unit/core/utils/test_url_safety.py similarity index 100% rename from openrag/core/utils/test_url_safety.py rename to tests/unit/core/utils/test_url_safety.py diff --git a/openrag/di/test_container.py b/tests/unit/di/test_container.py similarity index 100% rename from openrag/di/test_container.py rename to tests/unit/di/test_container.py diff --git a/openrag/di/test_factories.py b/tests/unit/di/test_factories.py similarity index 100% rename from openrag/di/test_factories.py rename to tests/unit/di/test_factories.py diff --git a/openrag/di/test_inference.py b/tests/unit/di/test_inference.py similarity index 100% rename from openrag/di/test_inference.py rename to tests/unit/di/test_inference.py diff --git a/openrag/di/test_workers.py b/tests/unit/di/test_workers.py similarity index 100% rename from openrag/di/test_workers.py rename to tests/unit/di/test_workers.py diff --git a/openrag/services/inference/parsers/test_openai_audio.py b/tests/unit/services/inference/parsers/test_openai_audio.py similarity index 98% rename from openrag/services/inference/parsers/test_openai_audio.py rename to tests/unit/services/inference/parsers/test_openai_audio.py index 4389e2f52..ef997a3fc 100644 --- a/openrag/services/inference/parsers/test_openai_audio.py +++ b/tests/unit/services/inference/parsers/test_openai_audio.py @@ -27,8 +27,7 @@ sys.modules["pydub"] = fake_pydub from core.models.document import Document, DocumentType # noqa: E402 - -from .openai_audio import OpenAIAudioClient # noqa: E402 +from services.inference.parsers.openai_audio import OpenAIAudioClient # noqa: E402 # ---- shared fixtures ------------------------------------------------------- diff --git a/openrag/services/inference/test_circuit_breaker.py b/tests/unit/services/inference/test_circuit_breaker.py similarity index 100% rename from openrag/services/inference/test_circuit_breaker.py rename to tests/unit/services/inference/test_circuit_breaker.py diff --git a/openrag/services/inference/test_distributed_semaphore.py b/tests/unit/services/inference/test_distributed_semaphore.py similarity index 100% rename from openrag/services/inference/test_distributed_semaphore.py rename to tests/unit/services/inference/test_distributed_semaphore.py diff --git a/openrag/services/inference/test_healthcheck.py b/tests/unit/services/inference/test_healthcheck.py similarity index 99% rename from openrag/services/inference/test_healthcheck.py rename to tests/unit/services/inference/test_healthcheck.py index 24ff2054e..e80bd587d 100644 --- a/openrag/services/inference/test_healthcheck.py +++ b/tests/unit/services/inference/test_healthcheck.py @@ -4,8 +4,7 @@ import httpx import pytest - -from .healthcheck import ( +from services.inference.healthcheck import ( EndpointStatus, check_endpoint_health, check_infinity, diff --git a/openrag/services/inference/test_ollama_client.py b/tests/unit/services/inference/test_ollama_client.py similarity index 99% rename from openrag/services/inference/test_ollama_client.py rename to tests/unit/services/inference/test_ollama_client.py index f54f42f60..d43f401e2 100644 --- a/openrag/services/inference/test_ollama_client.py +++ b/tests/unit/services/inference/test_ollama_client.py @@ -13,8 +13,7 @@ InferenceTimeoutError, ) from services.inference._circuit_breaker import _breakers - -from .ollama_client import OllamaClient, OllamaEmbedder +from services.inference.ollama_client import OllamaClient, OllamaEmbedder @pytest.fixture(autouse=True) diff --git a/openrag/services/inference/test_reranker_clients.py b/tests/unit/services/inference/test_reranker_clients.py similarity index 98% rename from openrag/services/inference/test_reranker_clients.py rename to tests/unit/services/inference/test_reranker_clients.py index 860dd94c8..6447759b0 100644 --- a/openrag/services/inference/test_reranker_clients.py +++ b/tests/unit/services/inference/test_reranker_clients.py @@ -5,8 +5,7 @@ import httpx import pytest from core.utils.exceptions import InferenceConnectionError, InferenceTimeoutError - -from .reranker_clients import InfinityReranker, OpenAIReranker +from services.inference.reranker_clients import InfinityReranker, OpenAIReranker def _rerank_response(results: list[dict] | None = None) -> httpx.Response: diff --git a/openrag/services/inference/test_retry.py b/tests/unit/services/inference/test_retry.py similarity index 100% rename from openrag/services/inference/test_retry.py rename to tests/unit/services/inference/test_retry.py diff --git a/openrag/services/inference/test_runtime.py b/tests/unit/services/inference/test_runtime.py similarity index 100% rename from openrag/services/inference/test_runtime.py rename to tests/unit/services/inference/test_runtime.py diff --git a/openrag/services/inference/test_vllm_client.py b/tests/unit/services/inference/test_vllm_client.py similarity index 99% rename from openrag/services/inference/test_vllm_client.py rename to tests/unit/services/inference/test_vllm_client.py index 797bb2760..6ea1c48f8 100644 --- a/openrag/services/inference/test_vllm_client.py +++ b/tests/unit/services/inference/test_vllm_client.py @@ -13,8 +13,7 @@ InferenceTimeoutError, ) from services.inference._circuit_breaker import _breakers - -from .vllm_client import VLLMClient, VLLMEmbedder, VLLMVision +from services.inference.vllm_client import VLLMClient, VLLMEmbedder, VLLMVision @pytest.fixture(autouse=True) diff --git a/openrag/services/orchestrators/test_auth_service.py b/tests/unit/services/orchestrators/test_auth_service.py similarity index 100% rename from openrag/services/orchestrators/test_auth_service.py rename to tests/unit/services/orchestrators/test_auth_service.py diff --git a/openrag/services/orchestrators/test_conversion_service.py b/tests/unit/services/orchestrators/test_conversion_service.py similarity index 100% rename from openrag/services/orchestrators/test_conversion_service.py rename to tests/unit/services/orchestrators/test_conversion_service.py diff --git a/openrag/services/orchestrators/test_indexing_service.py b/tests/unit/services/orchestrators/test_indexing_service.py similarity index 100% rename from openrag/services/orchestrators/test_indexing_service.py rename to tests/unit/services/orchestrators/test_indexing_service.py diff --git a/openrag/services/orchestrators/test_job_service.py b/tests/unit/services/orchestrators/test_job_service.py similarity index 100% rename from openrag/services/orchestrators/test_job_service.py rename to tests/unit/services/orchestrators/test_job_service.py diff --git a/openrag/services/orchestrators/test_mcp_service.py b/tests/unit/services/orchestrators/test_mcp_service.py similarity index 100% rename from openrag/services/orchestrators/test_mcp_service.py rename to tests/unit/services/orchestrators/test_mcp_service.py diff --git a/openrag/services/orchestrators/test_partition_service.py b/tests/unit/services/orchestrators/test_partition_service.py similarity index 100% rename from openrag/services/orchestrators/test_partition_service.py rename to tests/unit/services/orchestrators/test_partition_service.py diff --git a/openrag/services/orchestrators/test_query_service.py b/tests/unit/services/orchestrators/test_query_service.py similarity index 100% rename from openrag/services/orchestrators/test_query_service.py rename to tests/unit/services/orchestrators/test_query_service.py diff --git a/openrag/services/orchestrators/test_retrieval_service.py b/tests/unit/services/orchestrators/test_retrieval_service.py similarity index 100% rename from openrag/services/orchestrators/test_retrieval_service.py rename to tests/unit/services/orchestrators/test_retrieval_service.py diff --git a/openrag/services/orchestrators/test_user_service.py b/tests/unit/services/orchestrators/test_user_service.py similarity index 100% rename from openrag/services/orchestrators/test_user_service.py rename to tests/unit/services/orchestrators/test_user_service.py diff --git a/openrag/services/orchestrators/test_workspace_service.py b/tests/unit/services/orchestrators/test_workspace_service.py similarity index 100% rename from openrag/services/orchestrators/test_workspace_service.py rename to tests/unit/services/orchestrators/test_workspace_service.py diff --git a/openrag/services/persistence/test_add_file_to_partition_user_id.py b/tests/unit/services/persistence/test_add_file_to_partition_user_id.py similarity index 100% rename from openrag/services/persistence/test_add_file_to_partition_user_id.py rename to tests/unit/services/persistence/test_add_file_to_partition_user_id.py diff --git a/openrag/services/persistence/test_add_partition_member.py b/tests/unit/services/persistence/test_add_partition_member.py similarity index 100% rename from openrag/services/persistence/test_add_partition_member.py rename to tests/unit/services/persistence/test_add_partition_member.py diff --git a/openrag/services/persistence/test_ancestor_recursion_cap.py b/tests/unit/services/persistence/test_ancestor_recursion_cap.py similarity index 100% rename from openrag/services/persistence/test_ancestor_recursion_cap.py rename to tests/unit/services/persistence/test_ancestor_recursion_cap.py diff --git a/openrag/services/persistence/test_connection.py b/tests/unit/services/persistence/test_connection.py similarity index 100% rename from openrag/services/persistence/test_connection.py rename to tests/unit/services/persistence/test_connection.py diff --git a/openrag/services/persistence/test_delete_workspace.py b/tests/unit/services/persistence/test_delete_workspace.py similarity index 100% rename from openrag/services/persistence/test_delete_workspace.py rename to tests/unit/services/persistence/test_delete_workspace.py diff --git a/openrag/services/persistence/test_ensure_admin_user.py b/tests/unit/services/persistence/test_ensure_admin_user.py similarity index 100% rename from openrag/services/persistence/test_ensure_admin_user.py rename to tests/unit/services/persistence/test_ensure_admin_user.py diff --git a/openrag/services/persistence/test_user_repo_external_id.py b/tests/unit/services/persistence/test_user_repo_external_id.py similarity index 100% rename from openrag/services/persistence/test_user_repo_external_id.py rename to tests/unit/services/persistence/test_user_repo_external_id.py diff --git a/openrag/services/storage/test_milvus_store.py b/tests/unit/services/storage/test_milvus_store.py similarity index 100% rename from openrag/services/storage/test_milvus_store.py rename to tests/unit/services/storage/test_milvus_store.py diff --git a/openrag/services/storage/test_vector_store_searcher.py b/tests/unit/services/storage/test_vector_store_searcher.py similarity index 100% rename from openrag/services/storage/test_vector_store_searcher.py rename to tests/unit/services/storage/test_vector_store_searcher.py diff --git a/openrag/services/websearch/test_content_fetcher.py b/tests/unit/services/websearch/test_content_fetcher.py similarity index 100% rename from openrag/services/websearch/test_content_fetcher.py rename to tests/unit/services/websearch/test_content_fetcher.py diff --git a/openrag/services/workers/parsers/legacy_loaders/audio/test_openai.py b/tests/unit/services/workers/parsers/legacy_loaders/audio/test_openai.py similarity index 96% rename from openrag/services/workers/parsers/legacy_loaders/audio/test_openai.py rename to tests/unit/services/workers/parsers/legacy_loaders/audio/test_openai.py index 667c790e9..fdd809423 100644 --- a/openrag/services/workers/parsers/legacy_loaders/audio/test_openai.py +++ b/tests/unit/services/workers/parsers/legacy_loaders/audio/test_openai.py @@ -15,6 +15,13 @@ from pathlib import Path # noqa: E402 import pytest # noqa: E402 + +# These tests exercise real pydub audio operations. pydub depends on the stdlib +# ``audioop`` module, which was removed in Python 3.13 — guard on it directly so +# the module skips cleanly (other tests register a mock ``pydub`` in sys.modules, +# so importorskip("pydub") would not catch the missing dependency). +pytest.importorskip("audioop") + from pydub import AudioSegment # noqa: E402 from pydub.generators import Sine # noqa: E402 diff --git a/openrag/services/workers/parsers/legacy_loaders/test_base_loader.py b/tests/unit/services/workers/parsers/legacy_loaders/test_base_loader.py similarity index 96% rename from openrag/services/workers/parsers/legacy_loaders/test_base_loader.py rename to tests/unit/services/workers/parsers/legacy_loaders/test_base_loader.py index 8f85f5acb..3ab2c342e 100644 --- a/openrag/services/workers/parsers/legacy_loaders/test_base_loader.py +++ b/tests/unit/services/workers/parsers/legacy_loaders/test_base_loader.py @@ -4,8 +4,7 @@ import pytest from PIL import Image - -from .base import BaseLoader, ensure_png_compatible_mode +from services.workers.parsers.legacy_loaders.base import BaseLoader, ensure_png_compatible_mode class ConcreteLoader(BaseLoader): diff --git a/openrag/services/workers/parsers/legacy_loaders/test_customdocloader.py b/tests/unit/services/workers/parsers/legacy_loaders/test_customdocloader.py similarity index 100% rename from openrag/services/workers/parsers/legacy_loaders/test_customdocloader.py rename to tests/unit/services/workers/parsers/legacy_loaders/test_customdocloader.py diff --git a/openrag/services/workers/parsers/legacy_loaders/test_doc_loader.py b/tests/unit/services/workers/parsers/legacy_loaders/test_doc_loader.py similarity index 100% rename from openrag/services/workers/parsers/legacy_loaders/test_doc_loader.py rename to tests/unit/services/workers/parsers/legacy_loaders/test_doc_loader.py diff --git a/openrag/services/workers/parsers/legacy_loaders/test_docx_loader.py b/tests/unit/services/workers/parsers/legacy_loaders/test_docx_loader.py similarity index 98% rename from openrag/services/workers/parsers/legacy_loaders/test_docx_loader.py rename to tests/unit/services/workers/parsers/legacy_loaders/test_docx_loader.py index 367ddac56..362522a7d 100644 --- a/openrag/services/workers/parsers/legacy_loaders/test_docx_loader.py +++ b/tests/unit/services/workers/parsers/legacy_loaders/test_docx_loader.py @@ -11,8 +11,7 @@ from pathlib import Path from PIL import Image - -from .docx import DocxLoader, convert_to_png_image +from services.workers.parsers.legacy_loaders.docx import DocxLoader, convert_to_png_image def _create_png_bytes(width=10, height=10, color="red"): diff --git a/openrag/services/workers/parsers/legacy_loaders/test_eml_recursion.py b/tests/unit/services/workers/parsers/legacy_loaders/test_eml_recursion.py similarity index 100% rename from openrag/services/workers/parsers/legacy_loaders/test_eml_recursion.py rename to tests/unit/services/workers/parsers/legacy_loaders/test_eml_recursion.py diff --git a/openrag/services/workers/stages/test_parse.py b/tests/unit/services/workers/stages/test_parse.py similarity index 100% rename from openrag/services/workers/stages/test_parse.py rename to tests/unit/services/workers/stages/test_parse.py diff --git a/openrag/services/workers/stages/test_pipeline_stages.py b/tests/unit/services/workers/stages/test_pipeline_stages.py similarity index 100% rename from openrag/services/workers/stages/test_pipeline_stages.py rename to tests/unit/services/workers/stages/test_pipeline_stages.py diff --git a/openrag/services/workers/test_batch_ingest.py b/tests/unit/services/workers/test_batch_ingest.py similarity index 100% rename from openrag/services/workers/test_batch_ingest.py rename to tests/unit/services/workers/test_batch_ingest.py diff --git a/openrag/services/workers/test_dispatcher.py b/tests/unit/services/workers/test_dispatcher.py similarity index 100% rename from openrag/services/workers/test_dispatcher.py rename to tests/unit/services/workers/test_dispatcher.py diff --git a/openrag/services/workers/test_indexer_pool.py b/tests/unit/services/workers/test_indexer_pool.py similarity index 100% rename from openrag/services/workers/test_indexer_pool.py rename to tests/unit/services/workers/test_indexer_pool.py diff --git a/openrag/services/workers/test_indexer_worker.py b/tests/unit/services/workers/test_indexer_worker.py similarity index 100% rename from openrag/services/workers/test_indexer_worker.py rename to tests/unit/services/workers/test_indexer_worker.py diff --git a/openrag/services/workers/test_pipeline_builder.py b/tests/unit/services/workers/test_pipeline_builder.py similarity index 100% rename from openrag/services/workers/test_pipeline_builder.py rename to tests/unit/services/workers/test_pipeline_builder.py diff --git a/openrag/test_app_front_secret.py b/tests/unit/test_app_front_secret.py similarity index 94% rename from openrag/test_app_front_secret.py rename to tests/unit/test_app_front_secret.py index e4a323d04..c75bf5c23 100644 --- a/openrag/test_app_front_secret.py +++ b/tests/unit/test_app_front_secret.py @@ -3,12 +3,12 @@ default secret. """ -import os import sys +from pathlib import Path import pytest -_FIX_SOURCE = os.path.dirname(__file__) + "/app_front.py" +_FIX_SOURCE = Path(__file__).resolve().parents[2] / "openrag" / "app_front.py" def test_no_hardcoded_default_secret_assignment_in_source(): diff --git a/openrag/test_auth_router.py b/tests/unit/test_auth_router.py similarity index 100% rename from openrag/test_auth_router.py rename to tests/unit/test_auth_router.py diff --git a/openrag/test_chainlit_api_auth.py b/tests/unit/test_chainlit_api_auth.py similarity index 100% rename from openrag/test_chainlit_api_auth.py rename to tests/unit/test_chainlit_api_auth.py diff --git a/openrag/test_token_validation.py b/tests/unit/test_token_validation.py similarity index 100% rename from openrag/test_token_validation.py rename to tests/unit/test_token_validation.py diff --git a/openrag/test_version.py b/tests/unit/test_version.py similarity index 100% rename from openrag/test_version.py rename to tests/unit/test_version.py diff --git a/ui b/ui new file mode 120000 index 000000000..42efd4325 --- /dev/null +++ b/ui @@ -0,0 +1 @@ +extern/indexer-ui \ No newline at end of file diff --git a/utility/requirements.txt b/utility/requirements.txt deleted file mode 100644 index cf196c5e8..000000000 --- a/utility/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -httpx -loguru -