diff --git a/CLAUDE.md b/CLAUDE.md index 5f6e03f60..f54248011 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,7 +17,6 @@ infra/ # All deployment infrastructure scripts/ # entrypoint.sh and other deployment scripts ansible/ # Ansible playbooks charts/ # Helm charts (openrag-stack) - quick_start/ # Getting-started compose cluster.yaml # Ray cluster config scripts/ # Developer/operational CLI tools (check_layer_imports.py, data_indexer.py, postgres-init/) tests/ # Integration tests (api_tests/, integration/) diff --git a/README.md b/README.md index 1614190df..1e572febe 100644 --- a/README.md +++ b/README.md @@ -143,50 +143,34 @@ Create a `.env` file under `infra/compose/`, mirroring the structure of `infra/c cp infra/compose/.env.example infra/compose/.env ``` #### 3. File Parser configuration -All supported file format parsers are pre-configured. For PDF processing, **[MarkerLoader](https://github.com/datalab-to/marker)** serves as the default parser, offering comprehensive support for OCR-scanned documents, complex layouts, tables, and embedded images. MarkerLoader operates efficiently on both GPU and CPU environments. +All supported file format parsers are pre-configured. For PDF processing, **[PyMuPDFLoader](https://pymupdf.readthedocs.io/)** is the default parser — a lightweight, fast, CPU-friendly engine well suited to searchable PDFs and quick local testing. + +> ⚠️ **Important**: `PyMuPDFLoader` cannot process non-searchable (image-based / scanned) PDFs and does not run OCR or extract embedded images.
For more PDF options -For CPU-only deployments or lightweight testing scenarios, you can consider switching to **`PyMuPDFLoader`**. To change the loader, set the **`PDFLoader`** variable like this `PDFLoader=PyMuPDFLoader`. - -> ⚠️ **Important**: These alternative loaders have limitations - they cannot process non-searchable (image-based) PDFs and do not extract or handle embedded images. +For OCR-scanned documents, complex layouts, tables, or embedded images, switch to **[`MarkerLoader`](https://github.com/datalab-to/marker)** (heavier; runs on GPU and CPU) by setting the **`PDFLOADER`** variable: `PDFLOADER=MarkerLoader`. Other options: `DoclingLoader`, `DotsOCRLoader`.
#### 4.Deployment: Launch the app >[!IMPORTANT] > The **admin UI** (a web interface for intuitive document ingestion, indexing, and management) ships bundled as the `admin-ui` service — no separate setup is required. Once the stack is up it is served at `http://localhost:ADMIN_UI_PORT/app/` (default port `8081`). -* **Simple and quick** launch for testing - >[!IMPORTANT] - > For a **simple `quick deployment`** using only the docker-compose file, only the [quick_start **folder**](./infra/quick_start/) is required. Follow these steps to launch the application: - - 1. Navigate to the **`infra/quick_start`** directory or download only that folder - 2. Place your **`.env`** file inside the **`infra/quick_start`** directory - 3. Run the appropriate command for your system: - - ```bash - # GPU deployment (recommended for optimal performance) - docker compose up -d - # docker compose down # to stop the application - - # CPU deployment - docker compose --profile cpu up -d - # docker compose --profile cpu down # to stop the application - ``` -* **Development Environment**: For development builds, use the **`--build`** flag to rebuild images: - >[!NOTE] - > The full stack and its service configs live under `infra/compose/`. Execute these commands from there (`cd infra/compose`). - - ```bash - # GPU deployment with rebuild (recommended for optimal performance) - docker compose up --build -d - # docker compose down # to stop the application - - # CPU deployment with rebuild - docker compose --profile cpu up --build -d - # docker compose --profile cpu down # to stop the application - ``` +The full stack and its service configs live under **`infra/compose/`**. Place the **`.env`** you created there and run the commands from that directory (`cd infra/compose`): + +```bash +# GPU deployment (recommended for optimal performance) +docker compose up -d +# docker compose down # to stop the application + +# CPU deployment +docker compose --profile cpu up -d +# docker compose --profile cpu down # to stop the application +``` + +>[!NOTE] +> For development builds, add the **`--build`** flag to rebuild images from your working tree, e.g. `docker compose up --build -d`. >[!WARNING] > The first startup may take longer as required dependencies are installed. diff --git a/docs/assets/compose_ollama_cpu.yaml b/docs/assets/compose_ollama_cpu.yaml deleted file mode 100644 index 01ba8aedc..000000000 --- a/docs/assets/compose_ollama_cpu.yaml +++ /dev/null @@ -1,112 +0,0 @@ -x-openrag: &openrag_template - image: linagoraai/openrag:macOS_poc - volumes: - - ./data:/app/data - - ./.cache/huggingface:/app/model_weights # Model weights for RAG - - ./ray_mount/.env:/ray_mount/.env # Shared environment variables - - ./ray_mount/logs:/app/logs - ports: - - 8090:8080 - # Localhost only: Ray dashboard/Jobs API is unauthenticated (CVE-2023-48022). Disable when in cluster mode - - 127.0.0.1:${RAY_DASHBOARD_PORT:-8265}:8265 - networks: - default: - aliases: - - openrag - env_file: - - .env - environment: - - APP_PORT=8090 - - AUTH_TOKEN=${AUTH_TOKEN:?Set a strong AUTH_TOKEN in your .env} - - RERANKER_ENABLED=false - - MARKER_MAX_PROCESSES=1 - - RAY_DEDUP_LOGS=0 - - RAY_ENABLE_UV_RUN_RUNTIME_ENV=0s - - RAY_memory_monitor_refresh_ms=0 - shm_size: 10.24gb - -services: - openrag: - <<: *openrag_template - deploy: {} - depends_on: - - milvus - - ollama - - rdb: - image: postgres:15 - environment: - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in your .env} - - POSTGRES_USER=root - volumes: - - ./db:/var/lib/postgresql/data - - ollama: - image: ollama/ollama:latest - ports: - - "11434:11434" - volumes: - - ./volumes/ollama:/root/.ollama - - ./ollama-entrypoint.sh:/entrypoint.sh - restart: unless-stopped - entrypoint: ["/usr/bin/bash", "/entrypoint.sh"] - - etcd: - image: quay.io/coreos/etcd:v3.5.16 - environment: - - ETCD_AUTO_COMPACTION_MODE=revision - - ETCD_AUTO_COMPACTION_RETENTION=1000 - - ETCD_QUOTA_BACKEND_BYTES=4294967296 - - ETCD_SNAPSHOT_COUNT=50000 - volumes: - - ./volumes/etcd:/etcd - command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd - healthcheck: - test: ["CMD", "etcdctl", "endpoint", "health"] - interval: 30s - timeout: 20s - retries: 3 - - minio: - image: minio/minio:RELEASE.2023-03-20T20-16-18Z - environment: - MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:?Set MINIO_ACCESS_KEY in your .env} - MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:?Set MINIO_SECRET_KEY in your .env} - volumes: - - ./volumes/minio:/minio_data - command: minio server /minio_data --console-address ":9001" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - - milvus: - image: milvusdb/milvus:v2.5.4 - command: ["milvus", "run", "standalone"] - security_opt: - - seccomp:unconfined - environment: - ETCD_ENDPOINTS: etcd:2379 - MINIO_ADDRESS: minio:9000 - MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY:?Set MINIO_ACCESS_KEY in your .env} - MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY:?Set MINIO_SECRET_KEY in your .env} - volumes: - - ./volumes/milvus:/var/lib/milvus - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] - interval: 30s - start_period: 90s - timeout: 20s - retries: 3 - ports: - - "19530:19530" - depends_on: - - "etcd" - - "minio" - - admin-ui: - image: linagoraai/openrag-admin-ui:latest - ports: - - "${ADMIN_UI_PORT:-8081}:8080" - restart: unless-stopped diff --git a/docs/assets/env_example.env b/docs/assets/env_example.env index 75b548a37..514aca329 100644 --- a/docs/assets/env_example.env +++ b/docs/assets/env_example.env @@ -1,62 +1,105 @@ -# LLM +# ============================================================================ +# OpenRAG — minimal .env +# +# Only the variables you must set for the default compose stack to boot are +# listed here. Every other knob (PDF/audio loaders, chunking, retriever, +# reranker, Ray Serve, admin UI, OIDC/SSO, rate limiting, MCP server, web +# search, …) has a sensible default and is documented in full at: +# +# https://linagora.github.io/openrag/documentation/env_vars/ +# ============================================================================ + +# ── LLM (external, OpenAI-compatible) ─────────────────────────────────────── BASE_URL= API_KEY= MODEL= +LLM_SEMAPHORE=10 -# VLM (Visual Language Model) you can set it to the same as LLM if your LLM supports images +# ── VLM (vision model, used for image understanding) ──────────────────────── +# Can reuse the LLM values above if that model accepts images. VLM_BASE_URL= VLM_API_KEY= VLM_MODEL= +VLM_SEMAPHORE=20 -## FastAPI App (no need to change it) -# APP_PORT=8080 # this is the forwarded port -# The uvicorn path runs a single worker by design (Ray provides concurrency). -# To scale the HTTP layer, use Ray Serve: ENABLE_RAY_SERVE=true with -# RAY_SERVE_NUM_REPLICAS=N (see the Ray Serve configuration section). +# ── Embedder (HuggingFace model served by the bundled vLLM) ───────────────── +# Point these at an external embedding service instead of the bundled vLLM: -## To enable API HTTP authentication via HTTPBearer -# AUTH_TOKEN=sk-openrag-1234 +EMBEDDER_MODEL_NAME=jinaai/jina-embeddings-v3 +# EMBEDDER_BASE_URL=http://vllm:8000/v1 +# EMBEDDER_API_KEY=EMPTY +# MAX_MODEL_LEN=2047 -# SAVE_UPLOADED_FILES=true # usefull for chainlit (chat interface) source viewing +# ── Reranker (re-scores retrieved chunks; bundled Infinity server) ─────────── +RERANKER_PROVIDER=infinity # 'infinity' (bundled, default) or 'openai' (external endpoint) +RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base +RERANKER_ENABLED=true -# Set to true, it will mount chainlit chat ui to the fastapi app (Default: true) -## WITH_CHAINLIT_UI=true +## Point these at an external reranker instead of the bundled Infinity / VLLM(Openai) server: +# RERANKER_BASE_URL=http://reranker:7997 +# RERANKER_API_KEY=EMPTY -# EMBEDDER -EMBEDDER_MODEL_NAME=jinaai/jina-embeddings-v3 # or other embedder from huggingface compatible with vllm -# EMBEDDER_BASE_URL=http://vllm:8000/v1 -# EMBEDDER_API_KEY=EMPTY +# ── PDF parser (default: PyMuPDF — lightweight, CPU-friendly) ──────────────── +# Switch to MarkerLoader for OCR / scanned PDFs, complex layouts & embedded +# images (heavier — more RAM/GPU). Other option: DoclingLoader. +PDFLOADER=PyMuPDFLoader +# Marker tuning (only when PDFLOADER=MarkerLoader): +# MARKER_POOL_SIZE=1 # marker worker actors (≈ 1 per cluster node / Machine). +# MARKER_MAX_PROCESSES=2 # concurrent PDFs per worker (raise with more GPU) + +# ── Image captioning & chunk contextualization (both ON by default) ────────── +# Both run during indexing and call the VLM/LLM — set to false to index +# faster and cheaper (with some retrieval-quality trade-off). +# IMAGE_CAPTIONING=false # stop describing images in documents via the VLM +# CONTEXTUAL_RETRIEVAL=false # stop prepending LLM-generated context to each chunk (Anthropic technique) + +# ── Secrets — dev defaults so the stack boots out of the box ───────────────── +# ⚠️ Production: replace these with strong values, e.g. `openssl rand -hex 16`. +# MINIO_* are shared by the minio service and Milvus — both sides must match. +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_KEY=minioadmin +POSTGRES_PASSWORD=postgres +# POSTGRES_USER=root + +# ── API and Chainlit (chat interface) authentication ────────────────────────────── +# Bearer token that bootstraps the admin user and guards the API. +# ⚠️ Production: replace with a strong value, e.g. `openssl rand -hex 16`. +# For SSO instead, see https://linagora.github.io/openrag/documentation/oidc/ +AUTH_TOKEN=or-openrag-1234 +# AUTH_MODE=token # 'token' (default) or 'oidc' for SSO +# SUPER_ADMIN_MODE=true -# RETRIEVER -# RETRIEVER_TOP_K=20 # number of top documents to retrieve, before reranking (lower (~10) is faster on CPU | on GPU, you can try to increase the value (~40) ). +# # FastAPI port (the API and Chainlit chat interface share the same port). +# APP_PORT=8080 -# RERANKER -RERANKER_ENABLED=true # deactivate the reranker if your CPU is not powerful enough -RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base # or jinaai/jina-reranker-v2-base-multilingual +# # Rate limiting, activated by default (recommended). +# RATE_LIMIT_ENABLED=false -# Prompts (templates ship inside the package at openrag/prompts/templates; -# set PROMPTS_DIR only to override with a custom template directory) -# PROMPTS_DIR=/path/to/custom/templates -# Ray -RAY_DEDUP_LOGS=0 # turns off ray log deduplication that appear across multiple processes -RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 # # to enable logs at task level in ray dashboard +# ──────────────── Chainlit: Chat interface ──────────────── +# Session-cookie signing secret — required once auth is enabled; keep it stable. +# ⚠️ Production: replace with a generated value: +# python -c "import secrets; print(secrets.token_urlsafe(32))" +CHAINLIT_AUTH_SECRET=openrag-dev-secret + + +# ──────────────── Admin / Indexer UI (React SPA) ──────────────── +# Host port for the admin UI — the document ingestion, indexing & management +# interface, served at http://:/app/. It also proxies the +# API/auth, so it is the OIDC front door. Zero-config otherwise (same-origin, no +# CORS); VITE_* build-time options are documented in the env vars reference. +# ADMIN_UI_PORT=8081 + + + +# ── Ray (kept as-is by the compose stack; see the docs for what each does) ─── +RAY_DEDUP_LOGS=0 +RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 RAY_task_retry_delay_ms=3000 -RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 # critical with the newest version of UV -# Attach to an external Ray cluster instead of starting an embedded one (disables the local dashboard). -# RAY_ADDRESS=ray://X.X.X.X:10001 -# Interface the embedded Ray dashboard binds to. Defaults to 127.0.0.1 (loopback) because the -# dashboard/job API is unauthenticated (CVE-2023-48022). Set 0.0.0.0 only behind a firewall/auth proxy. -# RAY_DASHBOARD_HOST=127.0.0.1 - -# Admin UI (React SPA served by the admin-ui / nginx container) -# VITE_* settings are baked into the bundle at BUILD time — rebuild with -# `docker compose build admin-ui` after changing them. Replace X.X.X.X with -# localhost (local) or your server IP, and APP_PORT with your FastAPI port. - -# ADMIN_UI_PORT=8081 # Host port for the admin UI (nginx). Default is 8081. -# VITE_API_BASE_URL= # API base baked into the SPA. EMPTY (default) = same-origin via -# # nginx (no CORS). Only set for a browser-direct build. -# VITE_GRAFANA_URL= # Optional Grafana dashboard link on the admin "System" page. -# VITE_APP_NAME=OpenRAG # App display name used in the UI branding. \ No newline at end of file +RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 + +# RAY_memory_monitor_refresh_ms=0 + +# ── Logging (DEBUG on dev, INFO on prod) ── +LOG_LEVEL=DEBUG \ No newline at end of file diff --git a/docs/assets/env_ollama_cpu.env b/docs/assets/env_ollama_cpu.env deleted file mode 100644 index 9d4413cf8..000000000 --- a/docs/assets/env_ollama_cpu.env +++ /dev/null @@ -1,14 +0,0 @@ -# LLM - For conversation -BASE_URL= -API_KEY= -MODEL= - -# VLM - For image interpretation -VLM_BASE_URL= -VLM_API_KEY= -VLM_MODEL= - -# EMBEDDER - For text vectorization -EMBEDDER_BASE_URL= -EMBEDDER_MODEL_NAME= -EMBEDDER_API_KEY= \ No newline at end of file diff --git a/docs/content/docs/documentation/API.mdx b/docs/content/docs/documentation/API.mdx index a3a424fb4..b2fdd675b 100644 --- a/docs/content/docs/documentation/API.mdx +++ b/docs/content/docs/documentation/API.mdx @@ -266,26 +266,136 @@ Search within a particular file in a partition. ### 📄 Document Extraction +Extracts are the individual chunks a document is split into during indexing. Each has a stable `extract_id`, surfaced as the `link` on search results and on the file/chunk-listing endpoints below. + #### Get Extract Details ```http GET /extract/{extract_id} ``` -Retrieve specific document extract (chunk) by ID. +Retrieve a specific document extract (chunk) by its ID. + +**Parameters:** +- `extract_id` (path): The unique chunk identifier (from search or chunk-listing results) -**Response:** JSON containing extract content and metadata +**Permissions:** Requires access to the partition containing the chunk — regular users are limited to their assigned partitions; admins can read any chunk. + +**Response:** `200 OK` +```json +{ + "page_content": "The text content of the chunk…", + "metadata": { + "file_id": "doc-a-id", + "filename": "Document A.pdf", + "partition": "my_partition", + "page": 3, + "indexed_at": "2026-01-01T12:00:00Z" + } +} +``` + +**Errors:** +- `403 Forbidden`: You don't have access to the chunk's partition +- `404 Not Found`: No extract with that ID --- ### Partitions & files Management +Partitions are the multi-tenant document collections OpenRAG indexes into. All routes below are prefixed with **`/partition`**. Access is role-based (hierarchy `owner` > `editor` > `viewer`): **viewer** for reads, **owner** for partition deletion, config changes, and member management. Admins with `SUPER_ADMIN_MODE=true` bypass membership checks. + +#### List Partitions +```http +GET /partition/ +``` +List the partitions you can access — admins see all partitions, regular users see only their memberships. Each entry includes `partition`, `document_count`, and (for non-admins) your `role`. + +#### Create Partition +```http +POST /partition/{partition} +``` +Create an empty partition; you automatically become its **owner**. Returns `201 Created`, or `409 Conflict` if the name is taken. Non-admins are capped by [`MAX_PARTITIONS_PER_USER`](/openrag/documentation/env_vars/) (a `403` is returned when the cap is reached). + +#### Delete Partition +```http +DELETE /partition/{partition} +``` +Permanently delete a partition and **all** its files and chunks. **Owner** only. Returns `204 No Content`. This cannot be undone. + +#### List Files in a Partition +```http +GET /partition/{partition} +``` +**Viewer**+. **Query:** `limit` (optional). Returns `{ "files": [ { "file_id", "filename", "link", … } ] }`, where `link` points at the file-detail endpoint below. + +#### Get File Details & Chunks +```http +GET /partition/{partition}/file/{file_id} +``` +**Viewer**+. **Query:** `limit` (max chunks, default `2000`). Returns `{ "metadata": {…}, "documents": [ { "link": "…/extract/{id}" } ] }`. Returns `404` if the file isn't in the partition. + +#### List Chunks in a Partition +```http +GET /partition/{partition}/chunks +``` +List document chunks (extracts) in a partition. **Viewer**+. + +**Query Parameters:** +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `include_embedding` | boolean | `true` | Include each chunk's vector embedding | +| `file_id` | string | None | Restrict to a single file's chunks (recommended for the document-detail view) | +| `limit` | integer | unbounded | Max chunks to return | + +Returns `{ "chunks": [ { "content", "metadata", "link", "embedding"? } ] }`. Note the chunk text is under `content` here, whereas the single-chunk `GET /extract/{extract_id}` endpoint returns it under `page_content`. + +:::caution +Without `file_id` or `limit` this can return a large amount of data for partitions with many documents. +::: + +#### Partition Pipeline Config + +Each partition references an **indexation** and a **retrieval** [preset](#-pipeline-presets), plus an embedder and chat LLM. + +* Get resolved config +```http +GET /partition/{partition}/config +``` +**Viewer**+. Returns the partition's preset references and the fully resolved indexation/retrieval pipeline configuration. + +* Update config +```http +PATCH /partition/{partition} +``` +**Owner** only. Body fields (all optional): `description`, `embedder`, `indexation_preset`, `retrieval_preset`, `chat_history_depth`, `chat_llm`. Returns the updated resolved config. + +```bash frame="none" +curl -X PATCH http://localhost:8080/partition/my_partition \ + -H "Authorization: Bearer YOUR_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"indexation_preset": "legal", "retrieval_preset": "hyde"}' +``` + +#### Partition Members + +Manage who can access a partition and with which role — all **owner** only. + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/partition/{partition}/users` | GET | List members → `{ "members": [...] }` | +| `/partition/{partition}/users` | POST | Add a member — form fields `user_id` (int), `role` (default `viewer`) → `201` | +| `/partition/{partition}/users/{user_id}` | PATCH | Update a member's role — form field `role` → `200` | +| `/partition/{partition}/users/{user_id}` | DELETE | Remove a member → `204` | + +#### Document Relationships + * Get Files by Relationship ```http -GET /{partition}/relationships/{relationship_id} +GET /partition/{partition}/relationships/{relationship_id} ``` -Returns all files sharing the same `relationship_id` within a partition. +Returns all files sharing the same `relationship_id` within a partition. **Viewer**+. **Parameters:** - `partition` — partition name @@ -314,10 +424,10 @@ Returns all files sharing the same `relationship_id` within a partition. * Get File Ancestors ```http -GET /{partition}/file/{file_id}/ancestors +GET /partition/{partition}/file/{file_id}/ancestors ``` -Returns the complete ancestor path from root to the specified file. +Returns the complete ancestor path from root to the specified file. **Viewer**+. :::note Returns only the direct ancestor path, not sibling branches. @@ -353,6 +463,108 @@ Returns only the direct ancestor path, not sibling branches. --- +### 🧩 Pipeline Presets + +Named, reusable **indexation** and **retrieval** pipeline configurations. Partitions reference a preset by name (see [`PATCH /partition/{partition}`](#partition-pipeline-config)) instead of carrying an inline config, so a change to a preset propagates to every partition using it. Six defaults are seeded on first boot (`default`, `legal`, `finance` for indexation; `default`, `multiquery`, `hyde` for retrieval); the `default` preset of each type cannot be deleted or renamed. + +All routes are prefixed with **`/presets`** and require the **admin** role. `preset_type` is one of `indexation` | `retrieval`. + +#### List available strategy options +```http +GET /presets/options +``` +Returns the choices valid inside a preset `config`: `chunking_strategies`, `parsing_strategies` (`pymupdf`, `marker`, `docling`), `retrieval_types`, and `reranker_providers`. + +#### Create a preset +```http +POST /presets/ +``` +**Body:** `name` (string), `preset_type` (`indexation` | `retrieval`), `config` (object — its keys depend on the type). Returns `201 Created` with the stored preset. + +```bash frame="none" title="Create a retrieval preset" +curl -X POST http://localhost:8080/presets/ \ + -H "Authorization: Bearer YOUR_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "hyde-large", + "preset_type": "retrieval", + "config": {"type": "hyde", "top_k": 50, "top_n": 10} + }' +``` + +An indexation `config` instead accepts keys such as `chunking` (`{name, chunk_size, chunk_overlap_rate}`), `parsing_strategy`, `enable_image_captioning`, `enable_contextualization`, and `contextualization_mode`. + +#### List presets +```http +GET /presets/ +``` +**Query:** `preset_type` (optional) to filter by type. Returns a list of presets with `name`, `preset_type`, `config`, `created_at`, `updated_at`. + +#### Get / Update / Delete a preset +```http +GET /presets/{preset_type}/{name} +PUT /presets/{preset_type}/{name} +DELETE /presets/{preset_type}/{name} +``` +`PUT` accepts a partial body (`name` to rename and/or `config`; at least one required) and returns the updated preset. `DELETE` returns `204 No Content`. + +--- + +### 🔌 Model Endpoints + +A registry of named inference endpoints (embedder, reranker, LLM, VLM) that partitions and presets can point at, so operators can manage and switch inference backends at runtime instead of via `.env`. Stored API keys are **redacted** in every response and only returned through the explicit reveal action below. + +All routes are prefixed with **`/model-endpoints`** and require the **admin** role. `model_type` is one of `embedder` | `reranker` | `llm` | `vlm`. + +#### Register an endpoint +```http +POST /model-endpoints/ +``` +**Body:** `name`, `model_type`, `endpoint` (URL), `model_name` (optional), `batch_size` (default `32`), `timeout` (seconds, default `30`), `extra` (object — put `api_key` here), `is_default` (default `false`). Returns `201 Created`; the response carries `has_api_key` rather than the key itself. + +```bash frame="none" +curl -X POST http://localhost:8080/model-endpoints/ \ + -H "Authorization: Bearer YOUR_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "prod-embedder", + "model_type": "embedder", + "endpoint": "https://vllm.internal/v1", + "model_name": "jinaai/jina-embeddings-v3", + "extra": {"api_key": "sk-…"} + }' +``` + +#### List / Get / Update / Delete +```http +GET /model-endpoints/ # ?model_type= to filter +GET /model-endpoints/{model_type}/{name} +PUT /model-endpoints/{model_type}/{name} +DELETE /model-endpoints/{model_type}/{name} # 204 No Content +``` +`PUT` takes a partial body (any of the create fields, plus `name` to rename) and returns the updated endpoint. + +#### Set as default for its type +```http +POST /model-endpoints/{model_type}/{name}/set-default +``` +Promotes the endpoint to the default used for its `model_type`, and returns it. + +#### Reveal the stored API key +```http +POST /model-endpoints/{model_type}/{name}/reveal-api-key +``` +Explicit admin action that returns `{ "api_key": "…" }` (or `null` if none is stored). This is the only endpoint that returns the key in clear text; the action is logged. + +#### Validate connectivity +```http +POST /model-endpoints/validate # probe a draft (unsaved) endpoint +POST /model-endpoints/{model_type}/{name}/validate # probe a registered endpoint +``` +Both probe the target for reachability and model availability, returning `{ "reachable", "model_found", "models_served", "detail" }`. The draft form takes `endpoint` (+ optional `model_name`, `api_key`); to reuse a saved key without resending it, pass `stored_api_key_model_type` and `stored_api_key_name` (both required together, and only accepted when the draft `endpoint` matches the saved one). + +--- + ### 💬 OpenAI-Compatible Chat These endpoints provide full OpenAI API compatibility for seamless integration with existing tools and workflows. For detailed example of openai usage [see this section](#example-openai-client-usage) diff --git a/docs/content/docs/documentation/env_vars.md b/docs/content/docs/documentation/env_vars.md index a35ea9e37..047e5a606 100644 --- a/docs/content/docs/documentation/env_vars.md +++ b/docs/content/docs/documentation/env_vars.md @@ -22,15 +22,15 @@ Openrag loads all files into a pivot markdown file format before proceeding to c | `IMAGE_CAPTIONING_URL` | `bool` | `true` | If `true`, HTTP/HTTPS image URLs in markdown files are fetched and described by the VLM. | | `SAVE_MARKDOWN` | `bool` | `false` | If `true`, the pivot-format markdown produced during parsing is saved. Useful for debugging and verifying the correctness of the generated markdown. | |`SAVE_UPLOADED_FILES`|`bool`|`false`| When `true`, uploaded files are stored on disk. You must enable this option if you want Chainlit to show sources while chatting.| -| `PDFLoader` | `str` | `MarkerLoader` | Specifies the PDF parsing engine to use. Available options: `PyMuPDFLoader`, `MarkerLoader` and `DotsOCRLoader`.| +| `PDFLOADER` | `str` | `PyMuPDFLoader` | PDF parsing engine. `PyMuPDFLoader` (default) is a lightweight, fast, CPU-friendly backend for searchable PDFs. Switch to `MarkerLoader` for OCR / scanned documents, complex layouts and embedded images (heavier; GPU-friendly). Other options: `DoclingLoader`, `DotsOCRLoader`.| :::caution -`PyMuPDFLoader` is a lightweight pdf loader that cannot process non-searchable (image-based) PDFs and does not extract or handle embedded images. +`PyMuPDFLoader` (the default) is a lightweight PDF loader that cannot process non-searchable (image-based) PDFs and does not extract or handle embedded images. Set `PDFLOADER=MarkerLoader` when you need those. ::: #### PDF Loader ##### Marker Loader Configuration -The `MarkerLoader` is the default PDF parsing engine. It can be configured using the following environment variables: +These settings apply when `MarkerLoader` is selected (`PDFLOADER=MarkerLoader`; the default is `PyMuPDFLoader`). It can be configured using the following environment variables: | Variable | Type | Default | Description | |----------|------|---------|-------------| @@ -87,7 +87,7 @@ For local whisper loader, here are the options to use | Variable | Type | Default | Description | |----------|------|---------|-------------| | `WHISPER_MODEL` | `str` | `base` | The whisper multilingual model to use depending on [available resources](https://github.com/openai/whisper?tab=readme-ov-file#available-models-and-languages). Other options: `base`, `small`, `large`, `large-v3`, etc. | -|`WHISPER_N_WORKERS`| `int` | 3 | Number of whisper workers| +|`WHISPER_N_WORKERS`| `int` | 2 | Number of whisper workers| | `WHISPER_CONCURRENCY_PER_WORKER` | `int` | 2 | Maximum number of audio transcription tasks processed concurrently by each Whisper worker. | ##### OpenAI-compatible audio Loader ( `OpenAIAudioLoader` ) @@ -203,6 +203,15 @@ The PostgreSQL database is configured using the following environment variables: | `POSTGRES_POOL_MAX_SIZE` | int | 20 | Maximum size of the async PostgreSQL connection pool. | | `POSTGRES_COMMAND_TIMEOUT` | int | 30 | Timeout in seconds for PostgreSQL commands issued through the async pool. | +* **`Object Storage (MinIO)`** + +Milvus stores its data in a MinIO object store, whose credentials are **required (no default)** — the compose stack refuses to start if they are unset. Generate strong random values (e.g. `openssl rand -hex 16`). The same values are shared between the `minio` service and Milvus, so both sides must match. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `MINIO_ACCESS_KEY` | str | _(required)_ | MinIO access key, shared by the `minio` service and Milvus. No default. | +| `MINIO_SECRET_KEY` | str | _(required)_ | MinIO secret key, shared by the `minio` service and Milvus. No default. | + ### Compose Storage Volumes The main Docker Compose stack keeps the historical host-path defaults. Set these variables when you want to move state elsewhere, including Docker named volumes. @@ -419,7 +428,7 @@ For multi-node distributed deployments, see [Distributed Deployment in a Ray Clu | `RAY_SERVE_NUM_REPLICAS` | int | 1 | Number of service replicas for load balancing | | `RAY_SERVE_HOST` | str | 0.0.0.0 | Host address for the Ray Serve deployment | | `RAY_SERVE_PORT` | int | 8080 | Port for the Ray Serve FastAPI endpoint | -| `CHAINLIT_PORT` | int | 8090 | Port for the Chainlit UI interface if ray serve is enable `ENABLE_RAY_SERVE`. If not chainlit UI is simply a subroute (`/chainlit` [see this](/openrag/getting_started/usage/#default-ports)) of the FastAPI **`base_url`**| +| `CHAINLIT_PORT` | int | 8090 | Port for the Chainlit UI interface if ray serve is enable `ENABLE_RAY_SERVE`. If not chainlit UI is simply a subroute (`/chainlit` [see this](/openrag/getting_started/quickstart/#default-ports)) of the FastAPI **`base_url`**| ### Web Search Configuration @@ -510,18 +519,49 @@ The following environment variables configure the FastAPI server and control acc | `DEFAULT_FILE_QUOTA` | `int` | `-1` | Default per-user file quota. `<0` disables quotas globally; `>=0` sets the default limit when a user has no explicit quota. | | `PREFERRED_URL_SCHEME` | `string` | `null` | URL scheme (`http` or `https`) used when generating URLs in API responses (e.g., `task_status_url`). When running behind a reverse proxy that terminates SSL, set this to `https` to ensure generated URLs use the correct scheme. If unset, the scheme from the incoming request is used. | | `CORS_EXTRA_ORIGINS` | `string` | _(unset)_ | Semicolon-separated list of additional origins allowed by CORS (e.g. `https://app.example.com;https://other.example.com`). Extends the default list without replacing it. | +| `UVICORN_FORWARDED_ALLOW_IPS` | `string` | `127.0.0.1` | Comma-separated CIDRs/IPs (or `*`) whose `X-Forwarded-*` headers uvicorn trusts. **Required when OpenRAG runs behind a TLS-terminating reverse proxy that lives outside loopback** (typical docker-compose / k8s); otherwise `X-Forwarded-Proto` is dropped and OIDC cookies ship with `Secure=False` even over HTTPS. | +| `MAX_UPLOAD_SIZE_MB` | `int` | `1024` | Maximum accepted upload size, in MB. `0` or a negative value means unlimited. | +| `MAX_PARTITIONS_PER_USER` | `int` | `100` | Maximum number of partitions a non-admin user may own. `-1` disables the cap (unlimited). Admin users always bypass it. | +| `APP_UID` | `int` | `1000` | UID the API container drops to before running the app. Override when your host user is not UID 1000 and bind-mounted folders (`data/`, `logs/`) would otherwise not be writable by the container user. | :::caution[Security Notice] Always set a strong **`AUTH_TOKEN`** in production environments. Never leave it empty or use default values in production deployments. ::: +### Rate Limiting + +Per-identity request rate limiting, tiered by path prefix. Requests are keyed on the authenticated user id, falling back to the client IP for unauthenticated paths (`/auth/*`). **Admin users bypass rate limiting entirely.** Limits use a moving window and are enforced **per worker/replica** — front OpenRAG with shared storage (e.g. Redis) if you scale out and need a global budget. Exceeding a limit returns **429** with a `Retry-After` header. + +Limit values use the `/` format from the [`limits`](https://limits.readthedocs.io/) library (e.g. `120/minute`, `10/second`). + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `RATE_LIMIT_ENABLED` | `bool` | `true` | Master switch for request rate limiting. When `false`, no limits are applied and malformed limit values are ignored. | +| `RATE_LIMIT_DEFAULT` | `str` | `600/minute` | Limit applied to every path except the tiers below. | +| `RATE_LIMIT_AUTH` | `str` | `60/minute` | Limit for `/auth/*` (login/callback/logout). Keyed on client IP because callers are unauthenticated there — keep it high enough that a shared corporate/NAT egress IP does not throttle a legitimate login rush. | +| `RATE_LIMIT_CHAT` | `str` | `120/minute` | Limit for `/v1/*` (chat completions, tools). | + ### Admin UI -The admin UI is a React SPA served by the `admin-ui` (nginx) container. Every -`VITE_*` setting is **baked into the bundle at build time** — Vite inlines them -when the image is built, so they are *not* read at container runtime. After -changing one, rebuild the image: `docker compose build admin-ui`. +The admin UI is a React SPA (the document ingestion, indexing & management interface) served by the `admin-ui` (nginx) container. Every `VITE_*` setting is **baked into the bundle at build time** — Vite inlines them when the image is built, so they are *not* read at container runtime. After changing one, rebuild the image: `docker compose build admin-ui`. + +**How the UI reaches the API (same-origin).** The browser only ever talks to a single origin — `http://:ADMIN_UI_PORT`. nginx inside the `admin-ui` container serves the static SPA under `/app/` and reverse-proxies every other path (`/v1`, `/auth`, `/chainlit`, `/indexer`, …) to the API at `openrag:8080` over the Docker network. Because the bundle is built with `VITE_API_BASE_URL=""`, its API calls are **relative**, so they land back on that same origin — there is **no CORS**, and the OIDC `openrag_session` cookie is first-party. You don't even need to publish the API's own `APP_PORT` to the host; the UI reaches the backend internally over the compose network. Set `VITE_API_BASE_URL` only for a *browser-direct* build, where the SPA calls the API on a different origin — then also add that origin to `CORS_EXTRA_ORIGINS`. + +```mermaid +flowchart TD + B["Browser — single origin
http://HOST:ADMIN_UI_PORT"] + subgraph AUC["admin-ui container"] + N{"nginx :8080
route by path"} + SPA["Static SPA files
(/app/*)"] + end + API["openrag:8080
API service · Docker network"] + + B -->|"GET /app/ (page load)"| N + B -->|"fetch /v1, /auth, /users, /indexer …
relative → same origin, no CORS"| N + N -->|"/app/*"| SPA + N -->|"everything else"| API +``` | Variable | Type | Default | Description | |----------|------|---------|-------------| @@ -541,3 +581,19 @@ changing one, rebuild the image: `docker compose build admin-ui`. | Variable | Type | Default | Description | |----------|------|---------|-------------| | `DEFAULT_LANGUAGE` | `str` | `` | UI language for Chainlit and the Admin UI (e.g. `en-US`, `fr`). When unset, the browser language is used, with `en-US` as the final fallback. | + +### MCP Server (Model Context Protocol) + +OpenRAG ships a standalone [Model Context Protocol](https://modelcontextprotocol.io/) server (`openrag/api/mcp/server.py`) that exposes retrieval to MCP clients. It runs as its own process (not part of the default compose stack). These variables configure the FastMCP transport binding and the search-tool defaults/bounds applied before a request reaches the retrieval service. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `OPENRAG_MCP_SERVER_NAME` | `str` | `OpenRAG MCP` | Display name advertised by the MCP server. | +| `OPENRAG_MCP_HOST` | `str` | `0.0.0.0` | Interface the MCP server binds to. | +| `OPENRAG_MCP_PORT` | `int` | `8081` | Port the MCP server listens on. | +| `OPENRAG_MCP_PATH` | `str` | `/mcp` | HTTP path the MCP endpoint is served under. | +| `OPENRAG_MCP_DEFAULT_TOP_K` | `int` | `5` | Number of chunks the search tool returns when the caller doesn't specify `top_k`. | +| `OPENRAG_MCP_MAX_TOP_K` | `int` | `50` | Upper bound clamped on a caller-supplied `top_k`. | +| `OPENRAG_MCP_SIMILARITY_THRESHOLD` | `float` | `0.8` | Minimum similarity score for a chunk to be returned by the search tool. | +| `OPENRAG_MCP_DOWNLOAD_TIMEOUT` | `float` | `30.0` | Timeout (seconds) for the server-side `index_url` fetch (SSRF/DoS hardening). | +| `OPENRAG_MCP_MAX_DOWNLOAD_BYTES` | `int` | `104857600` | Maximum bytes downloaded by an `index_url` fetch. Default is 100 MiB. | diff --git a/docs/content/docs/getting_started/quickstart.mdx b/docs/content/docs/getting_started/quickstart.mdx index a7059eccf..9a78f3cea 100644 --- a/docs/content/docs/getting_started/quickstart.mdx +++ b/docs/content/docs/getting_started/quickstart.mdx @@ -24,124 +24,116 @@ cd openrag/ git checkout main # or a given release ``` #### 2. Create a `.env` File -Create a `.env` file at the root of the project, mirroring the structure of `.env.example`, to configure your environment and supply blank environment variables. +The Docker Compose stack lives under **`infra/compose/`**, next to its **`.env.example`** template. Copy it to a `.env` in the same folder and fill in the blanks (LLM/VLM endpoints, embedder, and the **required** MinIO / Postgres secrets): -```bash title="Creating the .env file mirroring .env.example" +```bash title="Create your .env from the template" +cd infra/compose cp .env.example .env ``` -Here is a brief overview of key environment variables to configure: - +Here is the minimal set of variables to get started — see the [full environment-variable reference](/openrag/documentation/env_vars/) for every other option: + + #### 3. File Parser configuration -All supported file format parsers are pre-configured. For PDF processing, **[MarkerLoader](https://github.com/datalab-to/marker)** serves as the default parser, offering comprehensive support for OCR-scanned documents, complex layouts, tables, and embedded images. MarkerLoader operates efficiently on both GPU and CPU environments. +All supported file format parsers are pre-configured. For PDF processing, **[PyMuPDFLoader](https://pymupdf.readthedocs.io/)** is the default parser — a lightweight, fast, CPU-friendly engine well suited to searchable PDFs and quick local testing. -:::note -For **`CPU-only deployments`** or lightweight testing scenarios, you can consider switching to **`PyMuPDFLoader`**. To change the loader, set the **`PDFLoader`** variable like this `PDFLoader=PyMuPDFLoader`. :::caution[Important] -These alternative loaders have limitations - they cannot process non-searchable (image-based) PDFs and do not extract or handle embedded images. +`PyMuPDFLoader` cannot process non-searchable (image-based / scanned) PDFs and does not run OCR or extract embedded images. +::: +:::note +For OCR-scanned documents, complex layouts, tables, or embedded images, switch to **[`MarkerLoader`](https://github.com/datalab-to/marker)** (heavier; runs on GPU and CPU) by setting the **`PDFLOADER`** variable: `PDFLOADER=MarkerLoader`. Other options: `DoclingLoader`. ::: -#### 4. Deployment +#### 4. Run OpenRAG + :::tip[Admin UI] -The **admin UI** (a web interface for intuitive document ingestion, indexing, and management) ships bundled as the `admin-ui` service — no separate setup is required. Once the stack is up it is served at `http://localhost:ADMIN_UI_PORT/app/` (default port `8081`). +The **admin UI** — a web interface for document ingestion, indexing, and management — ships bundled as the `admin-ui` service, so there is no separate setup. Once the stack is up it is served at `http://localhost:ADMIN_UI_PORT/app/` (default port `8081`). ::: -##### `Simple and quick` launch for testing - :::info - [OpenRAG repository](https://github.com/linagora/openrag) contains a ready-to-use `docker-compose.yml` file in the **`quick_start` folder**. This setup is ideal for local testing and quick deployments. - - - quick_start - - extern reranker and embedder utils - - vllm cpu dockerfile for different architectures - - Dockerfile.cpu for x86 CPU - - infinity.yaml reranker service - - vdb - - milvus.yaml - - docker-compose.yml - - .env the configured .env file - - ::: - - 1. Navigate to the **`quick_start`** directory or copy it - 2. Place your **`.env`** file in the **`quick_start`** folder - 3. Run the appropriate command for your system: - - - - - GPU deployment, recommended for optimal performance - - ```bash frame="none" {4} - docker compose up -d - - # run the following command to stop the application - # docker compose down - ``` - - - CPU deployment - ```bash frame="none" "--profile cpu" {4} - docker compose --profile cpu up -d - - # to stop the application - # docker compose --profile cpu down - ``` - - - - -##### Development Environment - -For development builds, use the **`--build`** flag to rebuild images: - Execute these commands from the project root directory or the cloned repository: - - - - .github/ - - .hydra_config/ - - ... - - extern/ - - vdb/ - - docker-compose.yml - - README.md +All deployment assets now live under **`infra/`**: + + +- infra/ + - compose/ full stack (recommended) + - docker-compose.yaml - .env.example - - pyproject.toml - - uv.lock - - .env the configured .env file - - - - - GPU deployment - ```bash frame="none" {4} - docker compose up -d - - # run the following command to stop the application - # docker compose down - ``` - - - CPU deployment - ```bash frame="none" "--profile cpu" {4} - docker compose --profile cpu up -d - - # to stop the application - # docker compose --profile cpu down - ``` - - - -Once the app is up and running, you can access the provided services. See the next section. + - .env your configured env + - docker/ Dockerfiles + - ansible/ remote deployment +- openrag/ application code +- conf/ YAML configuration + + +Run the stack from **`infra/compose/`**, where the `.env` you just created lives. Use the **GPU** tab on a machine with an NVIDIA GPU, or the **CPU** tab otherwise. + + + + ```bash frame="none" + cd infra/compose + docker compose up -d + + # stop it later with: + # docker compose down + ``` + + + ```bash frame="none" "--profile cpu" + cd infra/compose + docker compose --profile cpu up -d + + # stop it later with: + # docker compose --profile cpu down + ``` + + + +:::note[Development builds] +Add `--build` to rebuild the images from your working tree: `docker compose up --build -d`. +::: -## Ansible +##### Inference services + +The **LLM** and **VLM** are always **external** OpenAI-compatible endpoints — set `BASE_URL`/`MODEL`/`API_KEY` (and the `VLM_*` equivalents) in `.env`. + +The **embedder** and **reranker**, by contrast, are **bundled**: `docker compose up -d` starts a `vllm` container serving `EMBEDDER_MODEL_NAME` and a reranker container (`RERANKER_PROVIDER`, default Infinity). No extra step is needed to use them. -Clone the OpenRAG repository: ```bash -git clone https://github.com/linagora/openrag.git -cd openrag +cd infra/compose +docker compose up -d # GPU — core stack + bundled embedder & reranker +# docker compose --profile cpu up -d # CPU-only host ``` -Run the provided deployment script and follow the instructions: +To use **external** embedding/reranking instead — reusing existing inference servers and keeping the footprint small: + +- **Embedder** — set `EMBEDDER_BASE_URL` (+ `EMBEDDER_API_KEY`) to your endpoint, then stop the local server by commenting out the `vllm-gpu` / `vllm-cpu` service in `infra/compose/docker-compose.yaml`. +- **Reranker** — set `RERANKER_ENABLED=false` to skip reranking, or point it at an endpoint with `RERANKER_PROVIDER=openai` and `RERANKER_BASE_URL` (+ `RERANKER_API_KEY`); to also stop the container, comment out the `extern/reranker/…` line in the `include:` block. + +Once the app is up and running, you can access the provided services — see [Default ports](#default-ports) below. + +## Ansible + +Clone the repository, then run the deployment script from **`infra/ansible/`** and follow the interactive prompts: + ```bash -./ansible/deploy.sh +git clone --recurse-submodules https://github.com/linagora/openrag.git +cd openrag/infra/ansible +./deploy.sh ``` + +## Default ports + +Once the stack is up, OpenRAG exposes the following services by default: + +| Service | Port | Description | +|-------------------|----------------|----------------------------------------------------------------| +| API Documentation | 8080/docs | Main FastAPI for document ingestion and querying. See [this](/openrag/documentation/api) | +| Chainlit UI | 8080/chainlit | User interface for interacting with the RAG system | +| Ray Dashboard | 8265 | Ray dashboard for monitoring and managing tasks | +| Admin UI | 8081/app/ | Main user interface for indexing and viewing indexed documents | + +:::note[Ray Serve mode] +The table above is for the default (uvicorn) deployment, where the Chainlit UI is the `/chainlit` subroute of the API. When `ENABLE_RAY_SERVE=true`, the API is served by Ray Serve on `RAY_SERVE_PORT` and the **Chainlit UI runs on its own port, `CHAINLIT_PORT`** (default `8090`), instead of the subroute. Uncomment the `${CHAINLIT_PORT}:${CHAINLIT_PORT}` mapping in `infra/compose/docker-compose.yaml` to expose it. See [`CHAINLIT_PORT`](/openrag/documentation/env_vars/#ray-serve-configuration). +::: + +More information about each service is available in its respective documentation page. diff --git a/docs/content/docs/getting_started/quickstart_mac.mdx b/docs/content/docs/getting_started/quickstart_mac.mdx deleted file mode 100644 index 59dcf31bb..000000000 --- a/docs/content/docs/getting_started/quickstart_mac.mdx +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Quickstart on MacOS -description: Get started with a Mac friendly deployment guide ---- - -import { Tabs, TabItem } from '@astrojs/starlight/components'; -import compose_ollama_cpu from '../../../assets/compose_ollama_cpu.yaml?raw'; -import env_ollama_cpu from '../../../assets/env_ollama_cpu.env?raw'; -import { Code } from '@astrojs/starlight/components'; - -:::note -The easiest way to deploy OpenRAG on MacOS is to use Docker. Since Docker for MacOS does not support the MPS backend, perfomance may be limited. See [here](#optimizations) for more details. -::: - -## Docker - -### Prerequisites -- [Docker](https://www.docker.com/get-started) and **Docker Compose** -- Your hardware should meet these specifications: - - A minimum of 24 GB of unified memory (32 GB recommended). 16 GB may work with varying degrees of success. - - An Apple Silicon based Mac - -### Installation - -We provide precompiled Docker images for [OpenRAG](https://hub.docker.com/r/linagoraai/openrag/tags) and its admin UI companion, [openrag-admin-ui](https://hub.docker.com/r/linagoraai/openrag-admin-ui/tags). - -You will need the following `docker-compose.yaml` and `.env` files to get started: - - - - - - - - - - -### Configuration - -By default, the only necessary configuration change is to set the model settings in the `.env` file. Make sure all three models are set (they can be the same one if it supports vision, language, and embedding) If using ollama, ensure you pull the desired models locally using the ollama CLI (keep in mind that ollama needs to be running to pull models): - -```bash title="Pulling models with ollama" -ollama pull qwen3:0.6b -``` -```env title=".env" -BASE_URL=http://ollama:11434 -API_KEY=EMPTY -MODEL=qwen3:0.6b -``` - -:::caution[Important] -Ollama does not support reranker models as of October 2025. Rerankers must be run on a separate server or disabled (as configured by default in the provided `docker-compose.yaml`). -::: - -### Optimizations - -As stated earlier, Docker for MacOS does not support GPU acceleration. Therefore, to maximize performance, we recommend using a non-dockerized installation of ollama or LlamaCpp, or running models from an external server. For simplicity, we still provide a dockerized setup here. \ No newline at end of file diff --git a/docs/content/docs/getting_started/usage.mdx b/docs/content/docs/getting_started/usage.mdx deleted file mode 100644 index a136fbf95..000000000 --- a/docs/content/docs/getting_started/usage.mdx +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Usage ---- - -Once you have installed your OpenRAG instance, you can start using it to upload and query your documents. - -## Default ports - -By default, OpenRAG services are exposed on the following ports: - -| Service | Port | Description | -|-------------------|----------------|----------------------------------------------------------------| -| API Documentation | 8080/docs | Main FastAPI’s for document ingestion and querying. See [this](/openrag/documentation/api)| -| Chainlit UI | 8080/chainlit | User interface for interacting with the RAG system | -| Ray Dashboard | 8265 | Ray dashboard for monitoring and managing tasks | -| Admin UI | 8081/app/ | Main user interface for indexing and viewing indexed documents | - -:::note[Ray Serve mode] -The table above is for the default (uvicorn) deployment, where the Chainlit UI is the `/chainlit` subroute of the API. When `ENABLE_RAY_SERVE=true`, the API is served by Ray Serve on `RAY_SERVE_PORT` and the **Chainlit UI runs on its own port, `CHAINLIT_PORT`** (default `8090`), instead of the subroute. Uncomment the `${CHAINLIT_PORT}:${CHAINLIT_PORT}` mapping in `docker-compose.yaml` to expose it. See [`CHAINLIT_PORT`](/openrag/documentation/env_vars/#ray-serve-configuration). -::: - -More information about the different services can be found in their respective documentation pages. \ No newline at end of file diff --git a/infra/compose/.env.example b/infra/compose/.env.example index 54faabfe1..514aca329 100644 --- a/infra/compose/.env.example +++ b/infra/compose/.env.example @@ -1,227 +1,105 @@ -# LLM +# ============================================================================ +# OpenRAG — minimal .env +# +# Only the variables you must set for the default compose stack to boot are +# listed here. Every other knob (PDF/audio loaders, chunking, retriever, +# reranker, Ray Serve, admin UI, OIDC/SSO, rate limiting, MCP server, web +# search, …) has a sensible default and is documented in full at: +# +# https://linagora.github.io/openrag/documentation/env_vars/ +# ============================================================================ + +# ── LLM (external, OpenAI-compatible) ─────────────────────────────────────── BASE_URL= API_KEY= MODEL= -# Optional: set false for Qwen-style models to suppress reasoning traces. -# Leave unset for Mistral tokenizers. -# LLM_ENABLE_THINKING=false +LLM_SEMAPHORE=10 -# VLM (Visual Language Model) you can set it to the same as LLM if your LLM supports images -VLM_API_KEY= +# ── VLM (vision model, used for image understanding) ──────────────────────── +# Can reuse the LLM values above if that model accepts images. VLM_BASE_URL= +VLM_API_KEY= VLM_MODEL= -# Optional: same behavior as LLM_ENABLE_THINKING for VLM chat templates. -# VLM_ENABLE_THINKING=false - -# OCR VLM loader (DotsOCR/OpenAILoader) thinking control. -# Leave unset for Mistral tokenizers. -# OPENAI_LOADER_ENABLE_THINKING=false - -## FastAPI App (no need to change it) -# APP_PORT=8080 # this is the forwarded port -# Local compose builds run OpenRAG as a non-root user. Override this if your -# host user is not UID 1000 and bind-mounted folders are not writable. -# APP_UID=1000 -# The uvicorn path runs a single worker by design (Ray provides concurrency). -# To scale the HTTP layer, use Ray Serve: ENABLE_RAY_SERVE=true with -# RAY_SERVE_NUM_REPLICAS=N (see the Ray Serve configuration section). - -## To enable API HTTP authentication via HTTPBearer -# AUTH_TOKEN=sk-openrag-1234 -# If AUTH_MODE=token and AUTH_TOKEN is unset, authentication fails closed by -# default. Local open mode requires an explicit opt-in: -# ALLOW_NO_AUTH=true # DEV ONLY — never set this in production -# Optional: semicolon-separated extra allowed origins -# CORS_EXTRA_ORIGINS='https://app.example.com;https://other.example.com' - -# Reverse-proxy trust list for uvicorn. Required when OpenRAG runs behind -# a TLS-terminating proxy that lives outside loopback (typical -# docker-compose / k8s); otherwise X-Forwarded-Proto is dropped and OIDC -# cookies ship with Secure=False even on HTTPS. Accepts a comma-separated -# list of CIDRs / IPs or "*" to trust all peers. Default: 127.0.0.1. -# UVICORN_FORWARDED_ALLOW_IPS=* - -# SAVE_UPLOADED_FILES=true # usefull for chainlit (chat interface) source viewing - -# Maximum accepted upload size in MB (0 or negative = unlimited). Default 1024. -# MAX_UPLOAD_SIZE_MB=1024 - -# Set to true, it will mount chainlit chat ui to the fastapi app (Default: true) -## WITH_CHAINLIT_UI=true - -# UI language for Chainlit and the admin UI (e.g. en-US, fr). -# When unset, the browser language is used, with en-US as the final fallback. -## DEFAULT_LANGUAGE=fr - -# EMBEDDER -EMBEDDER_MODEL_NAME=jinaai/jina-embeddings-v3 # or other embedder from huggingface compatible with vllm +VLM_SEMAPHORE=20 + +# ── Embedder (HuggingFace model served by the bundled vLLM) ───────────────── +# Point these at an external embedding service instead of the bundled vLLM: + +EMBEDDER_MODEL_NAME=jinaai/jina-embeddings-v3 # EMBEDDER_BASE_URL=http://vllm:8000/v1 # EMBEDDER_API_KEY=EMPTY +# MAX_MODEL_LEN=2047 + +# ── Reranker (re-scores retrieved chunks; bundled Infinity server) ─────────── +RERANKER_PROVIDER=infinity # 'infinity' (bundled, default) or 'openai' (external endpoint) +RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base +RERANKER_ENABLED=true + +## Point these at an external reranker instead of the bundled Infinity / VLLM(Openai) server: +# RERANKER_BASE_URL=http://reranker:7997 +# RERANKER_API_KEY=EMPTY + + +# ── PDF parser (default: PyMuPDF — lightweight, CPU-friendly) ──────────────── +# Switch to MarkerLoader for OCR / scanned PDFs, complex layouts & embedded +# images (heavier — more RAM/GPU). Other option: DoclingLoader. +PDFLOADER=PyMuPDFLoader +# Marker tuning (only when PDFLOADER=MarkerLoader): +# MARKER_POOL_SIZE=1 # marker worker actors (≈ 1 per cluster node / Machine). +# MARKER_MAX_PROCESSES=2 # concurrent PDFs per worker (raise with more GPU) + +# ── Image captioning & chunk contextualization (both ON by default) ────────── +# Both run during indexing and call the VLM/LLM — set to false to index +# faster and cheaper (with some retrieval-quality trade-off). +# IMAGE_CAPTIONING=false # stop describing images in documents via the VLM +# CONTEXTUAL_RETRIEVAL=false # stop prepending LLM-generated context to each chunk (Anthropic technique) + +# ── Secrets — dev defaults so the stack boots out of the box ───────────────── +# ⚠️ Production: replace these with strong values, e.g. `openssl rand -hex 16`. +# MINIO_* are shared by the minio service and Milvus — both sides must match. +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_KEY=minioadmin +POSTGRES_PASSWORD=postgres +# POSTGRES_USER=root +# ── API and Chainlit (chat interface) authentication ────────────────────────────── +# Bearer token that bootstraps the admin user and guards the API. +# ⚠️ Production: replace with a strong value, e.g. `openssl rand -hex 16`. +# For SSO instead, see https://linagora.github.io/openrag/documentation/oidc/ +AUTH_TOKEN=or-openrag-1234 +# AUTH_MODE=token # 'token' (default) or 'oidc' for SSO +# SUPER_ADMIN_MODE=true -# RETRIEVER -# RETRIEVER_TOP_K=20 # number of top documents to retrieve, before reranking (lower (~10) is faster on CPU | on GPU, you can try to increase the value (~40) ). +# # FastAPI port (the API and Chainlit chat interface share the same port). +# APP_PORT=8080 -# RERANKER -RERANKER_ENABLED=true # deactivate the reranker if your CPU is not powerful enough -RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base # or jinaai/jina-reranker-v2-base-multilingual +# # Rate limiting, activated by default (recommended). +# RATE_LIMIT_ENABLED=false -# Audio transcription -# Pipe-separated list of file extensions sent directly to the transcription endpoint -# without converting to WAV first. Defaults cover all formats the OpenAI Whisper -# API accepts natively. Override to restrict (e.g. to ".wav" only for vLLM deployments). -# TRANSCRIBER_DIRECT_UPLOAD_SUFFIXES=.wav|.flac|.ogg|.mp3|.mp4|.m4a|.webm|.mpeg|.mpga -# Object storage (MinIO, used by Milvus) — REQUIRED, no default. -# Generate strong random values, e.g. `openssl rand -hex 16`. These are shared -# between the minio service and Milvus; both must match. -MINIO_ACCESS_KEY= -MINIO_SECRET_KEY= +# ──────────────── Chainlit: Chat interface ──────────────── +# Session-cookie signing secret — required once auth is enabled; keep it stable. +# ⚠️ Production: replace with a generated value: +# python -c "import secrets; print(secrets.token_urlsafe(32))" +CHAINLIT_AUTH_SECRET=openrag-dev-secret + + +# ──────────────── Admin / Indexer UI (React SPA) ──────────────── +# Host port for the admin UI — the document ingestion, indexing & management +# interface, served at http://:/app/. It also proxies the +# API/auth, so it is the OIDC front door. Zero-config otherwise (same-origin, no +# CORS); VITE_* build-time options are documented in the env vars reference. +# ADMIN_UI_PORT=8081 -# PostgreSQL — REQUIRED, no default. The compose stacks fail to start if unset. -# Generate a strong value, e.g. `openssl rand -hex 16`. -POSTGRES_PASSWORD= -# POSTGRES_USER=root -# Prompts (templates ship inside the package at openrag/prompts/templates; -# set PROMPTS_DIR only to override with a custom template directory) -# PROMPTS_DIR=/path/to/custom/templates -# Ray -RAY_DEDUP_LOGS=0 # turns off ray log deduplication that appear across multiple processes -RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 # # to enable logs at task level in ray dashboard +# ── Ray (kept as-is by the compose stack; see the docs for what each does) ─── +RAY_DEDUP_LOGS=0 +RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 RAY_task_retry_delay_ms=3000 -RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 # critical with the newest version of UV -# # To disable worker killing +RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 + # RAY_memory_monitor_refresh_ms=0 -# Connect to an external Ray cluster instead of starting an embedded one. -# When set, the app attaches to this cluster and does NOT start a local dashboard -# (the head node owns it). See docs/documentation/deploy_ray_cluster. -# RAY_ADDRESS=ray://X.X.X.X:10001 -# Interface the embedded Ray dashboard binds to. Defaults to 127.0.0.1 (loopback) -# because the dashboard/job API is unauthenticated (CVE-2023-48022). Set to 0.0.0.0 -# only when the port is firewalled or behind an auth proxy. Ignored when RAY_ADDRESS is set. -# RAY_DASHBOARD_HOST=127.0.0.1 - -# Admin UI (React SPA served by the admin-ui / nginx container) -# Zero-config by default: the SPA is SAME-ORIGIN, so nginx reverse-proxies the -# API over the Docker network and the UI works on any host/IP (local or server) -# with NO CORS setup — just set ADMIN_UI_PORT if you want a port other than 8081. -# The VITE_* settings are baked into the bundle at BUILD time, so run -# `docker compose build admin-ui` after changing them. - -# ADMIN_UI_PORT=8081 # Host port for the admin UI (nginx). Serves /app/ and proxies -# # /auth, /v1, … to the backend, so it's the OIDC front door -# # (see OIDC_REDIRECT_URI below). Default is 8081. -# VITE_API_BASE_URL= # API base baked into the SPA. EMPTY (default) = same-origin via -# # nginx (recommended, no CORS). Only set an absolute API URL -# # (e.g. http://X.X.X.X:APP_PORT) for a browser-direct build — then -# # also add the UI's origin to CORS_EXTRA_ORIGINS (above). -# VITE_GRAFANA_URL= # Optional Grafana dashboard link shown on the admin "System" page. -# VITE_APP_NAME=OpenRAG # App display name used in the UI branding. - -# Web Search -# WEBSEARCH_API_TOKEN= # Web search provider API token. If unset, web search is silently disabled. -# WEBSEARCH_BASE_URL=https://api.staan.ai/search/web # Web search provider endpoint -# WEBSEARCH_TOP_K=5 # Number of web results to include (default: 5) -# WEBSEARCH_LANG=fr-FR # Search language/market (default: fr-FR) - -# Max partitions a non-admin user may own (-1 = unlimited; admins bypass). -# MAX_PARTITIONS_PER_USER=100 - -# Rate limiting (per-worker moving window, keyed on user id then client IP). -# Admin users bypass these limits entirely. The /auth/* tier is keyed on client -# IP (callers are unauthenticated there): keep it high enough that a shared -# corporate/NAT egress IP does not throttle a legitimate login rush. -# RATE_LIMIT_ENABLED=true -# RATE_LIMIT_DEFAULT=600/minute # all paths except those below -# RATE_LIMIT_AUTH=60/minute # /auth/* (login/callback/logout) -# RATE_LIMIT_CHAT=120/minute # /v1/* (chat completions, tools) - -# MCP SERVER -# Standalone Model Context Protocol server (openrag/api/mcp/server.py). -# OPENRAG_MCP_SERVER_NAME="OpenRAG MCP" -# OPENRAG_MCP_HOST=0.0.0.0 -# OPENRAG_MCP_PORT=8081 -# OPENRAG_MCP_PATH=/mcp -# OPENRAG_MCP_DEFAULT_TOP_K=5 -# OPENRAG_MCP_MAX_TOP_K=50 -# OPENRAG_MCP_SIMILARITY_THRESHOLD=0.8 -# OPENRAG_MCP_DOWNLOAD_TIMEOUT=30.0 -# OPENRAG_MCP_MAX_DOWNLOAD_BYTES=104857600 # 100 MiB - -# LOGGING -# INFO by default; DEBUG persists user queries and request data to logs. -LOG_LEVEL=INFO # See possible values https://loguru.readthedocs.io/en/stable/api/logger.html - -# SERVER -# Set the preferred URL scheme for generated URLs (e.g., task_status_url). -PREFERRED_URL_SCHEME=https -# ============================================================================ -# Authentication -# ============================================================================ -# AUTH_MODE controls how users authenticate. -# - token (default): Bearer AUTH_TOKEN-based auth (legacy). -# - oidc: OpenID Connect flow (auth code + PKCE). -# AUTH_MODE=token - -# Chainlit session-cookie signing secret. REQUIRED whenever authentication -# is enabled (either AUTH_TOKEN or AUTH_MODE=oidc). Keep it secret and stable -# across restarts. Generate one with: -# python -c "import secrets; print(secrets.token_urlsafe(32))" -# CHAINLIT_AUTH_SECRET= - -# --- OIDC (only required when AUTH_MODE=oidc) --- -# Issuer URL — MUST match EXACTLY the "issuer" field returned by the IdP's -# /.well-known/openid-configuration (trailing slash matters, per OIDC spec). -# Keycloak typically returns no trailing slash; LemonLDAP::NG and Auth0 return one. -# Check with: curl -s /.well-known/openid-configuration | jq -r .issuer -# OIDC_ENDPOINT=https://idp.example.com/realms/openrag -# OIDC_CLIENT_ID=openrag -# OIDC_CLIENT_SECRET=change-me -# Public URL of the FRONT DOOR that serves your UI AND reaches the backend's /auth/callback. -# Must match the IdP client config byte-for-byte. It's also where you land after login (the -# post-login redirect is relative), so pick the origin that serves your UI: -# - Bundled admin-ui (nginx) front door: http://:/auth/callback -# NOT APP_PORT — the bare API has /auth/callback but does NOT serve the /app/ UI, so SSO -# would complete yet silently land you on a blank page. -# - Backend serves the UI itself (Chainlit): http://:/auth/callback -# - Reverse proxy / single hostname (prod): https://openrag.example.com/auth/callback -# OIDC_REDIRECT_URI=https://openrag.example.com/auth/callback -# OIDC_SCOPES=openid email profile offline_access # include offline_access for refresh tokens (persistent sessions) -# Generate a Fernet key once: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" -# OIDC_TOKEN_ENCRYPTION_KEY= -# Where the IdP sends the user after RP-initiated logout. No default — -# pointing this to an OpenRag URL causes an instant re-auth loop. Prefer -# a URL OUTSIDE OpenRag (corporate intranet, static 'bye' page, IdP home). -# OIDC_POST_LOGOUT_REDIRECT_URI=https://intranet.example.com/ -# --- Optional claim mapping --- -# Where to read claims from when OIDC_CLAIM_MAPPING is set. -# 'id_token' (default) — read from the verified ID-token claims (no extra HTTP). -# 'userinfo' — fetch the IdP /userinfo endpoint (more complete, non-standard claims). -# OIDC_CLAIM_SOURCE=id_token -# CSV of `db_field:claim` pairs to sync into the users row on every login. -# Writable fields whitelist: display_name, email (never is_admin / external_user_id / file_quota / token). -# Leave unset to keep user rows untouched after login. -# OIDC_CLAIM_MAPPING=display_name:name,email:email -# --- Optional auto-provisioning on first login --- -# When the IdP returns a `sub` that isn't yet known to OpenRag, the callback -# normally returns 403 "User not registered" — admins must pre-create every -# user. Setting this to true makes the callback create a non-admin user on -# the fly using the ID-token claims (display_name from `name`/`preferred_username`, -# email from `email`). The new user inherits the default file quota. -# OIDC_AUTO_PROVISION_LOGIN=false -# --- Optional group → partition membership sync --- -# Map IdP groups to OpenRag partition memberships on every login. OFF unless -# OIDC_CLAIM_GROUPS names the claim that carries the user's group list (e.g. -# Keycloak's `groups`). Each group is matched against OIDC_GROUP_PATTERN after -# OIDC_GROUP_PREFIX is stripped, yielding (partition, role). Roles are the -# usual viewer/editor/owner. is_admin is NEVER derived from groups. -# /openrag/project-alpha/editor -> partition "project-alpha", role "editor" -# OIDC_CLAIM_GROUPS=groups -# OIDC_GROUP_PREFIX=/openrag/ -# OIDC_GROUP_PATTERN=(.+)/(owner|editor|viewer)$ -# When true, memberships absent from the token are removed on login (the IdP -# becomes the sole source of truth). Default keeps manually-granted memberships. -# OIDC_GROUP_SYNC_PRUNE=false +# ── Logging (DEBUG on dev, INFO on prod) ── +LOG_LEVEL=DEBUG \ No newline at end of file diff --git a/infra/compose/.env.ollama b/infra/compose/.env.ollama deleted file mode 100644 index 66e8f69be..000000000 --- a/infra/compose/.env.ollama +++ /dev/null @@ -1,47 +0,0 @@ -# LLM -BASE_URL=http://host.docker.internal:11434/v1 -API_KEY=ollama -MODEL=qwen2.5:0.5b -LLM_SEMAPHORE=1 -TIMEOUT=300 - -# VLM (Vision support) -VLM_BASE_URL=http://host.docker.internal:11434/v1 -VLM_API_KEY=ollama -VLM_MODEL=qwen3-vl:2b -VLM_SEMAPHORE=1 - -# EMBEDDER -EMBEDDER_MODEL_NAME=dengcao/Qwen3-Embedding-0.6B:Q8_0 -EMBEDDER_BASE_URL=http://host.docker.internal:11434/v1 -EMBEDDER_API_KEY=ollama -MAX_MODEL_LEN=2048 - -# RAG CONFIG -RAG_MODE=SimpleRag -CONTEXTUAL_RETRIEVAL=false -RERANKER_ENABLED=false -MAX_OUTPUT_TOKENS=2048 - -# App Settings -# Docker Compose expands PWD from the shell; customize if your shell does not set it. -SHARED_ENV=${PWD}/.env.ollama -APP_PORT=8002 -CHAINLIT_PORT=8090 -DEFAULT_LANGUAGE=en-US - -# Chainlit conversation history (SQLAlchemy -> existing rdb PostgreSQL) -CHAINLIT_DATABASE_URL=postgresql+asyncpg://root:root_password@rdb:5432/chainlit -DATABASE_URL=postgresql://root:root_password@rdb:5432/chainlit -CHAINLIT_AUTH_SECRET=openrag_local_dev_secret_2026 - -# RAY & System -RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 -RAY_DASHBOARD_PORT=8265 -RAY_memory_usage_threshold=0.99 -RAY_memory_monitor_refresh_ms=0 -SUPER_ADMIN_MODE=true -AUTH_TOKEN=sk-1234 -SAVE_UPLOADED_FILES=true -PDFLOADER=PyMuPDFLoader -WHISPER_N_WORKERS=0 diff --git a/infra/compose/docker-compose.yaml b/infra/compose/docker-compose.yaml index 61b08a979..ff6efb0a7 100644 --- a/infra/compose/docker-compose.yaml +++ b/infra/compose/docker-compose.yaml @@ -46,7 +46,7 @@ x-openrag: &openrag_template # (or a writable host path) override the running code. Uncomment for local # development; keep it off in production. # - ../../openrag:/app/openrag # For dev mode - - /$SHARED_ENV:/ray_mount/.env # Shared environment variables + - ${SHARED_ENV:-./.env}:/ray_mount/.env # Shared environment variables (falls back to ./.env) - ${LOG_VOLUME:-../../logs}:/app/logs ports: - ${APP_PORT:-8080}:${APP_iPORT:-8080} diff --git a/infra/quick_start/docker-compose.yaml b/infra/quick_start/docker-compose.yaml deleted file mode 100644 index db6f20f8c..000000000 --- a/infra/quick_start/docker-compose.yaml +++ /dev/null @@ -1,145 +0,0 @@ -include: - - vdb/milvus.yaml - - extern/infinity.yaml - -x-openrag: &openrag_template - # image: ghcr.io/linagora/openrag:dev-latest - image: linagoraai/openrag:latest - build: - context: . - dockerfile: Dockerfile - volumes: - - ${DATA_VOLUME:-./data}:/app/data - - ${MODEL_WEIGHTS_VOLUME:-~/.cache/huggingface}:/app/model_weights # Model weights for RAG - # - ./openrag:/app/openrag # For dev mode - - /$SHARED_ENV:/ray_mount/.env # Shared environment variables - # - ./logs:/app/logs - ports: - - ${APP_PORT:-8080}:${APP_iPORT:-8080} - - 127.0.0.1:${RAY_DASHBOARD_PORT:-8265}:8265 # Localhost only: Ray dashboard/Jobs API is unauthenticated. Disable when in cluster mode - networks: - default: - aliases: - - openrag - env_file: - - ${SHARED_ENV:-.env} - shm_size: 10.24gb - -x-vllm: &vllm_template - networks: - default: - aliases: - - vllm - restart: on-failure - environment: - - HUGGING_FACE_HUB_TOKEN - ipc: "host" - volumes: - - ${VLLM_CACHE:-/root/.cache/huggingface}:/root/.cache/huggingface # put ./vllm_cache if you want to have the weights on the vllm_cache folder in your project - command: > - --model ${EMBEDDER_MODEL_NAME:-jinaai/jina-embeddings-v3} - --trust-remote-code - --task embed - --gpu_memory_utilization 0.3 - --max-model-len ${MAX_MODEL_LEN:-8192} - # --max-num-seqs 1 - # gpu_memory_utilization, max-num-seqs et max-model-len can be tuned depending on your GPU memory - - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 20s - timeout: 5s - retries: 4 - start_period: 90s - # ports: - # - ${VLLM_PORT:-8000}:8000 -services: - # Admin UI (React SPA + nginx) — same-origin reverse proxy to the API, so the - # UI works out of the box with no CORS/build setup. nginx forwards /v1, /auth, - # … to the `openrag` service on the shared network (resolved at request time, - # so it starts even before the backend is up). Runs under any profile. - admin-ui: - image: linagoraai/openrag-admin-ui:latest - ports: - - "${ADMIN_UI_PORT:-8081}:8080" - restart: unless-stopped - - # GPU - default - openrag: - <<: *openrag_template - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [ gpu ] - profiles: - - '' - depends_on: - rdb: - condition: service_started - milvus: - condition: service_healthy - vllm-gpu: - condition: service_healthy - - # No GPU - openrag-cpu: - <<: *openrag_template - deploy: {} - profiles: - - 'cpu' - depends_on: - rdb: - condition: service_started - milvus: - condition: service_healthy - vllm-cpu: - condition: service_healthy - - rdb: - image: postgres:15 - environment: - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in your .env} - - POSTGRES_USER=${POSTGRES_USER:-root} - volumes: - - ${DB_VOLUME:-./db}:/var/lib/postgresql/data - - vllm-gpu: - <<: *vllm_template - image: vllm/vllm-openai:v0.9.2 - runtime: nvidia - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - profiles: - - '' # Empty string gives default behavior (but does not run when cpu requested) - - vllm-cpu: - <<: *vllm_template - build: - context: extern/vllm - dockerfile: Dockerfile.cpu - target: vllm-openai - image: openrag-vllm-openai-cpu - deploy: {} - environment: - - VLLM_CPU_KVCACHE_SPACE=8 - # Default value isn't sufficient for full context length - command: > - --model ${EMBEDDER_MODEL_NAME:-jinaai/jina-embeddings-v3} - --trust-remote-code - --dtype float32 - --max-model-len ${MAX_MODEL_LEN:-8192} - # --max-num-batched-tokens 32768 - # dtype is required for aarch64 (https://github.com/vllm-project/vllm/issues/11327) and improves speed on amd64. - # max-num-batched-tokens is required for aarch64 because chunked prefill isn't supported by V1 vllm backend - # for aarch64 yet. On aarch64 max-num-batched-tokens must be equal max-model-len for now (without chunked prefill). - # For details see https://github.com/vllm-project/vllm/issues/21179 - profiles: - - 'cpu' \ No newline at end of file diff --git a/infra/quick_start/extern/infinity.yaml b/infra/quick_start/extern/infinity.yaml deleted file mode 100644 index e9efc13d5..000000000 --- a/infra/quick_start/extern/infinity.yaml +++ /dev/null @@ -1,43 +0,0 @@ -x-reranker: &reranker_template - networks: - default: - aliases: - - reranker - volumes: - - ${VLLM_CACHE:-/root/.cache/huggingface}:/app/.cache/huggingface # Model weights for RAG - # ports: - # - ${RERANKER_PORT:-7997}:7997 - -services: - reranker: - <<: *reranker_template - image: michaelf34/infinity - runtime: nvidia - deploy: - resources: - reservations: - devices: - - driver: nvidia - count: all - capabilities: [gpu] - command: > - v2 - --model-id ${RERANKER_MODEL:-Alibaba-NLP/gte-multilingual-reranker-base} - --api-key ${RERANKER_API_KEY:-"EMPTY"} - --port 7997 - profiles: - - '' - - reranker-cpu: - <<: *reranker_template - image: michaelf34/infinity:latest-cpu - deploy: {} - command: > - v2 - --engine torch - --model-id ${RERANKER_MODEL:-Alibaba-NLP/gte-multilingual-reranker-base} - --api-key ${RERANKER_API_KEY:-"EMPTY"} - --port 7997 - profiles: - - 'cpu' - diff --git a/infra/quick_start/extern/vllm/Dockerfile.cpu b/infra/quick_start/extern/vllm/Dockerfile.cpu deleted file mode 100644 index 5f59e1563..000000000 --- a/infra/quick_start/extern/vllm/Dockerfile.cpu +++ /dev/null @@ -1,134 +0,0 @@ -# This file is the adaptation of https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.cpu - -# This vLLM Dockerfile is used to construct image that can build and run vLLM on x86 CPU platform. -# -# Build targets: -# vllm-openai (default): used for serving deployment -# vllm-test: used for CI tests -# vllm-dev: used for development -# -# Build arguments: -# PYTHON_VERSION=3.12 (default)|3.11|3.10|3.9 -# VLLM_CPU_DISABLE_AVX512=false (default)|true - -######################### BASE IMAGE ######################### -FROM ubuntu:22.04 AS base - -WORKDIR /workspace/ - -ARG PYTHON_VERSION=3.12 -ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" - -ENV LD_PRELOAD="" - -# Install minimal dependencies and uv -#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ -# --mount=type=cache,target=/var/lib/apt,sharing=locked \ -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends ccache git curl wget ca-certificates \ - gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 \ - && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \ - && curl -LsSf https://astral.sh/uv/install.sh | sh - -ENV CCACHE_DIR=/root/.cache/ccache -ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache - -ENV PATH="/root/.local/bin:$PATH" -ENV VIRTUAL_ENV="/opt/venv" -ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python -RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -ENV UV_HTTP_TIMEOUT=500 - -RUN git clone https://github.com/vllm-project/vllm/ . && git checkout v0.9.2 - -# Install Python dependencies -ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL} -ENV UV_INDEX_STRATEGY="unsafe-best-match" -ENV UV_LINK_MODE="copy" -RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install --upgrade pip && \ - uv pip install -r requirements/cpu.txt - -RUN export TCMALLOC_SO_PATH=$(ldconfig -p | awk 'BEGIN {FS="=>"} /libtcmalloc_minimal.so/ { sub(/^[ \t]+/, "", $2); print $2 }') -ENV LD_PRELOAD="$TCMALLOC_SO_PATH:/opt/venv/lib/libiomp5.so:$LD_PRELOAD" - -RUN echo 'ulimit -c 0' >> ~/.bashrc - -######################### BUILD IMAGE ######################### -FROM base AS vllm-build - -ARG GIT_REPO_CHECK=0 -# Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ... -ARG VLLM_CPU_DISABLE_AVX512 -ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512} - -WORKDIR /workspace/ - -RUN uv pip install -r requirements/cpu-build.txt --torch-backend auto -RUN uv pip install "transformers<4.54.0" # https://github.com/vllm-project/vllm-ascend/issues/2046 - -RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi - -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=cache,target=/root/.cache/ccache \ - VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel - -######################### DEV IMAGE ######################### -FROM vllm-build AS vllm-dev - -WORKDIR /workspace/vllm - -#RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ -# --mount=type=cache,target=/var/lib/apt,sharing=locked \ -RUN apt-get install -y --no-install-recommends vim numactl - -# install development dependencies (for testing) -RUN uv pip install -e tests/vllm_test_utils - -RUN VLLM_TARGET_DEVICE=cpu python3 setup.py develop - -RUN uv pip install -r requirements/dev.txt && \ - pre-commit install --hook-type pre-commit --hook-type commit-msg - -ENTRYPOINT ["bash"] - -######################### TEST IMAGE ######################### -FROM base AS vllm-test - -WORKDIR /workspace/ - -RUN uv pip install -r requirements/test.txt - -RUN --mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \ - uv pip install dist/*.whl - -ADD ./tests/ ./tests/ -ADD ./examples/ ./examples/ -ADD ./benchmarks/ ./benchmarks/ -ADD ./vllm/collect_env.py . - -# install development dependencies (for testing) -RUN uv pip install -e tests/vllm_test_utils - -ENTRYPOINT ["bash"] - -######################### RELEASE IMAGE ######################### -FROM base AS vllm-openai - -WORKDIR /workspace/ - -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=cache,target=/root/.cache/ccache \ - --mount=type=bind,from=vllm-build,src=/workspace/dist,target=dist \ - uv pip install dist/*.whl -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=cache,target=/root/.cache/ccache \ - --mount=type=bind,from=vllm-build,src=/workspace/dist,target=dist \ - uv pip install "transformers<4.54.0" # https://github.com/vllm-project/vllm-ascend/issues/2046 - -WORKDIR / - -ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"] \ No newline at end of file diff --git a/infra/quick_start/vdb/milvus.yaml b/infra/quick_start/vdb/milvus.yaml deleted file mode 100644 index b5743fde5..000000000 --- a/infra/quick_start/vdb/milvus.yaml +++ /dev/null @@ -1,54 +0,0 @@ -services: - etcd: - image: quay.io/coreos/etcd:v3.5.16 - environment: - - ETCD_AUTO_COMPACTION_MODE=revision - - ETCD_AUTO_COMPACTION_RETENTION=1000 - - ETCD_QUOTA_BACKEND_BYTES=4294967296 - - ETCD_SNAPSHOT_COUNT=50000 - volumes: - - ${MILVUS_VOLUME_DIRECTORY:-./volumes}/etcd:/etcd - command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd - healthcheck: - test: ["CMD", "etcdctl", "endpoint", "health"] - interval: 30s - timeout: 20s - retries: 3 - - minio: - image: minio/minio:RELEASE.2023-03-20T20-16-18Z - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in your .env} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in your .env} - volumes: - - ${MILVUS_VOLUME_DIRECTORY:-./volumes}/minio:/minio_data - command: minio server /minio_data --console-address ":9001" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 - - milvus: - image: milvusdb/milvus:v2.5.4 - command: ["milvus", "run", "standalone"] - # Run under Docker's default seccomp profile (do not disable syscall filtering). - environment: - ETCD_ENDPOINTS: etcd:2379 - MINIO_ADDRESS: minio:9000 - # Keep Milvus's MinIO credentials in sync with the minio service above. - MINIO_ACCESS_KEY_ID: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in your .env} - MINIO_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in your .env} - volumes: - - ${MILVUS_VOLUME_DIRECTORY:-./volumes}/milvus:/var/lib/milvus - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] - interval: 30s - start_period: 90s - timeout: 20s - retries: 3 - # ports: - # - "${VDB_PORT:-19530}:${VDB_iPORT:-19530}" - depends_on: - - "etcd" - - "minio" diff --git a/tests/unit/infra/test_admin_ui_compose.py b/tests/unit/infra/test_admin_ui_compose.py index 48a274802..432f697dd 100644 --- a/tests/unit/infra/test_admin_ui_compose.py +++ b/tests/unit/infra/test_admin_ui_compose.py @@ -16,17 +16,6 @@ def test_admin_ui_compose_service_uses_project_scoped_container_name(): assert admin_ui["ports"] == ["${ADMIN_UI_PORT:-8081}:8080"] -def test_ollama_compose_admin_ui_honors_port_override(): - compose_path = Path(__file__).resolve().parents[3] / "docs/assets/compose_ollama_cpu.yaml" - - with compose_path.open(encoding="utf-8") as handle: - compose = yaml.safe_load(handle) - - admin_ui = compose["services"]["admin-ui"] - - assert admin_ui["ports"] == ["${ADMIN_UI_PORT:-8081}:8080"] - - def test_admin_ui_nginx_upload_limit_matches_api_default(): nginx_conf = Path(__file__).resolve().parents[3] / "infra/compose/nginx/openrag-admin.conf" diff --git a/tests/unit/infra/test_compose_storage.py b/tests/unit/infra/test_compose_storage.py index 8f26f61eb..f19261811 100644 --- a/tests/unit/infra/test_compose_storage.py +++ b/tests/unit/infra/test_compose_storage.py @@ -85,30 +85,6 @@ def test_named_volume_profile_is_opt_in() -> None: assert milvus_env["MINIO_SECRET_ACCESS_KEY"] == "${MINIO_SECRET_KEY:?Set MINIO_SECRET_KEY in your .env}" -def test_quick_start_milvus_uses_current_minio_root_env_names() -> None: - quickstart = _load_yaml(ROOT / "infra" / "quick_start" / "vdb" / "milvus.yaml") - - minio_env = quickstart["services"]["minio"]["environment"] - milvus_env = quickstart["services"]["milvus"]["environment"] - - assert "MINIO_ACCESS_KEY" not in minio_env - assert "MINIO_SECRET_KEY" not in minio_env - assert minio_env["MINIO_ROOT_USER"] == "${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in your .env}" - assert minio_env["MINIO_ROOT_PASSWORD"] == "${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in your .env}" - assert milvus_env["MINIO_ACCESS_KEY_ID"] == "${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in your .env}" - assert milvus_env["MINIO_SECRET_ACCESS_KEY"] == "${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in your .env}" - - -def test_ollama_cpu_milvus_uses_matching_minio_credentials() -> None: - compose = _load_yaml(ROOT / "docs" / "assets" / "compose_ollama_cpu.yaml") - - minio_env = compose["services"]["minio"]["environment"] - milvus_env = compose["services"]["milvus"]["environment"] - - assert milvus_env["MINIO_ACCESS_KEY_ID"] == minio_env["MINIO_ACCESS_KEY"] - assert milvus_env["MINIO_SECRET_ACCESS_KEY"] == minio_env["MINIO_SECRET_KEY"] - - def test_model_serving_cache_preserves_host_path_default_with_named_volume_opt_in() -> None: compose = _load_yaml(COMPOSE_DIR / "docker-compose.yaml") infinity = _load_yaml(EXTERN_DIR / "reranker" / "infinity.yaml")