Merge for release v1.1.5

.coderabbit.yaml

@@ -0,0 +1,5 @@
+reviews:
+  auto_review:
+    base_branches:
+      - .*
+  abort_on_close: false

.github/workflows/smoke_test.yaml

@@ -97,7 +97,7 @@ jobs:
         name: Create backup
         run: |
           mkdir ${PWD}/backup
-          docker compose run --rm -v ${PWD}/backup:/backup:rw --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh simplewiki-500" openrag-cpu
+          docker compose --env-file .env run --rm -v ${PWD}/backup:/backup:rw --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh simplewiki-500" openrag-cpu
           wc -l backup/simplewiki-500.openrag
 
       -
@@ -107,12 +107,12 @@ jobs:
       -
         name: Restore from modified backup
         run: |
-          docker compose run --rm -v ${PWD}/backup:/backup:ro --entrypoint "bash /app/openrag/scripts/entrypoint-restore.sh simplewiki-500-2 /backup/tmp.openrag" openrag-cpu
+          docker compose --env-file .env run --rm -v ${PWD}/backup:/backup:ro --entrypoint "bash /app/openrag/scripts/entrypoint-restore.sh simplewiki-500-2 /backup/tmp.openrag" openrag-cpu
 
       -
         name: Create backup again
         run: |
-          docker compose run --rm -v ${PWD}/backup:/backup:rw --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh simplewiki-500-2" openrag-cpu
+          docker compose --env-file .env run --rm -v ${PWD}/backup:/backup:rw --entrypoint "bash /app/openrag/scripts/entrypoint-backup.sh simplewiki-500-2" openrag-cpu
           wc -l backup/simplewiki-500-2.openrag
 
       -

.github/workflows/smoke_test/.env

@@ -26,6 +26,8 @@ CONTEXTUAL_RETRIEVAL=false
 # EMBEDDER
 EMBEDDER_MODEL_NAME=ibm-granite/granite-embedding-small-english-r2
 EMBEDDER_BASE_URL=http://vllm:8000/v1
+# for ibm-granite/granite-embedding-small-english-r2. Adjust accordingly to your embedder model capabilities.
+MAX_MODEL_LEN=8192
 # EMBEDDER_API_KEY=EMPTY
 
 # RERANKER

.hydra_config/config.yaml

@@ -1,6 +1,6 @@
 defaults:
   - _self_ # TODO: Silences the hydra version migration warning (PLEASE REVIEW FOR BREAKING CHANGES)
-  - chunker: ${oc.env:CHUNKER, recursive_splitter} # recursive_splitter, semantic_splitter, markdown_splitter
+  - chunker: ${oc.env:CHUNKER, recursive_splitter} # recursive_splitter
   - retriever: ${oc.env:RETRIEVER_TYPE, single} # single # multiQuery # hyde
   - rag: ChatBotRag
 
@@ -31,6 +31,7 @@ embedder:
   model_name: ${oc.env:EMBEDDER_MODEL_NAME, jinaai/jina-embeddings-v3}
   base_url: ${oc.env:EMBEDDER_BASE_URL, http://vllm:8000/v1}
   api_key: ${oc.env:EMBEDDER_API_KEY, EMPTY}
+  max_model_len: ${oc.decode:${oc.env:MAX_MODEL_LEN, 8192}}
 
 vectordb:
   host: ${oc.env:VDB_HOST, milvus}
@@ -49,7 +50,7 @@ rdb:
 reranker:
   enable: ${oc.decode:${oc.env:RERANKER_ENABLED, true}}
   model_name: ${oc.env:RERANKER_MODEL, Alibaba-NLP/gte-multilingual-reranker-base}
-  top_k: ${oc.decode:${oc.env:RERANKER_TOP_K, 5}} # Number of documents to return after reranking. upgrade to 8 for better results if your llm has a wider context window
+  top_k: ${oc.decode:${oc.env:RERANKER_TOP_K, 10}} # Number of documents to return after reranking. Upgrade for better results if your llm has a wider context window.
   base_url: ${oc.env:RERANKER_BASE_URL, http://reranker:${oc.env:RERANKER_PORT, 7997}}
 
 map_reduce:

.hydra_config/rag/ChatBotRag.yaml

@@ -1,5 +1,3 @@
-# Config for chatbot RAG
-mode: ChatBotRag
-chat_history_depth: 4
-max_contextualized_query_len: 512
-
+defaults:
+  - base
+mode: ChatBotRag

.hydra_config/rag/SimpleRag.yaml

@@ -1,2 +1,3 @@
-mode: SimpleRag
-chat_history_depth: 4
+defaults:
+  - base
+mode: SimpleRag

.hydra_config/rag/base.yaml

@@ -0,0 +1,4 @@
+# Config for chatbot RAG
+mode: ''
+chat_history_depth: 4
+max_contextualized_query_len: 512

.hydra_config/retriever/base.yaml

@@ -1,3 +1,4 @@
 type: ''
 top_k: ${oc.decode:${oc.env:RETRIEVER_TOP_K, 50}} # Number of documents to return before reranking
-similarity_threshold: ${oc.decode:${oc.env:SIMILARITY_THRESHOLD, 0.6}} # Minimum similarity score for document retrieval
+similarity_threshold: ${oc.decode:${oc.env:SIMILARITY_THRESHOLD, 0.6}} # Minimum similarity score for document retrieval
+with_surrounding_chunks: ${oc.decode:${oc.env:WITH_SURROUNDING_CHUNKS, true}} # Whether to include surrounding chunks for each retrieved chunk

docker-compose.yaml

@@ -44,7 +44,7 @@ x-vllm: &vllm_template
     --trust-remote-code
     --task embed
     --gpu_memory_utilization 0.3
-    --max-model-len ${MAX_MODEL_LEN:-8194}
+    --max-model-len ${MAX_MODEL_LEN:-8192}
   # --max-num-seqs 1
   # gpu_memory_utilization, max-num-seqs et max-model-len can be tuned depending on your GPU memory
 
@@ -143,7 +143,7 @@ services:
       --model ${EMBEDDER_MODEL_NAME:-jinaai/jina-embeddings-v3}
       --trust-remote-code
       --dtype float32
-      --max-model-len ${MAX_MODEL_LEN:-8194}
+      --max-model-len ${MAX_MODEL_LEN:-8192}
     #  --max-num-batched-tokens 32768
     # dtype is required for aarch64 (https://github.com/vllm-project/vllm/issues/11327) and improves speed on amd64.
     # max-num-batched-tokens is required for aarch64 because chunked prefill isn't supported by V1 vllm backend

docs/content/docs/documentation/API.mdx

@@ -218,27 +218,31 @@ curl -X POST http://localhost:8080/v1/chat/completions \
         "content": "Your question here"
       }
     ],
-    "temperature": 0.7,
+    "temperature": 0.1,
     "stream": false
   }'
 ```
 
 You can also direclty use this endpoint with no RAG pipeline, i.e. to directly use the LLM.
-For that, simply do not specify any model: 
+For that, instead of using the `openrag` prefix for the model, you can:
+- Specify no model
+- Specify an empty model
+- Specify the openRAG configured model, e.g. `Mistral-Small-3.1-24B-Instruct-2503`.
 
 **Request Body:**
 ```bash frame="none" title="Testing the openai OpenRAG chat completions endpoint with curl"
 curl -X POST http://localhost:8080/v1/chat/completions \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer YOUR_AUTH_TOKEN" \
   -d '{
+    "model": "",
     "messages": [
       {
         "role": "user",
         "content": "Your question here"
       }
     ],
-    "temperature": 0.7,
+    "temperature": 0.1,
     "stream": false
   }'
 ```

docs/content/docs/documentation/backup_restore.md

@@ -10,7 +10,7 @@ It's better to stop `openrag-cpu` (or `openrag`) service before starting backup.
 :::
 
 ```bash
-docker compose \
+docker compose --env-file .env \
     run \
     --build \
     --rm \
@@ -23,7 +23,7 @@ docker compose \
 :::info
 By default backup script creates plan text uncomressed file. To make things faster you can use multithread compressor the following way:
 ```bash
-docker compose \
+docker compose --env-file .env \
     run \
     --build \
     --rm \
@@ -37,7 +37,8 @@ docker compose \
 ## Backup all partitions
 
 ```bash
-docker compose run --build --rm \
+docker compose --env-file .env \
+  run --build --rm \
   -v ~/backup:/backup:rw \
   --entrypoint "uv run /app/openrag/scripts/backup.py -o /backup/test.openrag" \
   openrag
@@ -51,7 +52,7 @@ docker compose run --build --rm \
 Start with dry run to ensure the backup file is correct:
 
 ```bash
-docker compose \
+docker compose --env-file .env \
     run \
     --build \
     --rm \
@@ -62,7 +63,7 @@ docker compose \
 Backup files are expected to be in `/my-backup-dir/`. If the dry run is successful, run the following script to insert the data :
 
 ```bash
-docker compose \
+docker compose --env-file .env \
     run \
     --build \
     --rm \
@@ -74,7 +75,8 @@ docker compose \
 ## Restore all partitions
 
 ```bash
-docker compose run --build --rm \
+docker compose --env-file .env \
+  run --build --rm \
   -v ~/backup:/backup:rw \
   --entrypoint "uv run /app/openrag/scripts/restore.py /backup/test.openrag"\
   openrag

docs/content/docs/documentation/env_vars.md

@@ -96,7 +96,7 @@ As noted in [this PR](https://github.com/linagora/openrag/pull/134), the current
 
 | Variable               | Type | Default              | Description |
 |------------------------|------|----------------------|-------------|
-| `CHUNKER`              | `str`  | recursive_splitter   | Defines the chunking strategy: `recursive_splitter`, `semantic_splitter`, or `markdown_splitter`. |
+| `CHUNKER`              | `str`  | recursive_splitter   | Defines the chunking strategy: `recursive_splitter`. |
 | `CONTEXTUAL_RETRIEVAL` | `bool` | true                 | Enables contextual retrieval to chunk context, a technique introduced by Anthropic to improve retrieval performance ([Contextual Retrieval](https://www.anthropic.com/news/contextual-retrieval)) |
 | `CHUNK_SIZE`           | `int`  | 512                  | Maximum size (in characters) of each chunk. |
 | `CHUNK_OVERLAP_RATE`   | `float`| 0.2                  | Percentage of overlap between consecutive chunks. |
@@ -108,10 +108,6 @@ After files are converted to Markdown, only the **text content** is chunked.
 
 * **`recursive_splitter`**: Uses hierarchical text structure (sections, paragraphs, sentences). Based on [RecursiveCharacterTextSplitter](https://docs.langchain.com/oss/python/integrations/splitters/index#text-structure-based), it preserves natural boundaries whenever possible while ensuring chunks never exceeding the `CHUNK_SIZE`.
 
-* **`markdown_splitter`**: Splits text using Markdown headers, then subdivides sections that exceed `CHUNK_SIZE`.
-
-* **`semantic_splitter`**: Uses embedding-based semantic similarity to create meaning-preserving chunks. Oversized chunks are chunked to be less than `CHUNK_SIZE`.
-
 ### Embedding
 Our embedder is **OpenAI-compatible** and runs on a **VLLM** instance configured with the following variables:
 
@@ -120,6 +116,7 @@ Our embedder is **OpenAI-compatible** and runs on a **VLLM** instance configured
 | `EMBEDDER_MODEL_NAME` | `str` | jinaai/jina-embeddings-v3 | HuggingFace Embedding model served by VLLM .i.e `Qwen/Qwen3-Embedding-0.6B` or `jinaai/jina-embeddings-v3`|
 | `EMBEDDER_BASE_URL` | `str` | http://vllm:8000/v1 | Base URL of the embedder (OpenAI-style).|
 | `EMBEDDER_API_KEY`  | `str` | EMPTY | API key for authenticating embedder calls.|
+| `MAX_MODEL_LEN` | `int` | 8192 | Maximum context length (in tokens) supported by the embedding model. If the chunk exceeds this limit, the embedder will truncate it.|
 
 If you prefer to use an **external embedding service**, simply comment out the embedder service in the [docker-compose.yaml](https://github.com/linagora/openrag/blob/dev/docker-compose.yaml#L117-L153) and provide the variables above in your environment.
 
@@ -188,9 +185,10 @@ The retriever fetches relevant documents from the vector database based on query
 
 | Variable | Type | Default | Description |
 |----------|------|---------|-------------|
+| `RETRIEVER_TYPE` | str | single | Retrieval strategy to use. Options: `single`, `multiQuery`, `hyde` |
 | `RETRIEVER_TOP_K` | int | 50 | Number of documents to retrieve before reranking.|
 | `SIMILARITY_THRESHOLD` | float | 0.6 | Minimum similarity score (0.0-1.0) for document retrieval. Documents below this threshold are filtered out |
-| `RETRIEVER_TYPE` | str | single | Retrieval strategy to use. Options: `single`, `multiQuery`, `hyde` |
+| `WITH_SURROUNDING_CHUNKS` | `bool` | true | When enabled, retrieves adjacent chunks (preceding and following) for each matched document to provide additional context.|
 
 #### Retrieval Strategies
 
@@ -213,7 +211,7 @@ The current Infinity server interface is not OpenAI-compatible, which limits int
 | `RERANKER_ENABLED` | `bool` | true | Enable or disable the reranking mechanism |
 | `RERANKER_MODEL` | `str` | Alibaba-NLP/gte-multilingual-reranker-base | Model used for reranking documents.|
 | `RERANKER_TOP_K` | `int` | 5 | Number of top documents to return after reranking. Increase to 8 for better results if your LLM has a wider context window |
-| `RERANKER_BASE_URL` | `str` | http://reranker:7997 | Base URL of the reranker service |
+| `RERANKER_BASE_URL` | `str` | `http://reranker:7997` | Base URL of the reranker service |
 | `RERANKER_PORT` | `int` | 7997 | Port on which the reranker service listens |
 
 ## Extra

openrag/components/files.py

@@ -1,11 +1,35 @@
+import asyncio
+import re
+import secrets
+import time
 from pathlib import Path
+from typing import Dict, Optional
+
 import aiofiles
-from fastapi import UploadFile
 import consts
-import time
-import secrets
-import asyncio
-from typing import Dict, Optional
+from fastapi import UploadFile
+
+
+def sanitize_filename(filename: str) -> str:
+    # Split filename into name and extension
+    path = Path(filename)
+    name = path.stem
+    ext = path.suffix
+
+    # Remove special characters (keep only word characters and hyphens temporarily)
+    name = re.sub(r"[^\w\-]", "_", name)
+
+    # Replace hyphens with underscores
+    name = name.replace("-", "_")
+
+    # Collapse multiple underscores
+    name = re.sub(r"_+", "_", name)
+
+    # Remove leading/trailing underscores
+    name = name.strip("_")
+
+    # Reconstruct filename
+    return name + ext
 
 
 def make_unique_filename(filename: str) -> Path:
@@ -64,6 +88,5 @@ async def serialize_file(task_id: str, path: str, metadata: Optional[Dict] = {})
         except Exception:
             raise
     else:
-
         ray.cancel(future, recursive=True)
         raise TimeoutError(f"Serialization task {task_id} timed out after seconds")