chore: Use running openRAG for tests

.github/workflows/unit_tests.yml

@@ -17,29 +17,108 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - name: env file
+        run: |
+          cp tests/.env-tests .env
+
+      # Move storage to /mnt to avoid disk space issues
+      - name: Free up space and move Docker storage to /mnt
+        run: |
+          echo "Stopping Docker..."
+          sudo systemctl stop docker
+
+          echo "Creating new Docker root at /mnt/docker..."
+          sudo mkdir -p /mnt/docker
+          sudo rsync -aqxP /var/lib/docker/ /mnt/docker
+
+          echo "Updating Docker daemon config..."
+          echo '{"data-root": "/mnt/docker"}' | sudo tee /etc/docker/daemon.json
+
+          cat /etc/docker/daemon.json
+
+          echo "Restarting Docker..."
+          sudo systemctl start docker
+
+          echo "Verifying Docker root directory:"
+          docker info | grep "Docker Root Dir"
+
+      # Setup docker builds with cache
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build openrag image
+        uses: docker/build-push-action@v6
         with:
-          python-version: ${{ matrix.python-version }}
+          context: .
+          file: ./Dockerfile
+          tags: linagoraai/openrag:latest
+          push: false
+          load: true
+          cache-from: type=gha,scope=openrag
+          cache-to: type=gha,mode=max,scope=openrag
 
-      - name: Install uv
+
+      # Start openRAG CPU
+      - name: Start openRAG CPU
         run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
+          docker compose \
+            -f docker-compose.yaml \
+            -f docker-compose-ci.yaml \
+            --profile cpu up -d \
+            --no-build
+
+      - name: Check containers
+        run: |
+          docker compose -f docker-compose.yaml -f docker-compose-ci.yaml --profile cpu ps
+
+
+      - name: Wait for RAG API
+        run: |
+          echo "Waiting for RAG API..."
+          for i in {1..10}; do
+            if curl -fsS "http://localhost:${APP_PORT}/health_check" > /dev/null 2>&1; then
+              echo "RAG API is up!"
+              exit 0
+            fi
+            sleep 10
+          done
+          docker compose logs
+          exit 1
+
+      # Install uv with cache
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
 
       - name: Cache uv
         uses: actions/cache@v4
         with:
-          path: |
-            ~/.cache/uv
+          path: ~/.cache/uv
           key: uv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml', '**/uv.lock') }}
           restore-keys: |
             uv-${{ runner.os }}-${{ matrix.python-version }}-
 
-      - name: Install dependencies with uv
+      # Install deps for tests
+      - name: Install with test dependencies
         run: |
-          uv sync
+          uv pip install -e ".[test]"
 
+      # Run actual tests
       - name: Run tests
         run: |
-          uv run pytest
+          uv run pytest -vv tests
+
+     # Dump logs on failure
+      - name: Dump logs on failure
+        if: failure()
+        run: |
+          docker compose logs milvus
+          docker compose logs etcd
+          docker compose logs minio
+          docker compose logs openrag-cpu
+
+      # Proper shutdown
+      - name: Shutdown openRAG
+        if: always()
+        run: |
+          docker compose down -v
+

Dockerfile

@@ -18,9 +18,9 @@
     apt install -y ffmpeg 
 
 # Set environment variables for Hugging Face cache location
 ENV XDG_CACHE_HOME=${XDG_CACHE_HOME:-/app/model_weights}
 ENV HF_HOME=${HF_HOME:-/app/model_weights}
 ENV HF_HUB_CACHE=${HF_HUB_CACHE:-/app/model_weights/hub}
 
 # Set workdir for uv
 WORKDIR /app
@@ -30,18 +30,23 @@
 RUN pip3 install uv && \
     uv python install 3.12.7 && \
     uv python pin 3.12.7
-    # && \ uv sync --no-dev
+#   && uv sync --no-dev
+
 COPY entrypoint.sh /app/entrypoint.sh
 RUN chmod +x /app/entrypoint.sh
-# Set workdir for source code
-WORKDIR /app/openrag
 
-# Copy source code
+# Copy source code, assets and config
+# COPY openrag/ /app/openrag
 COPY openrag/ .
-
-# Copy assests & config
 COPY prompts/ /app/prompts/
 COPY .hydra_config/ /app/.hydra_config/
+
 ENV PYTHONPATH=/app/openrag/
 ENV APP_iPORT=${APP_iPORT:-8080}
-ENTRYPOINT ../entrypoint.sh
+
+# Set workdir for source code
+WORKDIR /app/openrag
+
+# ENV UV_PROJECT_ENVIRONMENT=/app/.venv
+
+ENTRYPOINT ../entrypoint.sh

docker-compose-ci.yaml

@@ -0,0 +1,21 @@
+services:
+  vllm-cpu:
+    image: openeuler/vllm-cpu:0.9.1-oe2403lts
+    build: null
+    environment:
+      - VLLM_TARGET_DEVICE=cpu
+      - VLLM_LOGGING_LEVEL=DEBUG
+      - VLLM_CPU_KVCACHE_SPACE=8
+    command: >
+      --model ${EMBEDDER_MODEL_NAME:-jinaai/jina-embeddings-v3}
+      --trust-remote-code
+      --dtype float32
+      --device cpu
+      --max-num-seqs 8
+      --max-model-len ${MAX_MODEL_LEN:-2048}
+    profiles:
+      - "cpu"
+    security_opt:
+      - seccomp=unconfined
+    healthcheck:
+      disable: true

entrypoint.sh

@@ -1,13 +1,18 @@
 #!/bin/bash
+set -e
+
 ENV_ARG=""
 if [[ -n "${SHARED_ENV}" ]]; then
   ENV_ARG="--env-file=${SHARED_ENV}"
 fi
 
+#cd /app/openrag
+#export UV_PROJECT_ENVIRONMENT=/app/.venv
+
 if [[ "${ENABLE_RAY_SERVE}" == "true" ]]; then
   echo "🔁 Starting with Ray Serve..."
   uv run $ENV_ARG api.py
 else
   echo "🚀 Starting with Uvicorn..."
-  uv run --no-dev $ENV_ARG uvicorn api:app --host 0.0.0.0 --port ${APP_iPORT:-8080} --reload --workers ${API_NUM_WORKERS:-1}
+  uv run --no-dev --no-sync $ENV_ARG uvicorn api:app --host 0.0.0.0 --port ${APP_iPORT:-8080} --reload --workers ${API_NUM_WORKERS:-1}
 fi

extern/vllm/Dockerfile.cpu

@@ -41,7 +41,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 ENV UV_HTTP_TIMEOUT=500
 
-RUN git clone https://github.com/vllm-project/vllm/ . && git checkout v0.9.2
+RUN git clone --depth=1 --branch v0.9.2 https://github.com/vllm-project/vllm/ .
 
 # Install Python dependencies 
 ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
@@ -67,8 +67,9 @@ ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
 
 WORKDIR /workspace/
 
-RUN uv pip install -r requirements/cpu-build.txt --torch-backend auto
-RUN uv pip install "transformers<4.54.0" # https://github.com/vllm-project/vllm-ascend/issues/2046
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install -r requirements/cpu-build.txt --torch-backend auto && \
+    uv pip install "transformers<4.54.0" # https://github.com/vllm-project/vllm-ascend/issues/2046
 
 RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
 

pyproject.toml

@@ -54,6 +54,14 @@ dev = [
     "pytest>=8.4.1",
     "pytest-asyncio>=1.3.0",
 ]
+
+[project.optional-dependencies]
+test = [
+  "pytest",
+  "httpx",
+  "pytest-timeout"
+]
+
 lint = [
     "ruff>=0.14.1",
 ]

pytest.ini

@@ -1,6 +1,7 @@
 [pytest]
 testpaths =
   openrag
+  tests
 pythonpath = ./openrag
 python_files =
     test_*.py

tests/.env-tests

@@ -0,0 +1,67 @@
+# LLM
+BASE_URL=http://localhost:8001/
+API_KEY=sk-
+MODEL=HuggingFaceTB/SmolLM2-135M-Instruct
+
+# VLM (Visual Language Model) you can set it to the same as LLM if your LLM supports images
+VLM_BASE_URL=http://localhost:8002/
+VLM_API_KEY=sk-
+VLM_MODEL=HuggingFaceTB/SmolVLM-Instruct
+
+
+## FastAPI App (no need to change it)
+APP_PORT=8080 # this is the forwarded port
+API_NUM_WORKERS=1 # Number of uvicorn workers for the FastAPI app
+
+## To enable API HTTP authentication via HTTPBearer
+# AUTH_TOKEN=sk-openrag-1234
+
+SAVE_UPLOADED_FILES=false
+
+# Set to true, it will mount chainlit chat ui to the fastapi app (Default: true)
+WITH_CHAINLIT_UI=false
+
+# RETRIEVER
+CONTEXTUAL_RETRIEVAL=false
+
+# EMBEDDER
+EMBEDDER_MODEL_NAME=ibm-granite/granite-embedding-small-english-r2
+EMBEDDER_BASE_URL=http://vllm:8000/v1
+# EMBEDDER_API_KEY=EMPTY
+
+# RERANKER
+RERANKER_ENABLED=true
+RERANKER_MODEL=Alibaba-NLP/gte-multilingual-reranker-base # or jinaai/jina-reranker-v2-base-multilingual
+
+# Prompts
+PROMPTS_DIR=../prompts/example1
+
+# Loaders
+PDFLoader=MarkerLoader
+XDG_CACHE_HOME=/app/model_weights
+# If using MarkerLoader
+MARKER_MAX_TASKS_PER_CHILD=1
+MARKER_MAX_PROCESSES=1
+MARKER_MIN_PROCESSES=1
+MARKER_POOL_SIZE=1 # Value au increment if you have a cluster of machines
+MARKER_NUM_GPUS=0.01
+
+# Ray
+RAY_POOL_SIZE=1 # Number of serializer actor instances
+RAY_MAX_TASKS_PER_WORKER=2 # Number of tasks per serializer
+RAY_DEDUP_LOGS=0 # turns off ray log deduplication that appear across multiple processes
+RAY_ENABLE_RECORD_ACTOR_TASK_LOGGING=1 # # to enable logs at task level in ray dashboard
+RAY_task_retry_delay_ms=3000
+RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 # critical with the newest version of UV
+RAY_memory_monitor_refresh_ms=0
+
+# Indexer UI 
+## 1. replace X.X.X.X with localhost if launching local or with your server IP
+## 2. APP_PORT with your FastAPI port (8080 by default)
+## 3. Base URL of the Indexer UI (required to prevent CORS issues). Replace INDEXERUI_PORT with its value
+## 4. Base URL of your FastAPI backend. Used by the frondend. Replace APP_PORT with the actual port number of your FastAPI backend
+
+VITE_INCLUDE_CREDENTIALS=false # set true if fastapi authentification is enabled
+INDEXERUI_PORT=8060 # Port to expose the Indexer UI (default is 3042)
+INDEXERUI_URL='http://X.X.X.X:INDEXERUI_PORT'                 
+VITE_API_BASE_URL='http://X.X.X.X:APP_PORT'

tests/conftest.py

@@ -0,0 +1,29 @@
+# tests/conftest.py
+import os
+import time
+import httpx
+import pytest
+
+
+@pytest.fixture(scope="session")
+def base_url():
+    return os.environ.get("RAG_BASE_URL", "http://localhost:8080")
+
+
+@pytest.fixture(scope="session", autouse=True)
+def wait_for_api(base_url):
+    timeout = 60
+    start = time.time()
+
+    while True:
+        try:
+            r = httpx.get(f"{base_url}/health_check", timeout=3)
+            if r.status_code == 200:
+                return
+        except Exception:
+            pass
+
+        if time.time() - start > timeout:
+            raise RuntimeError("API not ready")
+
+        time.sleep(2)

tests/test_partitions.py

@@ -0,0 +1,12 @@
+# tests/test_partition.py
+import httpx
+import uuid
+
+
+def test_create_partition(base_url):
+    client = httpx.Client(timeout=5)
+
+    partition_name = "tets-partition"
+    r = client.post(f"{base_url}/partition/{partition_name}")
+    assert r.status_code is 200
+

vdb/milvus.yaml

@@ -35,14 +35,15 @@ services:
     security_opt:
     - seccomp:unconfined
     environment:
-      ETCD_ENDPOINTS: etcd:2379
+      ETCD_ENDPOINTS: http://etcd:2379
       MINIO_ADDRESS: minio:9000
+      MINIO_USE_SSL: false
     volumes:
       - ${MILVUS_VOLUME_DIRECTORY:-./volumes}/milvus:/var/lib/milvus
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
       interval: 30s
-      start_period: 90s
+      start_period: 180s
       timeout: 20s
       retries: 3
     # ports: