refactor(phase-7): decompose MilvusDB god object into VectorStore + CatalogStore adapters

.github/workflows/integration_tests.yml

@@ -0,0 +1,84 @@
+name: Integration tests
+
+on:
+  push:
+    branches:
+      - main
+      - dev
+      - "refactor/hexagonal"
+      - "refactor/phase-**"
+  pull_request:
+
+jobs:
+  milvus-integration:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@v1
+        with:
+          version: v2.34.0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Cache uv
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: uv-${{ runner.os }}-3.12-${{ hashFiles('**/pyproject.toml', '**/uv.lock') }}
+          restore-keys: |
+            uv-${{ runner.os }}-3.12-
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Start Milvus stack
+        working-directory: tests/integration
+        run: docker compose up -d --wait --wait-timeout 180
+
+      - name: Verify Milvus reachable from host
+        run: |
+          for i in {1..30}; do
+            if curl -sf http://localhost:9091/healthz 2>/dev/null; then
+              echo "Milvus healthy after $i attempts"
+              break
+            fi
+            echo "Attempt $i/30 - waiting..."
+            sleep 2
+          done
+          curl -sf http://localhost:9091/healthz
+
+      - name: Run integration tests
+        env:
+          OPENRAG_TEST_VDB_HOST: localhost
+          OPENRAG_TEST_VDB_PORT: "19530"
+        run: uv run pytest tests/integration/ -m integration -v --tb=short
+
+      - name: Show logs on failure
+        if: failure()
+        working-directory: tests/integration
+        run: |
+          echo "=== Milvus Logs ==="
+          docker compose logs milvus --tail=200
+          echo "=== etcd Logs ==="
+          docker compose logs etcd --tail=50
+          echo "=== minio Logs ==="
+          docker compose logs minio --tail=50
+
+      - name: Cleanup
+        if: always()
+        working-directory: tests/integration
+        run: docker compose down -v

docs/content/docs/documentation/milvus_migration.mdx

@@ -139,13 +139,13 @@ docker compose ps milvus
   <TabItem label="GPU">
 ```bash
 docker compose run --no-deps --rm --build --entrypoint "" openrag \
-    uv run python scripts/migrations/milvus/migrate.py --dry-run
+    uv run python services/persistence/migrations/milvus/migrate.py --dry-run
 ```
   </TabItem>
   <TabItem label="CPU">
 ```bash
 docker compose --profile cpu run --no-deps --rm --build --entrypoint "" openrag-cpu \
-    uv run python scripts/migrations/milvus/migrate.py --dry-run
+    uv run python services/persistence/migrations/milvus/migrate.py --dry-run
 ```
   </TabItem>
 </Tabs>
@@ -158,13 +158,13 @@ Review the output to confirm which migrations are pending and what changes they
   <TabItem label="GPU">
 ```bash
 docker compose run --no-deps --rm --entrypoint "" openrag \
-    uv run python scripts/migrations/milvus/migrate.py
+    uv run python services/persistence/migrations/milvus/migrate.py
 ```
   </TabItem>
   <TabItem label="CPU">
 ```bash
 docker compose --profile cpu run --no-deps --rm --entrypoint "" openrag-cpu \
-    uv run python scripts/migrations/milvus/migrate.py
+    uv run python services/persistence/migrations/milvus/migrate.py
 ```
   </TabItem>
 </Tabs>
@@ -198,22 +198,22 @@ To upgrade or downgrade to a specific schema version rather than the latest:
 ```bash
 # Upgrade to version 2 only
 docker compose run --no-deps --rm --entrypoint "" openrag \
-    uv run python scripts/migrations/milvus/migrate.py --target 2
+    uv run python services/persistence/migrations/milvus/migrate.py --target 2
 
 # Downgrade to version 0 (resets version stamp and drops indexes)
 docker compose run --no-deps --rm --entrypoint "" openrag \
-    uv run python scripts/migrations/milvus/migrate.py --downgrade --target 0
+    uv run python services/persistence/migrations/milvus/migrate.py --downgrade --target 0
 ```
   </TabItem>
   <TabItem label="CPU">
 ```bash
 # Upgrade to version 2 only
 docker compose --profile cpu run --no-deps --rm --entrypoint "" openrag-cpu \
-    uv run python scripts/migrations/milvus/migrate.py --target 2
+    uv run python services/persistence/migrations/milvus/migrate.py --target 2
 
 # Downgrade to version 0 (resets version stamp and drops indexes)
 docker compose --profile cpu run --no-deps --rm --entrypoint "" openrag-cpu \
-    uv run python scripts/migrations/milvus/migrate.py --downgrade --target 0
+    uv run python services/persistence/migrations/milvus/migrate.py --downgrade --target 0
 ```
   </TabItem>
 </Tabs>
@@ -226,13 +226,13 @@ Milvus does not support dropping fields. A downgrade only removes indexes and re
   <TabItem label="GPU">
 ```bash
 docker compose run --no-deps --rm --entrypoint "" openrag \
-    uv run python scripts/migrations/milvus/migrate.py --downgrade
+    uv run python services/persistence/migrations/milvus/migrate.py --downgrade
 ```
   </TabItem>
   <TabItem label="CPU">
 ```bash
 docker compose --profile cpu run --no-deps --rm --entrypoint "" openrag-cpu \
-    uv run python scripts/migrations/milvus/migrate.py --downgrade
+    uv run python services/persistence/migrations/milvus/migrate.py --downgrade
 ```
   </TabItem>
 </Tabs>
@@ -243,14 +243,14 @@ To fully remove the fields you would need to recreate the collection from scratc
 
 ## Adding a New Migration Script
 
-Migration scripts live in `openrag/scripts/migrations/milvus/`. The runner discovers them automatically — no registration step required.
+Migration scripts live in `openrag/services/persistence/migrations/milvus/`. The runner discovers them automatically — no registration step required.
 
 ### Naming convention
 
 Files must follow the pattern `N.short_description.py`, where `N` is the **target schema version** as a positive integer:
 
 ```
-openrag/scripts/migrations/milvus/
+openrag/services/persistence/migrations/milvus/
   1.add_temporal_fields.py    ← brings the schema to version 1
   2.your_new_migration.py     ← brings the schema to version 2
   migrate.py                  ← generic runner (do not rename)

docs/content/docs/documentation/sql_migration.mdx

@@ -34,11 +34,11 @@ Replace **`<revision_message>`** with your descriptive message before running th
 docker compose up -d rdb
 docker compose \
     run --no-deps --build --rm \
-    --entrypoint "uv run alembic -c /app/openrag/scripts/migrations/alembic/alembic.ini revision --autogenerate -m '<revision_message>'" \
+    --entrypoint "uv run alembic -c /app/openrag/services/persistence/migrations/alembic/alembic.ini revision --autogenerate -m '<revision_message>'" \
     openrag
 ```
 
-It will create a new migration script in the `openrag/scripts/migrations/alembic/versions/` directory.
+It will create a new migration script in the `openrag/services/persistence/migrations/alembic/versions/` directory.
 
 ### Step 2: Apply the Migration
 
@@ -52,7 +52,7 @@ Rebuilds the image to ensure the migration runs against the exact code being dep
 docker compose up -d rdb
 docker compose \
     run --no-deps --build --rm \
-    --entrypoint "uv run alembic -c /app/openrag/scripts/migrations/alembic/alembic.ini upgrade head" \
+    --entrypoint "uv run alembic -c /app/openrag/services/persistence/migrations/alembic/alembic.ini upgrade head" \
     openrag; docker compose down
 ```
 
@@ -62,7 +62,7 @@ If the stack is already running, skip the rebuild:
 
 ```bash title="Apply migrations (exec)"
 docker compose exec openrag \
-    uv run alembic -c /app/openrag/scripts/migrations/alembic/alembic.ini upgrade head
+    uv run alembic -c /app/openrag/services/persistence/migrations/alembic/alembic.ini upgrade head
 ```
 
 #### Option C: Local (development)
@@ -71,14 +71,14 @@ Run alembic directly with your local venv — no Docker needed:
 
 ```bash title="Apply migrations (local)"
 DATABASE_URL=postgresql://user:pass@localhost:5432/openrag \
-    uv run alembic -c openrag/scripts/migrations/alembic/alembic.ini upgrade head
+    uv run alembic -c openrag/services/persistence/migrations/alembic/alembic.ini upgrade head
 ```
 
 :::tip[Quick alternative: run inside a running container]
 If your stack is already running, you can apply migrations directly without rebuilding:
 ```bash title="Apply migrations via exec"
 docker compose exec -w /app/openrag openrag \
-  /app/.venv/bin/alembic -c scripts/migrations/alembic/alembic.ini upgrade head
+  /app/.venv/bin/alembic -c services/persistence/migrations/alembic/alembic.ini upgrade head
 ```
 :::
 
@@ -108,7 +108,7 @@ This usually happens when two or more migration scripts are created independentl
      ```bash title="Merge Alembic heads"
      docker compose up -d rdb
      docker compose run --no-deps --build --rm \
-         --entrypoint "uv run alembic -c /app/openrag/scripts/migrations/alembic/alembic.ini merge -m 'merge heads' <head1> <head2>" \
+         --entrypoint "uv run alembic -c /app/openrag/services/persistence/migrations/alembic/alembic.ini merge -m 'merge heads' <head1> <head2>" \
          openrag
      ```
      This will generate a new migration script that merges the two branches.
@@ -118,7 +118,7 @@ This usually happens when two or more migration scripts are created independentl
      ```bash title="Apply migrations after merge"
      docker compose up -d rdb
      docker compose run --no-deps --build --rm \
-         --entrypoint "uv run alembic -c /app/openrag/scripts/migrations/alembic/alembic.ini upgrade head" \
+         --entrypoint "uv run alembic -c /app/openrag/services/persistence/migrations/alembic/alembic.ini upgrade head" \
          openrag; docker compose down
      ```