diff --git a/docker-compose.yaml b/docker-compose.yaml index 34b265d5b..064bd2b84 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -169,4 +169,4 @@ services: networks: default: - name: openrag_default + name: openrag_default \ No newline at end of file diff --git a/docs/content/docs/documentation/API.mdx b/docs/content/docs/documentation/API.mdx index 070c53449..5331566f2 100644 --- a/docs/content/docs/documentation/API.mdx +++ b/docs/content/docs/documentation/API.mdx @@ -78,6 +78,22 @@ Upload a new file to a specific partition for indexing. - `201 Created`: Returns task status URL - `409 Conflict`: File already exists in partition +##### Temporal Filtering +OpenRAG supports temporal filtering to retrieve documents from specific time periods. +The client can include temporal fields to allow temporal aware search in search endpoints. + +4 temporal fields are automatically added to the schema of the collection: + +* `datetime`: ISO 8601 format date of the primary timestamp of when a file is created in your system +* `created_at`: ISO 8601 format date of when the file was created +* `updated_at`: ISO 8601 format date of when the file was last modified +* `indexed_at`: ISO 8601 format date of when the file was indexed in the vector database + +:::info +`datetime`, `created_at` and `updated_at` are provided by the client in the metadata of the file during upload, while `indexed_at` is automatically set by openRAG at indexing time. +::: + + ##### Upload files while modeling relations between them OpenRAG supports document relationships to enable context-aware retrieval. @@ -202,6 +218,9 @@ Perform semantic search across specified partitions. | `include_related` (optional) | boolean | `false` | Include chunks from files with same `relationship_id` | | `include_ancestors` (optional) | boolean | `false` | Include chunks from ancestor files (via `parent_id` chain) | | `related_limit` (optional) | integer | 20 | Max related/ancestor chunks to fetch per result (used when `include_related` or `include_ancestors` is true) | +| `filter` (optional) | string | None | Milvus filter expression string for additional filtering (optional). Supports comparison, range, and logical operators. | +| `filter_params` (optional) | object | None | Dictionary of parameter values for templated filters. Use with placeholders in filter expression for better performance. | + **Responses:** - `200 OK`: JSON list of document links (HATEOAS format) @@ -223,6 +242,8 @@ Search within a specific partition only. | `include_related` (optional) | boolean | `false` | Include chunks from files with same `relationship_id` | | `include_ancestors` (optional) | boolean | `false` | Include chunks from ancestor files (via `parent_id` chain) | | `related_limit` (optional) | integer | 20 | Max related/ancestor chunks to fetch per result (used when `include_related` or `include_ancestors` is true) | +| `filter` (optional) | string | None | Milvus filter expression string for additional filtering (optional). Supports comparison, range, and logical operators. | +| `filter_params` (optional) | object | None | Dictionary of parameter values for templated filters. Use with placeholders in filter expression for better performance. | **Response:** Same as multi-partition search @@ -233,7 +254,7 @@ GET /search/partition/{partition}/file/{file_id} Search within a particular file in a partition. -**Query Parameters:** Same as partition search +**Query Parameters:** Same as partition search, including `filter` and `filter_params`. **Response:** Same as other search endpoints --- diff --git a/docs/content/docs/documentation/milvus_migration.md b/docs/content/docs/documentation/milvus_migration.md new file mode 100644 index 000000000..2e01b374d --- /dev/null +++ b/docs/content/docs/documentation/milvus_migration.md @@ -0,0 +1,128 @@ +--- +title: Milvus Migrations +--- + +# Milvus Version Migration +OpenRAG has been upgraded from Milvus **2.5.4** to **2.6.11** to leverage the enhancements introduced in the latest releases, particularly the new temporal querying capabilities added in version **2.6.6+**. + +## What's New in 2.6.x + +Milvus 2.6.6+ introduced the **`TIMESTAMPTZ`** field type, which enables: + +- **Comparison and range filtering** using standard operators (`=`, `!=`, `<`, `>`, etc.) +- **Interval arithmetic** — add or subtract durations (days, hours, minutes) directly in filter expressions +- **Time-based indexing** for faster temporal queries +- **Combined filtering** — pair timestamp conditions with vector similarity search + +**Example — basic comparison:** +```python +expr = "tsz != ISO '2025-01-03T00:00:00+08:00'" +results = client.query( + collection_name, + filter=expr, + output_fields=["id", "tsz"], + limit=10 +) +``` + +**Example — interval arithmetic:** +```python +expr = "tsz + INTERVAL 'P1D' > ISO '2025-01-03T00:00:00+08:00'" +results = client.query( + collection_name, + filter=expr, + output_fields=["id", "tsz"], + limit=10 +) +``` + +> `INTERVAL` values follow [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) syntax: +> * `P1D` = 1 day +> * `PT3H` = 3 hours +> * `P2DT6H` = 2 days and 6 hours. + +## Current State + +:::info +Temporal fields are currently stored as **strings**, not **`TIMESTAMPTZ`**. Migrating to `TIMESTAMPTZ` requires a schema and index change, and Milvus doesn't support migrations on schema and index changes: it has to be handled manually. + +Until a migration strategy is define, filtering still works via **lexicographic string comparison** on ISO 8601 strings: +```python +expr = "tsz != '2025-01-03T00:00:00+08:00'" # No ISO/INTERVAL keywords +results = client.query( + collection_name, + filter=expr, + output_fields=["id", "tsz"], + limit=10 +) +``` +Full `TIMESTAMPTZ` support will be activated in a future release once the migration is established. +::: + +## Milvus Update Steps +These steps must be performed on a deployment running OpenRAG **prior to version 1.1.6** (Milvus 2.5.4). + +> For the full official reference, see the [Milvus upgrade guide](https://milvus.io/docs/upgrade_milvus_standalone-docker.md#Upgrade-process). + +### Step 1 — Upgrade to Milvus 2.5.16 first + +Milvus requires an intermediate upgrade to **v2.5.16** before jumping to 2.6.x. + +Edit `vdb/milvus.yaml` and set the Milvus image tag: + +```diff lang=yaml +// vdb/milvus.yaml +milvus: +- image: milvusdb/milvus:v2.5.4 ++ image: milvusdb/milvus:v2.5.16 # Migrate to milvus 2.5.16 +``` + +Then restart the stack: + +```bash +docker compose down +docker compose up -d +``` + +Wait for all services to be healthy before continuing. + +### Step 2 — Upgrade to Milvus 2.6.11 + +Update `vdb/milvus.yaml` with the target versions (MinIO must also be updated for compatibility): + +```diff lang=yaml +// vdb/milvus.yaml +minio: +- image: minio/minio:RELEASE.2023-03-20T20-16-18Z ++ image: minio/minio:RELEASE.2024-12-18T13-15-44Z + +... +milvus: +- image: milvusdb/milvus:v2.5.16 ++ image: milvusdb/milvus:v2.6.11 +``` + +### Step 3 — Stop all services + +```bash +docker compose down +``` + +Verify that all containers are stopped before proceeding: + +```bash +docker ps | grep milvus +``` + +### Step 4 — Start with the new image + +```bash +docker compose up -d +``` + +Once healthy, confirm the running version: + +```bash +docker inspect milvus-standalone --format '{{ .Config.Image }}' +# Expected: milvusdb/milvus:v2.6.11 +``` \ No newline at end of file diff --git a/openrag/components/indexer/indexer.py b/openrag/components/indexer/indexer.py index 6c3930b7d..0a79ee87e 100644 --- a/openrag/components/indexer/indexer.py +++ b/openrag/components/indexer/indexer.py @@ -225,12 +225,12 @@ async def asearch( self, query: str, top_k: int = 5, - similarity_threshold: float = 0.80, + similarity_threshold: float = 0.60, partition: str | list[str] | None = None, - filter: dict | None = None, + filter: str | None = None, + filter_params: dict | None = None, ) -> list[Document]: partition_list = self._check_partition_list(partition) - filter = filter or {} vectordb = ray.get_actor("Vectordb", namespace="openrag") return await vectordb.async_search.remote( query=query, @@ -238,6 +238,7 @@ async def asearch( top_k=top_k, similarity_threshold=similarity_threshold, filter=filter, + filter_params=filter_params, ) def _check_partition_str(self, partition: str | None) -> str: diff --git a/openrag/components/indexer/vectordb/vectordb.py b/openrag/components/indexer/vectordb/vectordb.py index 7c156c0f8..aaf1fedfe 100644 --- a/openrag/components/indexer/vectordb/vectordb.py +++ b/openrag/components/indexer/vectordb/vectordb.py @@ -1,6 +1,7 @@ import asyncio import time from abc import ABC, abstractmethod +from datetime import UTC, datetime import numpy as np import ray @@ -139,6 +140,7 @@ def __init__(self): self.config = load_config() self.logger = get_logger() + self.time_fields = ["datetime", "created_at", "updated_at", "indexed_at"] # init milvus clients self.port = self.config.vectordb.get("port") @@ -274,6 +276,9 @@ def _create_schema(self): dim=self.embedder.embedding_dimension, ) + # for time_field in self.time_fields: + # schema.add_field(field_name=time_field, datatype=DataType.TIMESTAMPTZ, nullable=True) + if self.hybrid_search: # Add sparse field for BM25 - this will be auto-generated schema.add_field( @@ -327,6 +332,13 @@ def _create_index(self): "bm25_b": 0.75, }, ) + # # indexes for dates TIMESTAMPTZ field + # for time_field in self.time_fields: + # index_params.add_index( + # field_name=time_field, + # index_type="STL_SORT", # Index for TIMESTAMPTZ + # index_name=f"{time_field}_idx", + # ) return index_params @@ -370,11 +382,14 @@ async def async_add_documents(self, chunks: list[Document], user: dict) -> None: entities = [] vectors = await self.embedder.aembed_documents(chunks) order_metadata_l: list[dict] = _gen_chunk_order_metadata(n=len(chunks)) + indexed_at = datetime.now(UTC).isoformat() + for chunk, vector, order_metadata in zip(chunks, vectors, order_metadata_l): entities.append( { "text": chunk.page_content, "vector": vector, + "indexed_at": indexed_at, **order_metadata, **chunk.metadata, } @@ -386,6 +401,7 @@ async def async_add_documents(self, chunks: list[Document], user: dict) -> None: ) # insert file_id and partition into partition_file_manager + file_metadata.update({"indexed_at": indexed_at}) self.partition_file_manager.add_file_to_partition( file_id=file_id, partition=partition, @@ -443,9 +459,10 @@ async def async_search( self, query: str, top_k: int = 5, - similarity_threshold: int = 0.80, + similarity_threshold: float = 0.80, partition: list[str] = None, - filter: dict | None = None, + filter: str | None = None, + filter_params: dict | None = None, with_surrounding_chunks: bool = False, ) -> list[Document]: expr_parts = [] @@ -453,11 +470,11 @@ async def async_search( expr_parts.append(f"partition in {partition}") if filter: - for key, value in filter.items(): - expr_parts.append(f"{key} == '{value}'") + expr_parts.append(filter) # Join all parts with " and " only if there are multiple conditions expr = " and ".join(expr_parts) if expr_parts else "" + filter_params = filter_params or {} try: query_vector = await self.embedder.aembed_query(query) @@ -474,6 +491,7 @@ async def async_search( }, "limit": top_k, "expr": expr, + "expr_params": filter_params, } if self.hybrid_search: sparse_param = { @@ -485,6 +503,7 @@ async def async_search( }, "limit": top_k, "expr": expr, + "expr_params": filter_params, } reqs = [ AnnSearchRequest(**vector_param), @@ -498,10 +517,24 @@ async def async_search( limit=top_k, ) else: + vector_param = { + "data": [query_vector], + "anns_field": "vector", + "search_params": { + "metric_type": "COSINE", + "params": { + "ef": 64, + "radius": similarity_threshold, + "range_filter": 1.0, + }, + }, + "limit": top_k, + } response = await self._async_client.search( collection_name=self.collection_name, output_fields=["*"], - limit=top_k, + filter=expr, + filter_params=filter_params, **vector_param, ) @@ -612,29 +645,22 @@ async def get_file_chunks(self, file_id: str, partition: str, include_id: bool = log = self.logger.bind(file_id=file_id, partition=partition) try: self._check_file_exists(file_id, partition) - # Adjust filter expression based on the type of value - filter_expression = "partition == {partition} and file_id == {file_id}" - filter_params = {"partition": partition, "file_id": file_id} - - # Pagination parameters - offset = 0 - results = [] + filter_expr = f'partition == "{partition}" and file_id == "{file_id}"' excluded_keys = ["text", "vector", "_id"] if not include_id else ["text", "vector"] + results = [] + iterator = self._client.query_iterator( + collection_name=self.collection_name, + filter=filter_expr, + batch_size=limit, + output_fields=["*"], + ) while True: - response = await self._async_client.query( - collection_name=self.collection_name, - filter=filter_expression, - filter_params=filter_params, - limit=limit, - offset=offset, - ) - - if not response: - break # No more results - - results.extend(response) - offset += len(response) # Move offset forward + batch = iterator.next() + if not batch: + iterator.close() + break + results.extend(batch) docs = [ Document( diff --git a/openrag/components/retriever.py b/openrag/components/retriever.py index 56bc3ddbb..99223a713 100644 --- a/openrag/components/retriever.py +++ b/openrag/components/retriever.py @@ -75,12 +75,16 @@ async def retrieve( self, partition: list[str], query: str, + filter: str | None = None, + filter_params: dict | None = None, ) -> list[Document]: db = get_vectordb() chunks = await db.async_search.remote( query=query, partition=partition, top_k=self.top_k, + filter=filter, + filter_params=filter_params, similarity_threshold=self.similarity_threshold, with_surrounding_chunks=self.with_surrounding_chunks, ) @@ -137,7 +141,7 @@ def __init__( prompt: ChatPromptTemplate = ChatPromptTemplate.from_template(MULTI_QUERY_PROMPT) self.generate_queries = prompt | llm | StrOutputParser() | (lambda x: x.split("[SEP]")) - async def retrieve(self, partition: list[str], query: str) -> list[Document]: + async def retrieve(self, partition, query, filter=None, filter_params=None): db = get_vectordb() logger.debug("Generating multiple queries", k_queries=self.k_queries) generated_queries = await self.generate_queries.ainvoke( @@ -150,6 +154,8 @@ async def retrieve(self, partition: list[str], query: str) -> list[Document]: queries=generated_queries, partition=partition, top_k_per_query=self.top_k, + filter=filter, + filter_params=filter_params, similarity_threshold=self.similarity_threshold, with_surrounding_chunks=self.with_surrounding_chunks, ) @@ -196,7 +202,7 @@ async def get_hyde(self, query: str): hyde_document = await self.hyde_generator.ainvoke({"query": query}) return hyde_document - async def retrieve(self, partition: list[str], query: str) -> list[Document]: + async def retrieve(self, partition: list[str], query: str, filter=None, filter_params=None) -> list[Document]: db = get_vectordb() hyde = await self.get_hyde(query) queries = [hyde] @@ -209,6 +215,8 @@ async def retrieve(self, partition: list[str], query: str) -> list[Document]: top_k_per_query=self.top_k, similarity_threshold=self.similarity_threshold, with_surrounding_chunks=self.with_surrounding_chunks, + filter=filter, + filter_params=filter_params, ) diff --git a/openrag/routers/extract.py b/openrag/routers/extract.py index 32f51e5aa..3284fe131 100644 --- a/openrag/routers/extract.py +++ b/openrag/routers/extract.py @@ -32,7 +32,7 @@ - `partition`: Partition name - `page`: Page number in source document - `datetime`: Document date (if set) - - `modified_at`: File modification timestamp + - `updated_at`: File modification timestamp - `created_at`: File creation timestamp - `indexed_at`: Chunk indexing timestamp - Additional custom metadata diff --git a/openrag/routers/indexer.py b/openrag/routers/indexer.py index 8f023d549..26192d35c 100644 --- a/openrag/routers/indexer.py +++ b/openrag/routers/indexer.py @@ -48,6 +48,30 @@ # URL scheme configuration PREFERRED_URL_SCHEME = config.server.preferred_url_scheme +# DATETIME FIELDS: Fields provided by the client + +TEMPORAL_FIELDS = ["datetime", "created_at", "updated_at"] + + +def get_temporal_fields(metadata: dict, file_stat, log) -> None: + temporal_fields = {} + + ## Use provided created_at if available, otherwise extract from file system + for field in TEMPORAL_FIELDS: + datetime_str = metadata.get(field, None) + if datetime_str: + try: + # Try parsing the provided datetime to ensure it's valid + d = datetime.fromisoformat(datetime_str) + temporal_fields[field] = d.isoformat() + except ValueError: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid ISO 8601 datetime field ({datetime_str}) for field '{field}'.", + ) + + return temporal_fields + def build_url(request: Request, route_name: str, **path_params) -> str: """Build a URL using the preferred scheme if configured.""" @@ -99,9 +123,14 @@ async def get_supported_types(): "mimetype": "text/plain", "author": "John Doe", ... + "created_at": "2024-01-01T12:00:00Z" // Optional temporal field (ISO 8601) } ``` +**Temporal Fields:** +- You can provide temporal fields such as `created_at`, `updated_at`, or `datetime` in the metadata for time-based queries and filtering. +- Datetime values must be in ISO 8601 format (e.g., `2024-01-01T12:00:00Z`). + **Common Mimetypes:** - `text/plain` - Plain text files - `text/markdown` - Markdown files @@ -159,9 +188,12 @@ async def add_file( # Append extra metadata metadata["file_size"] = human_readable_size(file_stat.st_size) - metadata["created_at"] = datetime.fromtimestamp(file_stat.st_ctime).isoformat() metadata["file_id"] = file_id + ## Add temporal fields to metadata, using provided values if available, otherwise extracting from file system + temporal_fields = get_temporal_fields(metadata, file_stat, log) + metadata.update(temporal_fields) + # Indexing the file task = indexer.add_file.remote(path=file_path, metadata=metadata, partition=partition, user=user) await task_state_manager.set_state.remote(task.task_id().hex(), "QUEUED") @@ -222,9 +254,14 @@ async def delete_file( "mimetype": "text/plain", "author": "John Doe", ... + "created_at": "2024-01-01T12:00:00Z" // Optional temporal field (ISO 8601) } ``` +**Temporal Fields:** +- You can provide temporal fields such as `created_at`, `updated_at`, or `datetime` in the metadata for time-based queries and filtering. +- Datetime values must be in ISO 8601 format (e.g., `2024-01-01T12:00:00Z`). + **Response:** Returns 202 Accepted with a task status URL for tracking indexing progress. """, @@ -275,9 +312,12 @@ async def put_file( # Append extra metadata metadata["file_size"] = human_readable_size(file_stat.st_size) - metadata["created_at"] = datetime.fromtimestamp(file_stat.st_ctime).isoformat() metadata["file_id"] = file_id + ## Add temporal fields to metadata, using provided values if available, otherwise extracting from file system + temporal_fields = get_temporal_fields(metadata, file_stat, log) + metadata.update(temporal_fields) + # Indexing the file task = indexer.add_file.remote(path=file_path, metadata=metadata, partition=partition, user=user) await task_state_manager.set_state.remote(task.task_id().hex(), "QUEUED") diff --git a/openrag/routers/search.py b/openrag/routers/search.py index 16bea658b..866b80ac5 100644 --- a/openrag/routers/search.py +++ b/openrag/routers/search.py @@ -1,5 +1,8 @@ +import json +from typing import Annotated + from components.retriever import _expand_with_related_chunks -from fastapi import APIRouter, Depends, Query, Request, status +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from fastapi.responses import JSONResponse from utils.dependencies import get_indexer, get_vectordb from utils.logger import get_logger @@ -15,6 +18,66 @@ router = APIRouter() +class RelatedDocSearchParams: + def __init__( + self, + include_related: bool = Query(False, description="Include chunks from files with same relationship_id"), + include_ancestors: bool = Query(False, description="Include chunks from ancestor files in hierarchy"), + related_limit: int = Query(20, description="Maximum number of related/ancestor chunks to fetch per result"), + max_ancestor_depth: int | None = Query( + None, description="Maximum depth of ancestor files to include. None means unlimited." + ), + ): + self.include_related = include_related + self.include_ancestors = include_ancestors + self.related_limit = related_limit + self.max_ancestor_depth = max_ancestor_depth + + +class CommonSearchParams: + def __init__( + self, + text: str = Query(..., description="Text to search semantically"), + top_k: int = Query(5, description="Number of top results to return"), + filter: str | None = Query( + default=None, + description="""Milvus filter expression string.""", + ), + filter_params: str | None = Query( + default=None, + description="""Dictionary of parameter values for templated filters. Use with placeholders in filter expression for better performance.""", + ), + ): + self.text = text + self.top_k = top_k + self.filter = filter + self._filter_params = self._parse_filter_params(filter_params) + + @staticmethod + def _parse_filter_params(filter_params: str | None) -> dict | None: + if not filter_params: + return None + try: + parsed = json.loads(filter_params) + if not isinstance(parsed, dict): + raise HTTPException( + status_code=400, + detail="Invalid 'filter_params' field: must be a JSON object (dict), not a string or array. " + 'Example: {"page": 20} (use double quotes, no outer string quotes).', + ) + return parsed + except json.JSONDecodeError: + raise HTTPException( + status_code=400, + detail="Invalid 'filter_params' field: must be valid JSON. " + 'Use double quotes for keys and string values. Example: {"page": 20}', + ) + + @property + def filter_params(self) -> dict | None: + return self._filter_params + + @router.get( "", description="""Perform semantic search across multiple partitions. @@ -27,6 +90,22 @@ - `include_ancestors`: Include chunks from ancestor files in hierarchy (default: false) - `related_limit`: Maximum number of related/ancestor chunks to fetch per result (default: 20). This is used when `include_related` or `include_ancestors` is true. - `max_ancestor_depth`: Maximum depth of ancestor files to include. None means unlimited. (default: None) +- `filter`: Milvus filter expression string for additional filtering (optional) + Milvus supports the following operators: + - Comparison: ==, !=, >, <, >=, <= + - Range: IN, LIKE + - Logical: AND, OR, NOT (see https://milvus.io/docs/boolean.md) + Examples: + - `file_id == "abc123"` + - `created_at > {start_date}` + - `page >= 5 AND page <= 10` + - `file_id in ["id1", "id2", "id3"]` + +- `filter_params`: Dictionary of parameter values for templated filters (optional) + Use with placeholders in filter expression for better performance. + Example: + - filter: `created_at > {start_date} AND created_at < {end_date}` + - filter_params: {"start_date": "2024-01-01", "end_date": "2024-12-31"} **Behavior:** - `partitions=["all"]`: Search all accessible partitions @@ -55,15 +134,9 @@ ) async def search_multiple_partitions( request: Request, - partitions: list[str] = Query(default=["all"], description="List of partitions to search"), - text: str = Query(..., description="Text to search semantically"), - top_k: int = Query(5, description="Number of top results to return"), - include_related: bool = Query(False, description="Include chunks from files with same relationship_id"), - include_ancestors: bool = Query(False, description="Include chunks from ancestor files in hierarchy"), - related_limit: int = Query(20, description="Maximum number of related/ancestor chunks to fetch per result"), - max_ancestor_depth: int | None = Query( - None, description="Maximum depth of ancestor files to include. None means unlimited." - ), + search_params: Annotated[CommonSearchParams, Depends()], + related_params: Annotated[RelatedDocSearchParams, Depends()], + partitions: list[str] | None = Query(default=["all"], description="List of partitions to search"), indexer=Depends(get_indexer), vectordb=Depends(get_vectordb), partition_viewer=Depends(require_partitions_viewer), @@ -73,29 +146,29 @@ async def search_multiple_partitions( if partitions == ["all"]: partitions = user_partitions - log = logger.bind( - partitions=partitions, - query=text, - top_k=top_k, - include_related=include_related, - include_ancestors=include_ancestors, - ) + log = logger.bind(partitions=partitions, query=search_params.text, top_k=search_params.top_k) - results = await indexer.asearch.remote(query=text, top_k=top_k, partition=partitions) + results = await indexer.asearch.remote( + query=search_params.text, + top_k=search_params.top_k, + partition=partitions, + filter=search_params.filter, + filter_params=search_params.filter_params, + ) log.info( "Semantic search on multiple partitions completed.", result_count=len(results), ) # Expand with related/ancestor chunks if requested - if include_related or include_ancestors: + if related_params.include_related or related_params.include_ancestors: results = await _expand_with_related_chunks( results=results, db=vectordb, - include_related=include_related, - include_ancestors=include_ancestors, - related_limit=related_limit, - max_ancestor_depth=max_ancestor_depth, + include_related=related_params.include_related, + include_ancestors=related_params.include_ancestors, + related_limit=related_params.related_limit, + max_ancestor_depth=related_params.max_ancestor_depth, ) log.info( "Expanded results with related/ancestor chunks.", @@ -128,6 +201,22 @@ async def search_multiple_partitions( - `include_ancestors`: Include chunks from ancestor files in hierarchy (default: false) - `related_limit`: Maximum number of related/ancestor chunks to fetch per result (default: 20). This is used when `include_related` or `include_ancestors` is true. - `max_ancestor_depth`: Maximum depth of ancestor files to include. None means unlimited. (default: None) +- `filter`: Milvus filter expression string for additional filtering (optional) + Milvus supports the following operators: + - Comparison: ==, !=, >, <, >=, <= + - Range: IN, LIKE + - Logical: AND, OR, NOT (see https://milvus.io/docs/boolean.md) + Examples: + - `file_id == "abc123"` + - `created_at > {start_date}` + - `page >= 5 AND page <= 10` + - `file_id in ["id1", "id2", "id3"]` + +- `filter_params`: Dictionary of parameter values for templated filters (optional) + Use with placeholders in filter expression for better performance. + Example: + - filter: `created_at > {start_date} AND created_at < {end_date}` + - filter_params: {"start_date": "2024-01-01", "end_date": "2024-12-31"} **Permissions:** - Requires viewer role on the partition @@ -146,37 +235,33 @@ async def search_multiple_partitions( async def search_one_partition( request: Request, partition: str, - text: str = Query(..., description="Text to search semantically"), - top_k: int = Query(5, description="Number of top results to return"), - include_related: bool = Query(False, description="Include chunks from files with same relationship_id"), - include_ancestors: bool = Query(False, description="Include chunks from ancestor files in hierarchy"), - related_limit: int = Query(20, description="Maximum number of related/ancestor chunks to fetch per result"), - max_ancestor_depth: int | None = Query( - None, description="Maximum depth of ancestor files to include. None means unlimited." - ), + search_params: Annotated[CommonSearchParams, Depends()], + related_params: Annotated[RelatedDocSearchParams, Depends()], indexer=Depends(get_indexer), vectordb=Depends(get_vectordb), partition_viewer=Depends(require_partition_viewer), ): - log = logger.bind( + log = logger.bind(partition=partition, query=search_params.text, top_k=search_params.top_k) + + results = await indexer.asearch.remote( + query=search_params.text, + top_k=search_params.top_k, partition=partition, - query=text, - top_k=top_k, - include_related=include_related, - include_ancestors=include_ancestors, + filter=search_params.filter, + filter_params=search_params.filter_params, ) - results = await indexer.asearch.remote(query=text, top_k=top_k, partition=partition) + log.info("Semantic search on single partition completed.", result_count=len(results)) # Expand with related/ancestor chunks if requested - if include_related or include_ancestors: + if related_params.include_related or related_params.include_ancestors: results = await _expand_with_related_chunks( results=results, db=vectordb, - include_related=include_related, - include_ancestors=include_ancestors, - related_limit=related_limit, - max_ancestor_depth=max_ancestor_depth, + include_related=related_params.include_related, + include_ancestors=related_params.include_ancestors, + related_limit=related_params.related_limit, + max_ancestor_depth=related_params.max_ancestor_depth, ) log.info( "Expanded results with related/ancestor chunks.", @@ -206,6 +291,22 @@ async def search_one_partition( **Query Parameters:** - `text`: Search query text (required) - `top_k`: Number of results to return (default: 5) +- `filter`: Milvus filter expression string for additional filtering (optional) + Milvus supports the following operators: + - Comparison: ==, !=, >, <, >=, <= + - Range: IN, LIKE + - Logical: AND, OR, NOT (see https://milvus.io/docs/boolean.md) + Examples: + - `file_id == "abc123"` + - `created_at > {start_date}` + - `page >= 5 AND page <= 10` + - `file_id in ["id1", "id2", "id3"]` + +- `filter_params`: Dictionary of parameter values for templated filters (optional) + Use with placeholders in filter expression for better performance. + Example: + - filter: `created_at > {start_date} AND created_at < {end_date}` + - filter_params: {"start_date": "2024-01-01", "end_date": "2024-12-31"} **Permissions:** - Requires viewer role on the partition @@ -224,21 +325,20 @@ async def search_file( request: Request, partition: str, file_id: str, - text: str = Query(..., description="Text to search semantically"), - top_k: int = Query(5, description="Number of top results to return"), + search_params: Annotated[CommonSearchParams, Depends()], indexer=Depends(get_indexer), vectordb=Depends(get_vectordb), partition_viewer=Depends(require_partition_viewer), ): - log = logger.bind( + log = logger.bind(partition=partition, file_id=file_id, query=search_params.text, top_k=search_params.top_k) + filter = f'file_id == "{file_id}"' + (f" AND {search_params.filter}" if search_params.filter else "") + results = await indexer.asearch.remote( + query=search_params.text, + top_k=search_params.top_k, partition=partition, - file_id=file_id, - query=text, - top_k=top_k, - include_related=False, - include_ancestors=False, + filter=filter, + filter_params=search_params.filter_params, ) - results = await indexer.asearch.remote(query=text, top_k=top_k, partition=partition, filter={"file_id": file_id}) log.info("Semantic search on specific file completed.", result_count=len(results)) documents = [ diff --git a/pyproject.toml b/pyproject.toml index a5b598d5f..b35582efa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,8 @@ dependencies = [ "ruff>=0.14.1", "librosa>=0.11.0", "cairosvg>=2.7.0", - "pymilvus>=2.5.12", + "pymilvus>=2.6.9", + "protobuf>=5.27,<6.0", ] [dependency-groups] diff --git a/tests/api_tests/test_search.py b/tests/api_tests/test_search.py index 54f566718..4fbf13db1 100644 --- a/tests/api_tests/test_search.py +++ b/tests/api_tests/test_search.py @@ -3,6 +3,7 @@ import json import time import uuid +from datetime import datetime import pytest @@ -480,3 +481,394 @@ def test_max_ancestor_depth_limits_results(self, api_client, indexed_email_threa assert actual_order_limited == expected_order_limited, ( f"Expected {expected_order_limited}, got {actual_order_limited}" ) + + +class TestSearchFiltering: + """Test search filtering functionality on search_one_partition endpoint.""" + + COMMON_CONTENT = """This is a test document for filter testing. +It contains information about machine learning and artificial intelligence. +The document is used to verify that search filtering works correctly. +Key topics include: neural networks, deep learning, and natural language processing. +This content is intentionally repeated across multiple files to test filtering. +""" + + @pytest.fixture + def filter_test_files(self, tmp_path): + """Create 6 files with the same content but different metadata.""" + files_config = [ + { + "file_id": "filter-file-1", + "origin": "source_A", + "file_number": 1, + "datetime": "2020-06-15T00:00:00+00:00", + }, + { + "file_id": "filter-file-2", + "origin": "source_A", + "file_number": 2, + "datetime": "2021-06-15T00:00:00+00:00", + }, + { + "file_id": "filter-file-3", + "origin": "source_B", + "file_number": 3, + "datetime": "2022-06-15T00:00:00+00:00", + }, + { + "file_id": "filter-file-4", + "origin": "source_B", + "file_number": 4, + "datetime": "2023-06-15T00:00:00+00:00", + }, + { + "file_id": "filter-file-5", + "origin": "source_C", + "file_number": 5, + "datetime": "2024-06-15T00:00:00+00:00", + }, + { + "file_id": "filter-file-6", + "origin": "source_C", + "file_number": 6, + "datetime": "2024-07-15T00:00:00+00:00", + }, + ] + + file_paths = {} + for config in files_config: + file_id = config.pop("file_id") + file_path = tmp_path / f"{file_id}.txt" + file_path.write_text(self.COMMON_CONTENT) + config["path"] = file_path + file_paths[file_id] = config + + return file_paths + + @pytest.fixture + def indexed_filter_partition(self, api_client, created_partition, filter_test_files): + """Create partition and index all 6 files with metadata.""" + for file_id, file_info in filter_test_files.items(): + file_path = file_info.pop("path") + with open(file_path, "rb") as f: + response = api_client.post( + f"/indexer/partition/{created_partition}/file/{file_id}", + files={"file": (f"{file_id}.txt", f, "text/plain")}, + data={"metadata": json.dumps(file_info)}, + ) + + data = response.json() + + # Wait for indexing to complete + if "task_status_url" in data: + task_url = data["task_status_url"] + task_path = "/" + "/".join(task_url.split("/")[3:]) + elif "task_id" in data: + task_path = f"/indexer/task/{data['task_id']}" + else: + time.sleep(3) + continue + + for _ in range(30): + task_response = api_client.get(task_path) + task_data = task_response.json() + state = task_data.get("task_state", "") + if state in ["SUCCESS", "COMPLETED", "success", "completed"]: + break + elif state in ["FAILED", "failed", "FAILURE", "failure"]: + pytest.skip(f"Indexing failed for {file_id}: {task_data}") + time.sleep(2) + + return created_partition + + # ========================================================================= + # Comparison filtering tests + # ========================================================================= + + def test_comparaison_filter_with_str(self, api_client, indexed_filter_partition): + """Test filtering with origin == 'source_A' returns only files with that origin.""" + response = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, # to ensure results as embeddings are random but deterministic based on content + "top_k": 10, + "filter": "origin == 'source_A'", + }, + ) + assert response.status_code == 200 + data = response.json() + assert "documents" in data + + documents = data["documents"] + assert len(documents) > 0, "Should find at least one document with origin='source_A'" + + # Verify all results have origin='source_A' + assert all(doc["metadata"].get("origin") == "source_A" for doc in documents), ( + "All documents should have origin='source_A'" + ) + + # Verify we got the expected file_ids + file_ids = {doc["metadata"].get("file_id") for doc in documents} + assert file_ids.issubset({"filter-file-1", "filter-file-2"}), f"Expected file_ids from source_A, got {file_ids}" + + def test_comparison_filter_with_int(self, api_client, indexed_filter_partition): + """Test filtering with file_number >= 3 returns files 3, 4, 5, 6.""" + response = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": "file_number >= 3", + }, + ) + + assert response.status_code == 200 + data = response.json() + assert "documents" in data + + documents = data["documents"] + assert len(documents) > 0, "Should find at least one document with file_number >= 3" + + assert all(doc["metadata"].get("file_number") >= 3 for doc in documents), ( + "All documents should have file_number >= 3" + ) + + # ========================================================================= + # Range filtering tests (IN and LIKE) + # ========================================================================= + + def test_filter_with_IN_operator(self, api_client, indexed_filter_partition): + """Test filtering with origin IN ['source_A', 'source_B'].""" + response = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'origin IN ["source_A", "source_B"]', + }, + ) + assert response.status_code == 200 + data = response.json() + assert "documents" in data + + documents = data["documents"] + assert len(documents) > 0, "Should find documents from source_A or source_B" + + assert all(doc["metadata"].get("origin") in ["source_A", "source_B"] for doc in documents), ( + "All documents should have origin in ['source_A', 'source_B']" + ) + + # Verify we got expected file_ids + file_ids = {doc["metadata"].get("file_id") for doc in documents} + expected_ids = {"filter-file-1", "filter-file-2", "filter-file-3", "filter-file-4"} + assert file_ids.issubset(expected_ids), f"Expected file_ids from source_A/B, got {file_ids}" + + def test_filter_with_LIKE_operator(self, api_client, indexed_filter_partition): + """Test filtering with origin LIKE 'source_%' (matches all).""" + response = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'origin LIKE "source_%"', + }, + ) + assert response.status_code == 200 + data = response.json() + assert "documents" in data + + documents = data["documents"] + assert len(documents) > 0, "Should find documents with origin matching 'source_%'" + + # Verify all results have origin starting with 'source_' + for doc in documents: + origin = doc["metadata"].get("origin", "") + assert origin.startswith("source_"), f"Expected origin starting with 'source_', got {origin}" + + # ========================================================================= + # Logical operator tests (AND, OR) + # ========================================================================= + + def test_logical_operator_AND(self, api_client, indexed_filter_partition): + """Test filtering with origin == 'source_B' AND file_number >= 4.""" + response = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'origin == "source_B" AND file_number >= 4', + }, + ) + assert response.status_code == 200 + data = response.json() + assert "documents" in data + + documents = data["documents"] + assert len(documents) > 0, "Should find documents matching both conditions" + + # Verify all results match both conditions + assert all( + doc["metadata"].get("origin") == "source_B" and doc["metadata"].get("file_number") >= 4 for doc in documents + ), "All documents should have origin='source_B' and file_number >= 4" + + # Only filter-file-4 should match (source_B and file_number=4) + file_ids = {doc["metadata"].get("file_id") for doc in documents} + assert file_ids == {"filter-file-4"}, f"Expected only filter-file-4, got {file_ids}" + + def test_logical_operator_OR(self, api_client, indexed_filter_partition): + """Test filtering with origin == 'source_A' OR file_number == 6.""" + response = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'origin == "source_A" OR file_number == 6', + }, + ) + assert response.status_code == 200 + data = response.json() + assert "documents" in data + + documents = data["documents"] + assert len(documents) > 0, "Should find documents matching either condition" + + assert all( + doc["metadata"].get("origin") == "source_A" or doc["metadata"].get("file_number") == 6 for doc in documents + ), "All documents should have origin='source_A' or file_number=6" + + # Should get filter-file-1, filter-file-2 (source_A) and filter-file-6 (file_number=6) + file_ids = {doc["metadata"].get("file_id") for doc in documents} + expected_ids = {"filter-file-1", "filter-file-2", "filter-file-6"} + assert file_ids == expected_ids, f"Expected {expected_ids}, got {file_ids}" + + # ========================================================================= + # Filter params tests (templated filters) + # ========================================================================= + + def test_filter_with_filter_params(self, api_client, indexed_filter_partition): + """Test that filtering with or without filter params yields the same results when values are the same.""" + # First, search with filter params + response_with_params = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": "origin == {origin_value} AND file_number >= {file_number_value}", + "filter_params": json.dumps({"origin_value": "source_B", "file_number_value": 3}), + }, + ) + assert response_with_params.status_code == 200 + data_with_params = response_with_params.json() + assert "documents" in data_with_params + + # Then, search with hardcoded values (no filter params) + response_hardcoded = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'origin == "source_B" AND file_number >= 3', + }, + ) + assert response_hardcoded.status_code == 200 + data_hardcoded = response_hardcoded.json() + assert "documents" in data_hardcoded + + # Results should be the same + docs_with_params = data_with_params["documents"] + docs_hardcoded = data_hardcoded["documents"] + + # Compare sets of file_ids to avoid ordering issues + file_ids_with_params = {doc["metadata"].get("file_id") for doc in docs_with_params} + file_ids_hardcoded = {doc["metadata"].get("file_id") for doc in docs_hardcoded} + assert file_ids_with_params == file_ids_hardcoded, ( + f"Expected same results with or without filter params, got {file_ids_with_params} vs {file_ids_hardcoded}" + ) + + # ========================================================================= + # Temporal filtering tests (datetime field, ISO 8601) + # ========================================================================= + + def test_temporal_fields_present_in_metadata(self, api_client, indexed_filter_partition): + """Test that the datetime field is present in the metadata of returned documents.""" + response = api_client.get( + f"/search/partition/{indexed_filter_partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + }, + ) + assert response.status_code == 200 + data = response.json() + assert "documents" in data + + documents = data["documents"] + assert len(documents) > 0, "Should find at least one document" + + # Verify that each document has a datetime field in metadata + for doc in documents: + metadata = doc.get("metadata", {}) + for temp_field in ["datetime", "created_at", "indexed_at", "updated_at"]: + k = metadata.get(temp_field) + assert k is not None, ( + f"Document {doc['metadata'].get('file_id')} is missing temporal field '{temp_field}'" + ) + # Verify it's a valid ISO 8601 datetime string + try: + datetime.fromisoformat(k) + except ValueError: + assert False, ( + f"Document {doc['metadata'].get('file_id')} has invalid datetime format in field '{temp_field}': {k}" + ) + + def test_temporal_filter_datetime_iso(self, api_client, indexed_filter_partition): + """Test that temporal filtering on the datetime field works with ISO 8601 strings.""" + partition = indexed_filter_partition + + # --- before 2022: should return files 1 and 2 --- + resp = api_client.get( + f"/search/partition/{partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'datetime < "2022-01-01T00:00:00+00:00"', + }, + ) + assert resp.status_code == 200 + data = resp.json() + assert "documents" in data + file_ids = {doc["metadata"].get("file_id") for doc in data["documents"]} + + assert file_ids == {"filter-file-1", "filter-file-2"}, f"Expected files before 2022, got {file_ids}" + + # --- after 2024: should return files 2 and 3 --- + resp = api_client.get( + f"/search/partition/{partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'datetime > "2024-01-01T00:00:00+00:00"', + }, + ) + assert resp.status_code == 200 + data = resp.json() + assert "documents" in data + file_ids = {doc["metadata"].get("file_id") for doc in data["documents"]} + assert file_ids == {"filter-file-5", "filter-file-6"}, f"Expected files after 2024, got {file_ids}" + + # --- range [2022, 2024]: should return only file 2 --- + resp = api_client.get( + f"/search/partition/{partition}", + params={ + "text": self.COMMON_CONTENT, + "top_k": 10, + "filter": 'datetime >= "2022-01-01T00:00:00+00:00" AND datetime <= "2024-01-01T00:00:00+00:00"', + }, + ) + assert resp.status_code == 200 + data = resp.json() + assert "documents" in data + file_ids = {doc["metadata"].get("file_id") for doc in data["documents"]} + assert file_ids == {"filter-file-3", "filter-file-4"}, ( + f"Expected only temporal-file-2 in range [2022, 2024], got {file_ids}" + ) diff --git a/uv.lock b/uv.lock index 107597b63..cd6970adf 100644 --- a/uv.lock +++ b/uv.lock @@ -2069,20 +2069,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, ] -[[package]] -name = "milvus-lite" -version = "2.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tqdm" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/b2/acc5024c8e8b6a0b034670b8e8af306ebd633ede777dcbf557eac4785937/milvus_lite-2.5.1-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6b014453200ba977be37ba660cb2d021030375fa6a35bc53c2e1d92980a0c512", size = 27934713 }, - { url = "https://files.pythonhosted.org/packages/9b/2e/746f5bb1d6facd1e73eb4af6dd5efda11125b0f29d7908a097485ca6cad9/milvus_lite-2.5.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2e031088bf308afe5f8567850412d618cfb05a65238ed1a6117f60decccc95a", size = 24421451 }, - { url = "https://files.pythonhosted.org/packages/2e/cf/3d1fee5c16c7661cf53977067a34820f7269ed8ba99fe9cf35efc1700866/milvus_lite-2.5.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:a13277e9bacc6933dea172e42231f7e6135bd3bdb073dd2688ee180418abd8d9", size = 45337093 }, - { url = "https://files.pythonhosted.org/packages/d3/82/41d9b80f09b82e066894d9b508af07b7b0fa325ce0322980674de49106a0/milvus_lite-2.5.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25ce13f4b8d46876dd2b7ac8563d7d8306da7ff3999bb0d14b116b30f71d706c", size = 55263911 }, -] - [[package]] name = "monotonic" version = "1.6" @@ -2626,6 +2612,7 @@ dependencies = [ { name = "numba" }, { name = "openai" }, { name = "pip" }, + { name = "protobuf" }, { name = "psutil" }, { name = "psycopg2" }, { name = "pydub" }, @@ -2684,10 +2671,11 @@ requires-dist = [ { name = "numba", specifier = ">=0.61.2" }, { name = "openai", specifier = ">=1.64.0" }, { name = "pip", specifier = ">=25.0.1" }, + { name = "protobuf", specifier = ">=5.27,<6.0" }, { name = "psutil", specifier = ">=7.0.0" }, { name = "psycopg2", specifier = ">=2.9.10" }, { name = "pydub", specifier = ">=0.25.1" }, - { name = "pymilvus", specifier = ">=2.5.12" }, + { name = "pymilvus", specifier = ">=2.6.9" }, { name = "pymupdf4llm", specifier = ">=0.0.17" }, { name = "pytest-env", specifier = ">=1.1.5" }, { name = "python-dotenv", specifier = ">=1.0.1" }, @@ -3654,16 +3642,16 @@ wheels = [ [[package]] name = "protobuf" -version = "6.31.1" +version = "5.29.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/f3/b9655a711b32c19720253f6f06326faf90580834e2e83f840472d752bc8b/protobuf-6.31.1.tar.gz", hash = "sha256:d8cac4c982f0b957a4dc73a80e2ea24fab08e679c0de9deb835f4a12d69aca9a", size = 441797 } +sdist = { url = "https://files.pythonhosted.org/packages/7e/57/394a763c103e0edf87f0938dafcd918d53b4c011dfc5c8ae80f3b0452dbb/protobuf-5.29.6.tar.gz", hash = "sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723", size = 425623 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/6f/6ab8e4bf962fd5570d3deaa2d5c38f0a363f57b4501047b5ebeb83ab1125/protobuf-6.31.1-cp310-abi3-win32.whl", hash = "sha256:7fa17d5a29c2e04b7d90e5e32388b8bfd0e7107cd8e616feef7ed3fa6bdab5c9", size = 423603 }, - { url = "https://files.pythonhosted.org/packages/44/3a/b15c4347dd4bf3a1b0ee882f384623e2063bb5cf9fa9d57990a4f7df2fb6/protobuf-6.31.1-cp310-abi3-win_amd64.whl", hash = "sha256:426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447", size = 435283 }, - { url = "https://files.pythonhosted.org/packages/6a/c9/b9689a2a250264a84e66c46d8862ba788ee7a641cdca39bccf64f59284b7/protobuf-6.31.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402", size = 425604 }, - { url = "https://files.pythonhosted.org/packages/76/a1/7a5a94032c83375e4fe7e7f56e3976ea6ac90c5e85fac8576409e25c39c3/protobuf-6.31.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39", size = 322115 }, - { url = "https://files.pythonhosted.org/packages/fa/b1/b59d405d64d31999244643d88c45c8241c58f17cc887e73bcb90602327f8/protobuf-6.31.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6", size = 321070 }, - { url = "https://files.pythonhosted.org/packages/f7/af/ab3c51ab7507a7325e98ffe691d9495ee3d3aa5f589afad65ec920d39821/protobuf-6.31.1-py3-none-any.whl", hash = "sha256:720a6c7e6b77288b85063569baae8536671b39f15cc22037ec7045658d80489e", size = 168724 }, + { url = "https://files.pythonhosted.org/packages/d4/88/9ee58ff7863c479d6f8346686d4636dd4c415b0cbeed7a6a7d0617639c2a/protobuf-5.29.6-cp310-abi3-win32.whl", hash = "sha256:62e8a3114992c7c647bce37dcc93647575fc52d50e48de30c6fcb28a6a291eb1", size = 423357 }, + { url = "https://files.pythonhosted.org/packages/1c/66/2dc736a4d576847134fb6d80bd995c569b13cdc7b815d669050bf0ce2d2c/protobuf-5.29.6-cp310-abi3-win_amd64.whl", hash = "sha256:7e6ad413275be172f67fdee0f43484b6de5a904cc1c3ea9804cb6fe2ff366eda", size = 435175 }, + { url = "https://files.pythonhosted.org/packages/06/db/49b05966fd208ae3f44dcd33837b6243b4915c57561d730a43f881f24dea/protobuf-5.29.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269", size = 418619 }, + { url = "https://files.pythonhosted.org/packages/b7/d7/48cbf6b0c3c39761e47a99cb483405f0fde2be22cf00d71ef316ce52b458/protobuf-5.29.6-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6", size = 320284 }, + { url = "https://files.pythonhosted.org/packages/e3/dd/cadd6ec43069247d91f6345fa7a0d2858bef6af366dbd7ba8f05d2c77d3b/protobuf-5.29.6-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9", size = 320478 }, + { url = "https://files.pythonhosted.org/packages/5a/cb/e3065b447186cb70aa65acc70c86baf482d82bf75625bf5a2c4f6919c6a3/protobuf-5.29.6-py3-none-any.whl", hash = "sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86", size = 173126 }, ] [[package]] @@ -3872,20 +3860,20 @@ sdist = { url = "https://files.pythonhosted.org/packages/5d/ab/34ec41718af73c001 [[package]] name = "pymilvus" -version = "2.5.12" +version = "2.6.9" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "cachetools" }, { name = "grpcio" }, - { name = "milvus-lite", marker = "sys_platform != 'win32'" }, + { name = "orjson" }, { name = "pandas" }, { name = "protobuf" }, { name = "python-dotenv" }, { name = "setuptools" }, - { name = "ujson" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fa/53/4af820a37163225a76656222ee43a0eb8f1bd2ceec063315680a585435da/pymilvus-2.5.12.tar.gz", hash = "sha256:79ec7dc0616c2484f77abe98bca8deafb613645b5703c492b51961afd4f985d8", size = 1265893 } +sdist = { url = "https://files.pythonhosted.org/packages/94/0c/92adff800a04cd3e9b3f17c06fa972c8d590846b1e0bac0ccf39e054b596/pymilvus-2.6.9.tar.gz", hash = "sha256:c53a3d84ff15814e251be13edda70a98a1c8a6090d7597a908387cbb94a9504a", size = 1493560 } wheels = [ - { url = "https://files.pythonhosted.org/packages/68/4f/80a4940f2772d10272c3292444af767a5aa1a5bbb631874568713ca01d54/pymilvus-2.5.12-py3-none-any.whl", hash = "sha256:ef77a4a0076469a30b05f0bb23b5a058acfbdca83d82af9574ca651764017f44", size = 231425 }, + { url = "https://files.pythonhosted.org/packages/6a/56/ab7f0a5aba6fc06dc210a059d6f6d2ee1f3371d40e2b4366a409576554b8/pymilvus-2.6.9-py3-none-any.whl", hash = "sha256:3e14e8072f6429dcd79d52a24dc021c594cb80841ddd76cb974bc539d1f4cdda", size = 301225 }, ] [[package]] @@ -5333,34 +5321,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839 }, ] -[[package]] -name = "ujson" -version = "5.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f0/00/3110fd566786bfa542adb7932d62035e0c0ef662a8ff6544b6643b3d6fd7/ujson-5.10.0.tar.gz", hash = "sha256:b3cd8f3c5d8c7738257f1018880444f7b7d9b66232c64649f562d7ba86ad4bc1", size = 7154885 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e8/a6/fd3f8bbd80842267e2d06c3583279555e8354c5986c952385199d57a5b6c/ujson-5.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:98ba15d8cbc481ce55695beee9f063189dce91a4b08bc1d03e7f0152cd4bbdd5", size = 55642 }, - { url = "https://files.pythonhosted.org/packages/a8/47/dd03fd2b5ae727e16d5d18919b383959c6d269c7b948a380fdd879518640/ujson-5.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9d2edbf1556e4f56e50fab7d8ff993dbad7f54bac68eacdd27a8f55f433578e", size = 51807 }, - { url = "https://files.pythonhosted.org/packages/25/23/079a4cc6fd7e2655a473ed9e776ddbb7144e27f04e8fc484a0fb45fe6f71/ujson-5.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6627029ae4f52d0e1a2451768c2c37c0c814ffc04f796eb36244cf16b8e57043", size = 51972 }, - { url = "https://files.pythonhosted.org/packages/04/81/668707e5f2177791869b624be4c06fb2473bf97ee33296b18d1cf3092af7/ujson-5.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8ccb77b3e40b151e20519c6ae6d89bfe3f4c14e8e210d910287f778368bb3d1", size = 53686 }, - { url = "https://files.pythonhosted.org/packages/bd/50/056d518a386d80aaf4505ccf3cee1c40d312a46901ed494d5711dd939bc3/ujson-5.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3caf9cd64abfeb11a3b661329085c5e167abbe15256b3b68cb5d914ba7396f3", size = 58591 }, - { url = "https://files.pythonhosted.org/packages/fc/d6/aeaf3e2d6fb1f4cfb6bf25f454d60490ed8146ddc0600fae44bfe7eb5a72/ujson-5.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6e32abdce572e3a8c3d02c886c704a38a1b015a1fb858004e03d20ca7cecbb21", size = 997853 }, - { url = "https://files.pythonhosted.org/packages/f8/d5/1f2a5d2699f447f7d990334ca96e90065ea7f99b142ce96e85f26d7e78e2/ujson-5.10.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a65b6af4d903103ee7b6f4f5b85f1bfd0c90ba4eeac6421aae436c9988aa64a2", size = 1140689 }, - { url = "https://files.pythonhosted.org/packages/f2/2c/6990f4ccb41ed93744aaaa3786394bca0875503f97690622f3cafc0adfde/ujson-5.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:604a046d966457b6cdcacc5aa2ec5314f0e8c42bae52842c1e6fa02ea4bda42e", size = 1043576 }, - { url = "https://files.pythonhosted.org/packages/14/f5/a2368463dbb09fbdbf6a696062d0c0f62e4ae6fa65f38f829611da2e8fdd/ujson-5.10.0-cp312-cp312-win32.whl", hash = "sha256:6dea1c8b4fc921bf78a8ff00bbd2bfe166345f5536c510671bccececb187c80e", size = 38764 }, - { url = "https://files.pythonhosted.org/packages/59/2d/691f741ffd72b6c84438a93749ac57bf1a3f217ac4b0ea4fd0e96119e118/ujson-5.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:38665e7d8290188b1e0d57d584eb8110951a9591363316dd41cf8686ab1d0abc", size = 42211 }, - { url = "https://files.pythonhosted.org/packages/0d/69/b3e3f924bb0e8820bb46671979770c5be6a7d51c77a66324cdb09f1acddb/ujson-5.10.0-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:618efd84dc1acbd6bff8eaa736bb6c074bfa8b8a98f55b61c38d4ca2c1f7f287", size = 55646 }, - { url = "https://files.pythonhosted.org/packages/32/8a/9b748eb543c6cabc54ebeaa1f28035b1bd09c0800235b08e85990734c41e/ujson-5.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:38d5d36b4aedfe81dfe251f76c0467399d575d1395a1755de391e58985ab1c2e", size = 51806 }, - { url = "https://files.pythonhosted.org/packages/39/50/4b53ea234413b710a18b305f465b328e306ba9592e13a791a6a6b378869b/ujson-5.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67079b1f9fb29ed9a2914acf4ef6c02844b3153913eb735d4bf287ee1db6e557", size = 51975 }, - { url = "https://files.pythonhosted.org/packages/b4/9d/8061934f960cdb6dd55f0b3ceeff207fcc48c64f58b43403777ad5623d9e/ujson-5.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7d0e0ceeb8fe2468c70ec0c37b439dd554e2aa539a8a56365fd761edb418988", size = 53693 }, - { url = "https://files.pythonhosted.org/packages/f5/be/7bfa84b28519ddbb67efc8410765ca7da55e6b93aba84d97764cd5794dbc/ujson-5.10.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:59e02cd37bc7c44d587a0ba45347cc815fb7a5fe48de16bf05caa5f7d0d2e816", size = 58594 }, - { url = "https://files.pythonhosted.org/packages/48/eb/85d465abafb2c69d9699cfa5520e6e96561db787d36c677370e066c7e2e7/ujson-5.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2a890b706b64e0065f02577bf6d8ca3b66c11a5e81fb75d757233a38c07a1f20", size = 997853 }, - { url = "https://files.pythonhosted.org/packages/9f/76/2a63409fc05d34dd7d929357b7a45e3a2c96f22b4225cd74becd2ba6c4cb/ujson-5.10.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:621e34b4632c740ecb491efc7f1fcb4f74b48ddb55e65221995e74e2d00bbff0", size = 1140694 }, - { url = "https://files.pythonhosted.org/packages/45/ed/582c4daba0f3e1688d923b5cb914ada1f9defa702df38a1916c899f7c4d1/ujson-5.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b9500e61fce0cfc86168b248104e954fead61f9be213087153d272e817ec7b4f", size = 1043580 }, - { url = "https://files.pythonhosted.org/packages/d7/0c/9837fece153051e19c7bade9f88f9b409e026b9525927824cdf16293b43b/ujson-5.10.0-cp313-cp313-win32.whl", hash = "sha256:4c4fc16f11ac1612f05b6f5781b384716719547e142cfd67b65d035bd85af165", size = 38766 }, - { url = "https://files.pythonhosted.org/packages/d7/72/6cb6728e2738c05bbe9bd522d6fc79f86b9a28402f38663e85a28fddd4a0/ujson-5.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:4573fd1695932d4f619928fd09d5d03d917274381649ade4328091ceca175539", size = 42212 }, -] - [[package]] name = "umap-learn" version = "0.5.9.post2" diff --git a/vdb/milvus.yaml b/vdb/milvus.yaml index 6a9f37e0c..b8bde4759 100644 --- a/vdb/milvus.yaml +++ b/vdb/milvus.yaml @@ -1,6 +1,6 @@ services: etcd: - image: quay.io/coreos/etcd:v3.5.16 + image: quay.io/coreos/etcd:v3.5.25 environment: - ETCD_AUTO_COMPACTION_MODE=revision - ETCD_AUTO_COMPACTION_RETENTION=1000 @@ -8,7 +8,7 @@ services: - ETCD_SNAPSHOT_COUNT=50000 volumes: - ${MILVUS_VOLUME_DIRECTORY:-./volumes}/etcd:/etcd - command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd healthcheck: test: ["CMD", "etcdctl", "endpoint", "health"] interval: 30s @@ -16,7 +16,7 @@ services: retries: 3 minio: - image: minio/minio:RELEASE.2023-03-20T20-16-18Z + image: minio/minio:RELEASE.2024-12-18T13-15-44Z environment: MINIO_ACCESS_KEY: minioadmin MINIO_SECRET_KEY: minioadmin @@ -30,7 +30,7 @@ services: retries: 3 milvus: - image: milvusdb/milvus:v2.5.4 + image: milvusdb/milvus:v2.6.11 command: ["milvus", "run", "standalone"] security_opt: - seccomp:unconfined @@ -45,8 +45,8 @@ services: start_period: 90s timeout: 20s retries: 3 - # ports: - # - "${VDB_PORT:-19530}:${VDB_iPORT:-19530}" + ports: + - "${VDB_PORT:-19530}:${VDB_iPORT:-19530}" depends_on: - "etcd" - "minio" \ No newline at end of file