refactoring: Standardize format of search params in engine configs (#122)

* refactoring: Standardize format of search params in engine configs

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* address review and rename params to config

* address review

* fix: upgrade milvusdb image version

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: tellet-q

engine/clients/elasticsearch/search.py

@@ -51,7 +51,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
             "field": "vector",
             "query_vector": vector,
             "k": top,
-            **{"num_candidates": 100, **cls.search_params},
+            **cls.search_params["config"],
         }
 
         meta_conditions = cls.parser.parse(meta_conditions)

engine/clients/milvus/config.py

@@ -23,4 +23,5 @@
     DataType.INT64: 0,
     DataType.VARCHAR: "---MILVUS DOES NOT ACCEPT EMPTY STRINGS---",
     DataType.FLOAT: 0.0,
+    DataType.DOUBLE: 0.0,
 }

engine/clients/milvus/search.py

@@ -38,7 +38,7 @@ def get_mp_start_method(cls):
 
     @classmethod
     def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
-        param = {"metric_type": cls.distance, "params": cls.search_params["params"]}
+        param = {"metric_type": cls.distance, "params": cls.search_params["config"]}
         try:
             res = cls.collection.search(
                 data=[vector],

engine/clients/opensearch/search.py

@@ -84,5 +84,5 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
     def setup_search(cls):
         if cls.search_params:
             cls.client.indices.put_settings(
-                body=cls.search_params, index=OPENSEARCH_INDEX
+                body=cls.search_params["config"], index=OPENSEARCH_INDEX
             )

engine/clients/pgvector/search.py

@@ -23,9 +23,7 @@ def init_client(cls, host, distance, connection_params: dict, search_params: dic
         cls.conn = psycopg.connect(**get_db_config(host, connection_params))
         register_vector(cls.conn)
         cls.cur = cls.conn.cursor()
-        cls.cur.execute(
-            f"SET hnsw.ef_search = {search_params['search_params']['hnsw_ef']}"
-        )
+        cls.cur.execute(f"SET hnsw.ef_search = {search_params['config']['hnsw_ef']}")
         if distance == Distance.COSINE:
             cls.query = f"SELECT id, embedding <=> %s AS _score FROM items ORDER BY _score LIMIT %s"
         elif distance == Distance.L2:

engine/clients/qdrant/search.py

@@ -40,8 +40,6 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
             query_vector=vector,
             query_filter=cls.parser.parse(meta_conditions),
             limit=top,
-            search_params=rest.SearchParams(
-                **cls.search_params.get("search_params", {})
-            ),
+            search_params=rest.SearchParams(**cls.search_params.get("config", {})),
         )
         return [(hit.id, hit.score) for hit in res]

engine/clients/redis/search.py

@@ -64,7 +64,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
         params_dict = {
             "vec_param": np.array(vector).astype(np.float32).tobytes(),
             "K": top,
-            "EF": cls.search_params["search_params"]["ef"],
+            **cls.search_params["config"],
             **params,
         }
         results = cls._ft.search(q, query_params=params_dict)

engine/clients/weaviate/search.py

@@ -45,7 +45,7 @@ def search_one(self, vector, meta_conditions, top) -> List[Tuple[int, float]]:
     def setup_search(self):
         self.collection.config.update(
             vector_index_config=Reconfigure.VectorIndex.hnsw(
-                ef=self.search_params["vectorIndexConfig"]["ef"]
+                ef=self.search_params["config"]["ef"]
             )
         )
 

engine/servers/milvus-limit-ram/docker-compose.yaml

@@ -39,7 +39,7 @@ services:
 
   standalone:
     container_name: milvus-standalone
-    image: milvusdb/milvus:v2.3.0-beta
+    image: milvusdb/milvus:v2.3.1
     command: ["milvus", "run", "standalone"]
     environment:
       ETCD_ENDPOINTS: etcd:2379

experiments/configurations/elasticsearch-single-node.json

@@ -7,8 +7,8 @@
     },
     "collection_params": { "index_options": { "m": 16, "ef_construction": 100 } },
     "search_params": [
-      { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
+      { "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
+      { "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
     ],
     "upload_params": { "parallel": 16 }
   },
@@ -20,8 +20,8 @@
     },
     "collection_params": { "index_options": { "m": 16, "ef_construction": 128 } },
     "search_params": [
-      { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
+      { "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
+      { "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
     ],
     "upload_params": { "parallel": 16 }
   },
@@ -33,8 +33,8 @@
     },
     "collection_params": { "index_options": { "m": 32, "ef_construction": 128 } },
     "search_params": [
-      { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
+      { "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
+      { "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
     ],
     "upload_params": { "parallel": 16 }
   },
@@ -46,8 +46,8 @@
     },
     "collection_params": { "index_options": { "m": 32, "ef_construction": 256 } },
     "search_params": [
-      { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
+      { "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
+      { "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
     ],
     "upload_params": { "parallel": 16 }
   },
@@ -59,8 +59,8 @@
     },
     "collection_params": { "index_options": { "m": 32, "ef_construction": 512 } },
     "search_params": [
-      { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
+      { "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
+      { "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
     ],
     "upload_params": { "parallel": 16 }
   },
@@ -72,8 +72,8 @@
     },
     "collection_params": { "index_options": { "m": 64, "ef_construction": 256 } },
     "search_params": [
-      { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
+      { "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
+      { "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
     ],
     "upload_params": { "parallel": 16 }
   },
@@ -85,8 +85,8 @@
     },
     "collection_params": { "index_options": { "m": 64, "ef_construction": 512 } },
     "search_params": [
-      { "parallel": 1, "num_candidates": 128 }, { "parallel": 1, "num_candidates": 256 }, { "parallel": 1, "num_candidates": 512 },
-      { "parallel": 100, "num_candidates": 128 }, { "parallel": 100, "num_candidates": 256 }, { "parallel": 100, "num_candidates": 512 }
+      { "parallel": 1, "config": { "num_candidates": 128 } }, { "parallel": 1, "config": { "num_candidates": 256 } }, { "parallel": 1, "config": { "num_candidates": 512 } },
+      { "parallel": 100, "config": { "num_candidates": 128 } }, { "parallel": 100, "config": { "num_candidates": 256 } }, { "parallel": 100, "config": { "num_candidates": 512 } }
     ],
     "upload_params": { "parallel": 16 }
   }