Dbpedia configuration (#28)

* Add dbpedia calibration json

* Add LVQ's to dbpedia-calibration configuration

* Fix create-svs

* Add LeanVec4X8

Author: mpozniak95

engine/clients/redis/search.py

@@ -38,7 +38,7 @@ def init_client(cls, host, distance, connection_params: dict, search_params: dic
             if cls.hybrid_policy != "ADHOC_BF":
                 cls.knn_conditions = "EF_RUNTIME $EF"
         elif cls.algorithm == "SVS-VAMANA":
-            cls.knn_conditions = "WS_SEARCH $SEARCH_WINDOW_SIZE"
+            cls.knn_conditions = "WS_SEARCH $WS_SEARCH"
         cls.data_type = "FLOAT32"
         if "search_params" in cls.search_params:
             cls.data_type = (
@@ -97,7 +97,7 @@ def search_one(cls, vector, meta_conditions, top) -> List[Tuple[int, float]]:
             if cls.hybrid_policy != "ADHOC_BF":
                 params_dict["EF"] = cls.search_params["search_params"]["ef"]
         if cls.algorithm == "SVS-VAMANA":
-            params_dict["SEARCH_WINDOW_SIZE"] = cls.search_params["search_params"]["SEARCH_WINDOW_SIZE"]
+            params_dict["WS_SEARCH"] = cls.search_params["search_params"]["WS_SEARCH"]
         results = cls._ft.search(q, query_params=params_dict)
 
         return [(int(result.id), float(result.vector_score)) for result in results.docs]

experiments/configurations/create-svs.py

@@ -1,14 +1,14 @@
 import json
 
 threads = [16]
-ws_constructs = [100]
-ws_search = [32, 40, 48, 64]
+ws_constructs = [200]
+ws_search = [177]
 #ws_search = [48]
 graph_degree = [32]
-#quantization = ["NO_COMPRESSION", "LVQ4x4", "LVQ4x8", "LVQ4", "LVQ8", "LeanVec4x8", "LeanVec8x8"]
-quantization = ["LVQ4x8"]
-topKs = [10]
-data_types = ["FLOAT32"]
+#quantization = ["LVQ4X4", "LVQ4x8", "LVQ8", "LVQ4"]
+quantization = ["LVQ4X4"]
+topKs = [100]
+data_types = ["FLOAT16", "FLOAT32"]
 
 for algo in ["svs-vamana"]:
     for data_type in data_types:
@@ -18,34 +18,34 @@
                     configs = []
                     for thread in threads:
                         config = {
-                            "name": f"svs-test-algo-{algo}-graph-{graph_d}-ws-con-{ws_construct}-quant-{quant}-threads-{thread}-dt-{data_type}",
+                            "name": f"svs-{algo}-quant-{quant}-dt-{data_type}",
                             "engine": "redis",
                             "connection_params": {},
                             "collection_params": {
                                 "algorithm": algo,
                                 "data_type": data_type,
-                                f"{algo}_config": {"NUM_THREADS": thread, "GRAPH_MAX_DEGREE": graph_d, "CONSTRUCTION_WINDOW_SIZE": ws_construct, "COMPRESSION": quant},
+                                f"{algo}_config": {"NUM_THREADS": thread, "GRAPH_MAX_DEGREE": graph_d, "CONSTRUCTION_WINDOW_SIZE": ws_construct, "compression": quant},
                             },
                             "search_params": [],
                             "upload_params": {
-                                "parallel": 128,
+                                "parallel": 100,
                                 "data_type": data_type,
                                 "algorithm": algo,
                             },
                         }
-                        for client in [1, 8, 16, 32, 64, 128]:
+                        for client in [100]:
                             for ws_s in ws_search:
                                 for top in topKs:
                                     test_config = {
                                         "algorithm": algo,
                                         "parallel": client,
                                         "top": top,
-                                        "search_params": {"SEARCH_WINDOW_SIZE": ws_s, "data_type": data_type},
+                                        "search_params": {"WS_SEARCH": ws_s, "data_type": data_type},
                                     }
                                     config["search_params"].append(test_config)
                         configs.append(config)
 
-                    fname = f"svs-test-algo-{algo}-graph-{graph_d}-ws-con-{ws_construct}-quant-{quant}-threads-{thread}-dt-{data_type}.json"
+                    fname = f"svs-{algo}-quant-{quant}-dt-{data_type}.json"
                     with open(fname, "w") as json_fd:
                         json.dump(configs, json_fd, indent=2)
-                        print(f"Created {len(configs)} configs for {fname}.")
+                        print(f"Created {len(configs)} configs for {fname}.")