Add: Compactions

Author: ashvardanian

cpp/README.md

@@ -81,7 +81,7 @@ Instead of spawning additional threads within USearch, we focus on the thread sa
 ```cpp
 #pragma omp parallel for
     for (std::size_t i = 0; i < n; ++i)
-        native.add(key, span_t{vector, dims}, index_add_config_t { .thread = omp_get_thread_num() });
+        native.add(key, span_t{vector, dims}, index_update_config_t { .thread = omp_get_thread_num() });
 ```
 
 During initialization, we allocate enough temporary memory for all the cores on the machine.

cpp/bench.cpp

@@ -286,7 +286,7 @@ void index_many(index_at& index, std::size_t n, vector_id_at const* ids, real_at
 #pragma omp parallel for schedule(static, 32)
 #endif
     for (std::size_t i = 0; i < n; ++i) {
-        index_add_config_t config;
+        index_update_config_t config;
 #if USEARCH_USE_OPENMP
         config.thread = omp_get_thread_num();
 #endif
@@ -373,7 +373,7 @@ static void single_shot(dataset_at& dataset, index_at& index, bool construct = t
                     if (progress % 1000 == 0)
                         printer.print(progress, total);
                 });
-            join_attempts = result.cycles;
+            join_attempts = result.visited_members;
         }
     }
     // Evaluate join quality

cpp/test.cpp

@@ -96,7 +96,7 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
     executor_default_t executor;
     index.reserve({vectors.size(), executor.size()});
     executor.execute_bulk(vectors.size() - 3, [&](std::size_t thread, std::size_t task) {
-        index_add_config_t config;
+        index_update_config_t config;
         config.thread = thread;
         index.add(key_max - task - 3, vectors[task + 3].data(), args..., config);
     });
@@ -114,6 +114,9 @@ void test_cosine(index_at& index, std::vector<std::vector<scalar_at>> const& vec
         std::vector<scalar_t> vec_recovered_from_view(dimensions);
         index.get(key_second, vec_recovered_from_view.data());
         expect(std::equal(vector_second, vector_second + dimensions, vec_recovered_from_view.data()));
+
+        auto compaction_result = index.compact();
+        expect(bool(compaction_result));
     }
 
     expect(index.memory_usage() > 0);
@@ -201,7 +204,7 @@ template <typename key_at, typename slot_at> void test_tanimoto(std::size_t dime
 
     index.reserve({batch_size + index.size(), executor.size()});
     executor.execute_bulk(batch_size, [&](std::size_t thread, std::size_t task) {
-        index_add_config_t config;
+        index_update_config_t config;
         config.thread = thread;
         index.add(task + 25000, scalars.data() + index.scalar_words() * task, config);
     });

include/usearch/index.hpp

@@ -268,10 +268,15 @@ One may often want to evaluate the quality of the constructed index before runni
 The trivial way is to measure `recall@1` on the entries already present in the index.
 
 ```py
-from usearch.eval import recall_members
+from usearch.eval import self_recall
 
-assert recall_members(index, exact=True) == (1, 0)
-print(recall_members(index, exact=False))
+stats: SearchStats = self_recall(index, exact=True)
+assert stats.visited_members == 0, "Exact search won't attend index nodes"
+assert stats.computed_distances == len(index), "And will compute the distance to every node"
+
+stats: SearchStats = self_recall(index, exact=False)
+assert stats.visited_members > 0
+assert stats.computed_distances <= len(index)
 ```
 
 In case you have some ground-truth data for more than one entry, you compare search results against expected values:

include/usearch/index_dense.hpp

@@ -186,7 +186,7 @@ static void add_typed_to_index(                                            //
     byte_t const* keys_data = reinterpret_cast<byte_t const*>(keys_info.ptr);
 
     executor_default_t{threads}.execute_bulk(vectors_count, [&](std::size_t thread_idx, std::size_t task_idx) {
-        index_dense_add_config_t config;
+        index_dense_update_config_t config;
         config.force_vector_copy = force_copy;
         config.thread = thread_idx;
         key_t key = *reinterpret_cast<key_t const*>(keys_data + task_idx * keys_info.strides[0]);
@@ -246,7 +246,7 @@ static void search_typed(                                   //
     dense_index_py_t& index, py::buffer_info& vectors_info, //
     std::size_t wanted, bool exact, std::size_t threads,    //
     py::array_t<key_t>& keys_py, py::array_t<distance_t>& distances_py, py::array_t<Py_ssize_t>& counts_py,
-    std::atomic<std::size_t>& stats_lookups, std::atomic<std::size_t>& stats_measurements) {
+    std::atomic<std::size_t>& stats_visited_members, std::atomic<std::size_t>& stats_computed_distances) {
 
     auto keys_py2d = keys_py.template mutable_unchecked<2>();
     auto distances_py2d = distances_py.template mutable_unchecked<2>();
@@ -270,8 +270,8 @@ static void search_typed(                                   //
         counts_py1d(task_idx) =
             static_cast<Py_ssize_t>(result.dump_to(&keys_py2d(task_idx, 0), &distances_py2d(task_idx, 0)));
 
-        stats_lookups += result.lookups;
-        stats_measurements += result.measurements;
+        stats_visited_members += result.visited_members;
+        stats_computed_distances += result.computed_distances;
         if (PyErr_CheckSignals() != 0)
             throw py::error_already_set();
     });
@@ -282,7 +282,7 @@ static void search_typed(                                       //
     dense_indexes_py_t& indexes, py::buffer_info& vectors_info, //
     std::size_t wanted, bool exact, std::size_t threads,        //
     py::array_t<key_t>& keys_py, py::array_t<distance_t>& distances_py, py::array_t<Py_ssize_t>& counts_py,
-    std::atomic<std::size_t>& stats_lookups, std::atomic<std::size_t>& stats_measurements) {
+    std::atomic<std::size_t>& stats_visited_members, std::atomic<std::size_t>& stats_computed_distances) {
 
     auto keys_py2d = keys_py.template mutable_unchecked<2>();
     auto distances_py2d = distances_py.template mutable_unchecked<2>();
@@ -324,8 +324,8 @@ static void search_typed(                                       //
                     wanted));
             }
 
-            stats_lookups += result.lookups;
-            stats_measurements += result.measurements;
+            stats_visited_members += result.visited_members;
+            stats_computed_distances += result.computed_distances;
             if (PyErr_CheckSignals() != 0)
                 throw py::error_already_set();
         }
@@ -363,16 +363,16 @@ static py::tuple search_many_in_index( //
     py::array_t<key_t> keys_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<distance_t> distances_py({vectors_count, static_cast<Py_ssize_t>(wanted)});
     py::array_t<Py_ssize_t> counts_py(vectors_count);
-    std::atomic<std::size_t> stats_lookups(0);
-    std::atomic<std::size_t> stats_measurements(0);
+    std::atomic<std::size_t> stats_visited_members(0);
+    std::atomic<std::size_t> stats_computed_distances(0);
 
     // clang-format off
     switch (numpy_string_to_kind(vectors_info.format)) {
-    case scalar_kind_t::b1x8_k: search_typed<b1x8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_lookups, stats_measurements); break;
-    case scalar_kind_t::f8_k: search_typed<f8_bits_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_lookups, stats_measurements); break;
-    case scalar_kind_t::f16_k: search_typed<f16_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_lookups, stats_measurements); break;
-    case scalar_kind_t::f32_k: search_typed<f32_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_lookups, stats_measurements); break;
-    case scalar_kind_t::f64_k: search_typed<f64_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_lookups, stats_measurements); break;
+    case scalar_kind_t::b1x8_k: search_typed<b1x8_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
+    case scalar_kind_t::f8_k: search_typed<f8_bits_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
+    case scalar_kind_t::f16_k: search_typed<f16_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
+    case scalar_kind_t::f32_k: search_typed<f32_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
+    case scalar_kind_t::f64_k: search_typed<f64_t>(index, vectors_info, wanted, exact, threads, keys_py, distances_py, counts_py, stats_visited_members, stats_computed_distances); break;
     default: throw std::invalid_argument("Incompatible scalars in the query matrix: " + vectors_info.format);
     }
     // clang-format on
@@ -381,8 +381,8 @@ static py::tuple search_many_in_index( //
     results[0] = keys_py;
     results[1] = distances_py;
     results[2] = counts_py;
-    results[3] = stats_lookups.load();
-    results[4] = stats_measurements.load();
+    results[3] = stats_visited_members.load();
+    results[4] = stats_computed_distances.load();
     return results;
 }
 
@@ -391,21 +391,18 @@ static std::unordered_map<key_t, key_t> join_index(       //
     std::size_t max_proposals, bool exact) {
 
     std::unordered_map<key_t, key_t> a_to_b;
+    dummy_label_to_label_mapping_t b_to_a;
     a_to_b.reserve((std::min)(a.size(), b.size()));
 
-    // index_join_config_t config;
-
-    // config.max_proposals = max_proposals;
-    // config.exact = exact;
-    // config.expansion = (std::max)(a.expansion_search(), b.expansion_search());
-    // std::size_t threads = (std::min)(a.limits().threads(), b.limits().threads());
-    // executor_default_t executor{threads};
-    // join_result_t result = dense_index_py_t::join( //
-    //     a, b, config,                              //
-    //     a_to_b,                                    //
-    //     dummy_label_to_label_mapping_t{},          //
-    //     executor);
-    // result.error.raise();
+    index_join_config_t config;
+    config.max_proposals = max_proposals;
+    config.exact = exact;
+    config.expansion = (std::max)(a.expansion_search(), b.expansion_search());
+    std::size_t threads = (std::min)(a.limits().threads(), b.limits().threads());
+    executor_default_t executor{threads};
+    join_result_t result = a.join(b, config, a_to_b, b_to_a, executor);
+    result.error.raise();
+
     return a_to_b;
 }
 
@@ -418,6 +415,16 @@ static dense_index_py_t copy_index(dense_index_py_t const& index) {
     return std::move(result.index);
 }
 
+static void compact_index(dense_index_py_t& index, std::size_t threads) {
+
+    if (!threads)
+        threads = std::thread::hardware_concurrency();
+    if (!index.reserve(index_limits_t(index.size(), threads)))
+        throw std::invalid_argument("Out of memory!");
+
+    index.compact(executor_default_t{threads});
+}
+
 // clang-format off
 template <typename index_at> void save_index(index_at const& index, std::string const& path) { index.save(path.c_str()).error.raise(); }
 template <typename index_at> void load_index(index_at& index, std::string const& path) { index.load(path.c_str()).error.raise(); }
@@ -601,7 +608,7 @@ PYBIND11_MODULE(compiled, m) {
             if (!index.reserve(index_limits_t(index.size(), threads)))
                 throw std::invalid_argument("Out of memory!");
 
-            index.compact(executor_default_t{threads});
+            index.isolate(executor_default_t{threads});
             return result.completed;
         },
         py::arg("key"), py::arg("compact"), py::arg("threads"));
@@ -619,7 +626,7 @@ PYBIND11_MODULE(compiled, m) {
             if (!index.reserve(index_limits_t(index.size(), threads)))
                 throw std::invalid_argument("Out of memory!");
 
-            index.compact(executor_default_t{threads});
+            index.isolate(executor_default_t{threads});
             return result.completed;
         },
         py::arg("key"), py::arg("compact"), py::arg("threads"));
@@ -651,6 +658,7 @@ PYBIND11_MODULE(compiled, m) {
     i.def("reset", &reset_index<dense_index_py_t>, py::call_guard<py::gil_scoped_release>());
     i.def("clear", &clear_index<dense_index_py_t>, py::call_guard<py::gil_scoped_release>());
     i.def("copy", &copy_index, py::call_guard<py::gil_scoped_release>());
+    i.def("compact", &compact_index, py::call_guard<py::gil_scoped_release>());
     i.def("join", &join_index, py::arg("other"), py::arg("max_proposals") = 0, py::arg("exact") = false,
           py::call_guard<py::gil_scoped_release>());
 

python/README.md

@@ -108,21 +108,23 @@ def test_index(
     index.add(42, vector)
 
     assert len(index) == 1, "Size after addition"
-    assert 42 in index, "Presense in the index"
+    assert 42 in index, "Presence in the index"
     assert 42 in index.keys, "Presence among keys"
-    assert 43 not in index, "Presense in the index, false positive"
+    assert 43 not in index, "Presence in the index, false positive"
     assert index[42] is not None, "Vector recovery"
     assert index[43] is None, "Vector recovery, false positive"
     assert len(index[42]) == ndim
     if numpy_type != np.byte:
         assert np.allclose(index[42], vector, atol=0.1)
 
-    matches = index.search(vector, 10)
+    matches: Matches = index.search(vector, 10)
     assert len(matches.keys) == 1, "Number of matches"
-    assert len(matches.keys) == len(matches.distances), "Symmetric match subarrays"
+    assert len(matches.keys) == len(matches.distances), "Symmetric match sub-arrays"
     assert len({match.key for match in matches}) == 1, "Iteration over matches"
     assert matches[0].key == 42
     assert matches[0].distance == pytest.approx(0, abs=1e-3)
+    assert matches.computed_distances <= 2
+    assert matches.visited_members <= 2
 
     # Validating the index structure and metadata:
     assert index.max_level >= 0
@@ -137,7 +139,7 @@ def test_index(
     index.remove(43)
     assert len(index) == 1
 
-    # Try insreting back
+    # Try inserting back
     index.add(43, other_vector)
     assert len(index) == 2
     index.remove(43)
@@ -183,7 +185,7 @@ def test_index(
     index = Index.restore(temporary_usearch_filename)
     assert index is None
 
-    # Try openning a corrupt file
+    # Try opening a corrupt file
     with open(temporary_usearch_filename, "w") as file:
         file.write("Some random string")
     meta = Index.metadata(temporary_usearch_filename)
@@ -280,15 +282,20 @@ def test_exact_recall(
         matches: Matches = index.search(vectors[i], 10, exact=True)
         found_labels = matches.keys
         assert found_labels[0] == i
+        assert matches.computed_distances == len(index)
+        assert matches.visited_members == 0, "Exact search won't traverse the graph"
 
     # Search the whole batch
     if batch_size > 1:
         matches: BatchMatches = index.search(vectors, 10, exact=True)
+        assert matches.computed_distances == len(index) * len(vectors)
+        assert matches.visited_members == 0, "Exact search won't traverse the graph"
+
         found_labels = matches.keys
         for i in range(batch_size):
             assert found_labels[i, 0] == i
 
-    # Match entries aginst themselves
+    # Match entries against themselves
     index_copy: Index = index.copy()
     mapping: dict = index.join(index_copy, exact=True)
     for man, woman in mapping.items():

python/lib.cpp

@@ -55,24 +55,51 @@ def random_vectors(
         return x
 
 
-def recall_members(index: Index, sample: float = 1, **kwargs) -> Tuple[float, float]:
-    """Simplest benchmark for a quality of search, which queries every
-    existing member of the index, to make sure approximate search finds
-    the point itself. Reports 2 metrics - "self-recall" and "efficiency".
+@dataclass
+class SearchStats:
+    """
+    Contains statistics for one or more search runs, including the number of
+    internal nodes that were fetched (`visited_members`) and the number
+    of times the distance metric was invoked (`computed_distances`).
 
-    Self-recall is the share of queried vectors, that were succesfully found.
+    Other derivative metrics include the `mean_recall` and `mean_efficiency`.
+    Recall is the share of queried vectors, that were successfully found.
     Efficiency describes the number of distances that had to be computed for
     each query, normalized to size of the `index`. Highest efficiency is 0.(9),
     lowest is zero. Highest is achieved, when the distance metric was computed
     just once per query. Lowest happens during exact search, when every distance
     to every present vector had to be computed.
+    """
+
+    index_size: int
+    count_queries: int
+    count_matches: int
+
+    visited_members: int
+    computed_distances: int
+
+    @property
+    def mean_efficiency(self) -> float:
+        return 1 - float(self.computed_distances) / (
+            self.count_queries * self.index_size
+        )
+
+    @property
+    def mean_recall(self) -> float:
+        return self.count_matches / self.count_queries
+
+
+def self_recall(index: Index, sample: float = 1, **kwargs) -> SearchStats:
+    """Simplest benchmark for a quality of search, which queries every
+    existing member of the index, to make sure approximate search finds
+    the point itself.
 
     :param index: Non-empty pre-constructed index
     :type index: Index
     :param sample: Share of vectors to search, defaults to 1
     :type sample: float
-    :return: Value from 0 to 1, for the share of found self-references
-    :rtype: float, float
+    :return: Evaluation report with key metrics
+    :rtype: SearchStats
     """
     if len(index) == 0:
         return 0
@@ -85,9 +112,14 @@ def recall_members(index: Index, sample: float = 1, **kwargs) -> Tuple[float, fl
 
     queries = index.get_vectors(keys, index.dtype)
     matches: BatchMatches = index.search(queries, **kwargs)
-    recall_first: float = matches.recall_first(keys)
-    efficiency: float = 1 - float(matches.measurements) / (len(keys) * len(index))
-    return recall_first, efficiency
+    count_matches: float = matches.count_matches(keys)
+    return SearchStats(
+        index_size=len(index),
+        count_queries=len(keys),
+        count_matches=count_matches,
+        visited_members=matches.visited_members,
+        computed_distances=matches.computed_distances,
+    )
 
 
 def measure_seconds(f: Callable) -> Tuple[float, Any]: