diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index b547f806ff..a567e8f378 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -925,7 +925,12 @@ def _missing_dimensionality_appears_recoverable( return False label_count = len(id_to_label) - if int(total) != label_count or len(label_to_id) != label_count: + # total_elements_added is monotonic across every add, while id_to_label and + # label_to_id hold only live elements, so a segment that has had deletions + # carries total_elements_added > label_count. Require >= (not ==), otherwise + # every post-deletion dim-None segment is wrongly quarantined (#1710); the + # label-map size and bijection checks still reject inconsistent label maps. + if int(total) < label_count or len(label_to_id) != label_count: return False try: return all(label_to_id.get(label) == item_id for item_id, label in id_to_label.items()) diff --git a/tests/test_backends.py b/tests/test_backends.py index f7845825e6..8af4f2bba9 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -1467,6 +1467,35 @@ def test_quarantine_invalid_hnsw_metadata_keeps_consistent_missing_dimensionalit assert seg.exists() +def test_quarantine_invalid_hnsw_metadata_keeps_post_deletion_missing_dimensionality(tmp_path): + """A deleted-from segment has total_elements_added > live label count (the + counter is monotonic); that dim-None shape is recoverable, not corruption (#1710). + """ + palace = tmp_path / "palace" + palace.mkdir() + seg = palace / "abcd-1234-5678" + seg.mkdir() + (seg / "data_level0.bin").write_bytes(b"x" * 2048) + (seg / "link_lists.bin").write_bytes(b"x" * 128) + with open(seg / "index_metadata.pickle", "wb") as f: + pickle.dump( + { + "dimensionality": None, + "total_elements_added": 5, + "max_seq_id": None, + "id_to_label": {"a": 1, "b": 2}, + "label_to_id": {1: "a", 2: "b"}, + "id_to_seq_id": {}, + }, + f, + ) + + moved = quarantine_invalid_hnsw_metadata(str(palace)) + + assert moved == [] + assert seg.exists() + + def test_quarantine_invalid_hnsw_metadata_renames_mismatched_missing_dimensionality(tmp_path): palace = tmp_path / "palace" palace.mkdir()