Merge pull request #306 from bouthilx/hotfix/ephemeral_remove_and_update_index

Fix EphemeralDB index update after document deletion

Author: bouthilx

src/orion/core/io/database/ephemeraldb.py

@@ -301,11 +301,15 @@ def delete_many(self, query=None):
         retained_documents = []
         for document in self._documents:
             if not document.match(query):
-                retained_documents.append(document)
+                retained_documents.append(document.to_dict())
             else:
                 deleted += 1
 
-        self._documents = retained_documents
+        for name, (keys, _) in self._indexes.items():
+            self._indexes[name] = (keys, set())
+
+        self._documents = []
+        self.insert_many(retained_documents)
 
         return deleted
 

tests/unittests/core/test_ephemeraldb.py

@@ -310,6 +310,26 @@ def test_remove_with_id(self, exp_config, database, orion_db):
             backward.populate_space(loaded_config)
         assert loaded_configs == exp_config[0][1:]
 
+    def test_remove_update_indexes(self, exp_config, database, orion_db):
+        """Verify that indexes are properly update after deletion."""
+        with pytest.raises(DuplicateKeyError):
+            orion_db.write('experiments', {'_id': exp_config[0][0]['_id']})
+        with pytest.raises(DuplicateKeyError):
+            orion_db.write('experiments', {'_id': exp_config[0][1]['_id']})
+
+        filt = {'_id': exp_config[0][0]['_id']}
+
+        count_before = database['experiments'].count()
+        # call interface
+        assert orion_db.remove('experiments', filt) == 1
+        assert database['experiments'].count() == count_before - 1
+        # Should not fail now, otherwise it means the indexes were not updated properly during
+        # remove()
+        orion_db.write('experiments', filt)
+        # And this should still fail
+        with pytest.raises(DuplicateKeyError):
+            orion_db.write('experiments', {'_id': exp_config[0][1]['_id']})
+
 
 @pytest.mark.usefixtures("clean_db")
 class TestCount(object):

tests/unittests/core/test_pickleddb.py

@@ -301,6 +301,28 @@ def test_remove_with_id(self, exp_config, orion_db):
             backward.populate_space(loaded_config)
         assert loaded_configs == exp_config[0][1:]
 
+    def test_remove_update_indexes(self, exp_config, orion_db):
+        """Verify that indexes are properly update after deletion."""
+        with pytest.raises(DuplicateKeyError):
+            orion_db.write('experiments', {'_id': exp_config[0][0]['_id']})
+        with pytest.raises(DuplicateKeyError):
+            orion_db.write('experiments', {'_id': exp_config[0][1]['_id']})
+
+        filt = {'_id': exp_config[0][0]['_id']}
+
+        database = orion_db._get_database()._db
+        count_before = database['experiments'].count()
+        # call interface
+        assert orion_db.remove('experiments', filt) == 1
+        database = orion_db._get_database()._db
+        assert database['experiments'].count() == count_before - 1
+        # Should not fail now, otherwise it means the indexes were not updated properly during
+        # remove()
+        orion_db.write('experiments', filt)
+        # And this should still fail
+        with pytest.raises(DuplicateKeyError):
+            orion_db.write('experiments', {'_id': exp_config[0][1]['_id']})
+
 
 @pytest.mark.usefixtures("clean_db")
 class TestCount(object):