integrity error solved

Author: Dishant1804

backend/apps/ai/common/base/chunk_command.py

@@ -20,7 +20,7 @@ def help(self) -> str:
     def process_chunks_batch(self, entities: list[Model]) -> int:
         """Process a batch of entities to create or update chunks."""
         processed = 0
-        batch_chunks_to_create = {}
+        batch_chunks_to_create = []
         content_type = ContentType.objects.get_for_model(self.model_class)
 
         for entity in entities:
@@ -58,19 +58,19 @@ def process_chunks_batch(self, entities: list[Model]) -> int:
                     continue
 
                 chunk_texts = Chunk.split_text(full_content)
-                if not chunk_texts:
+                unique_chunk_texts = list(dict.fromkeys(chunk_texts))
+
+                if not unique_chunk_texts:
                     self.stdout.write(f"No chunks created for {self.entity_name} {entity_key}")
                     continue
 
                 if chunks := create_chunks_and_embeddings(
-                    chunk_texts=chunk_texts,
+                    chunk_texts=unique_chunk_texts,
                     context=context,
                     openai_client=self.openai_client,
                     save=False,
                 ):
-                    for chunk in chunks:
-                        key = (chunk.context_id, chunk.text)
-                        batch_chunks_to_create[key] = chunk
+                    batch_chunks_to_create.extend(chunks)
                     processed += 1
                     self.stdout.write(
                         self.style.SUCCESS(f"Created {len(chunks)} new chunks for {entity_key}")
@@ -79,21 +79,7 @@ def process_chunks_batch(self, entities: list[Model]) -> int:
                 self.stdout.write(f"Chunks for {entity_key} are already up to date.")
 
         if batch_chunks_to_create:
-            context_ids = {context_id for context_id, _ in batch_chunks_to_create}
-            candidate_chunk_texts = {text for _, text in batch_chunks_to_create}
-
-            existing_keys = set(
-                Chunk.objects.filter(
-                    context_id__in=context_ids, text__in=candidate_chunk_texts
-                ).values_list("context_id", "text")
-            )
-
-            chunks_to_insert = [
-                chunk for key, chunk in batch_chunks_to_create.items() if key not in existing_keys
-            ]
-
-            if chunks_to_insert:
-                Chunk.bulk_save(chunks_to_insert)
+            Chunk.bulk_save(batch_chunks_to_create)
 
         return processed
 

backend/tests/apps/ai/common/base/chunk_command_test.py

@@ -280,7 +280,7 @@ def test_process_chunks_batch_multiple_entities(
             assert mock_create_chunks.call_count == 3
             mock_bulk_save.assert_called_once()
             bulk_save_args = mock_bulk_save.call_args[0][0]
-            assert len(bulk_save_args) == 2
+            assert len(bulk_save_args) == 6
 
     @patch("apps.ai.common.base.chunk_command.ContentType.objects.get_for_model")
     @patch("apps.ai.common.base.chunk_command.Context.objects.filter")
@@ -449,27 +449,25 @@ def test_process_chunks_batch_with_duplicates(
         mock_content_type,
         mock_chunks,
     ):
-        """Test that duplicate chunks are filtered out before bulk save."""
+        """Test that duplicate chunk texts are filtered out before processing."""
         mock_get_content_type.return_value = mock_content_type
         mock_context_filter.return_value.first.return_value = mock_context
-        mock_split_text.return_value = ["chunk1", "chunk2", "chunk3"]
+        mock_split_text.return_value = ["chunk1", "chunk2", "chunk1", "chunk3", "chunk2"]
         mock_create_chunks.return_value = mock_chunks
         command.openai_client = Mock()
 
-        with (
-            patch("apps.ai.models.chunk.Chunk.objects.filter") as mock_chunk_filter,
-            patch.object(command.stdout, "write"),
-        ):
-            mock_qs = Mock()
-            mock_qs.values_list.return_value = [(1, "Chunk text 1")]
-            mock_chunk_filter.return_value = mock_qs
-
+        with patch.object(command.stdout, "write"):
             result = command.process_chunks_batch([mock_entity])
 
             assert result == 1
-            mock_bulk_save.assert_called_once()
-            bulk_save_args = mock_bulk_save.call_args[0][0]
-            assert len(bulk_save_args) == 2
+            mock_split_text.assert_called_once()
+            mock_create_chunks.assert_called_once_with(
+                chunk_texts=["chunk1", "chunk2", "chunk3"],
+                context=mock_context,
+                openai_client=command.openai_client,
+                save=False,
+            )
+            mock_bulk_save.assert_called_once_with(mock_chunks)
 
     def test_process_chunks_batch_whitespace_only_content(
         self, command, mock_entity, mock_context, mock_content_type