Fix #1650 Replace search field with generated search vector field

Author: pauloxnet

README.rst

@@ -87,7 +87,7 @@ Install and run locally from a virtual environment
 #. For docs (next step requires ``gettext``)::
 
     python -m manage loaddata doc_releases
-    python -m manage update_docs --update-index
+    python -m manage update_docs
 
 #. For dashboard:
 
@@ -236,16 +236,8 @@ minified version of it to this directory.
 Documentation search
 --------------------
 
-When running ``python -m manage update_docs --update-index`` to build all
-documents it will also automatically index every document it builds in the
-search engine as well. In case you've already built the documents and would like
-to reindex the search index, run the command::
-
-    python -m manage update_index
-
-This is also the right command to run when you work on the search feature
-itself. You can pass the ``-d`` option to try to drop the search index
-first before indexing all the documents.
+When running ``python -m manage update_docs`` to build all documents it will
+also automatically index every document it builds in the search engine as well.
 
 Updating metrics from production
 --------------------------------

docker-entrypoint.dev.sh

@@ -12,6 +12,5 @@ python -m manage loaddata doc_releases
 python -m manage loaddata dashboard_production_metrics
 # python -m manage loaddata dashboard_example_data
 python -m manage update_metrics
-#python -m manage update_index
 
 exec "$@"

docs/management/commands/update_docs.py

@@ -47,13 +47,6 @@ def add_arguments(self, parser):
             action="store_true",
             help="Ask before building each version",
         )
-        parser.add_argument(
-            "--update-index",
-            action="store_true",
-            dest="update_index",
-            default=False,
-            help="Also update the search vector field.",
-        )
         parser.add_argument(
             "--purge-cache",
             action="store_true",
@@ -98,25 +91,19 @@ def _get_doc_releases(self, versions, options):
 
     def handle(self, *versions, **kwargs):
         self.verbosity = kwargs["verbosity"]
-        self.update_index = kwargs["update_index"]
         self.purge_cache = kwargs["purge_cache"]
 
         self.default_builders = ["json", "djangohtml"]
 
         # Keep track of which Git sources have been updated, e.g.,
         # {'1.8': True} if the 1.8 docs updated.
         self.release_docs_changed = {}
-        # Only update the index if some docs rebuild.
-        self.update_index_required = False
 
         for release in self._get_doc_releases(versions, kwargs):
             self.build_doc_release(
                 release, force=kwargs["force"], interactive=kwargs["interactive"]
             )
 
-        if self.update_index_required:
-            call_command("update_index", **{"verbosity": self.verbosity})
-
         if self.purge_cache:
             changed_versions = {
                 version
@@ -172,8 +159,6 @@ def build_doc_release(self, release, force=False, interactive=False):
                 )
             return
 
-        self.update_index_required = self.update_index
-
         source_dir = checkout_dir.joinpath("docs")
 
         if release.lang != "en":
@@ -288,15 +273,6 @@ def zipfile_inclusion_filter(file_path):
         if release.is_default:
             self._setup_stable_symlink(release, built_dir)
 
-        #
-        # Rebuild the imported document list and search index.
-        #
-        if not self.update_index:
-            return
-
-        if self.verbosity >= 2:
-            self.stdout.write("  reindexing...")
-
         json_built_dir = parent_build_dir.joinpath("_built", "json")
         documents = gen_decoded_documents(json_built_dir)
         release.sync_to_db(documents)

docs/management/commands/update_index.py

@@ -16,7 +16,11 @@
 )
 from django.core.cache import cache
 from django.db import models, transaction
-from django.db.models import Q
+from django.db.models import (
+    Case,
+    Q,
+    When,
+)
 from django.db.models.fields.json import KeyTextTransform
 from django.utils.functional import cached_property
 from django.utils.html import strip_tags
@@ -27,10 +31,10 @@
 from . import utils
 from .search import (
     DEFAULT_TEXT_SEARCH_CONFIG,
-    DOCUMENT_SEARCH_VECTOR,
     START_SEL,
     STOP_SEL,
     TSEARCH_CONFIG_LANGUAGES,
+    get_document_search_vector,
 )
 
 
@@ -261,7 +265,7 @@ def search(self, query_text, release, document_category=None):
             search_query = SearchQuery(
                 query_text, config=models.F("config"), search_type="websearch"
             )
-            search_rank = SearchRank(models.F("search"), search_query)
+            search_rank = SearchRank(models.F("search_vector"), search_query)
             search = partial(
                 SearchHeadline,
                 start_sel=START_SEL,
@@ -296,7 +300,7 @@ def search(self, query_text, release, document_category=None):
             )
             vector_qs = (
                 base_qs.alias(rank=search_rank)
-                .filter(search=search_query)
+                .filter(search_vector=search_query)
                 .order_by("-rank")
             )
             if not vector_qs:
@@ -314,22 +318,6 @@ def search(self, query_text, release, document_category=None):
         else:
             return self.none()
 
-    def search_reset(self):
-        """Set to null all not null Document's search vector fields."""
-        return Document.objects.exclude(search=None).update(search=None)
-
-    def search_update(self):
-        """
-        Update Document's search vector fields using the document definition.
-
-        This method don't index the module pages (since source code is hard to
-        combine with full text search) and the big flattened index of the CBVs.
-        """
-        return Document.objects.exclude(
-            Q(path__startswith="_modules")
-            | Q(path__startswith="ref/class-based-views/flattened-index")
-        ).update(search=DOCUMENT_SEARCH_VECTOR)
-
 
 class Document(models.Model):
     """
@@ -344,8 +332,20 @@ class Document(models.Model):
     path = models.CharField(max_length=500)
     title = models.CharField(max_length=500)
     metadata = models.JSONField(default=dict)
-    search = SearchVectorField(null=True, editable=False)
-    config = models.SlugField(default=DEFAULT_TEXT_SEARCH_CONFIG)
+    search_vector = models.GeneratedField(
+        expression=Case(
+            *[
+                When(config=lang, then=get_document_search_vector(lang))
+                for lang in TSEARCH_CONFIG_LANGUAGES.values()
+            ],
+            default=get_document_search_vector(),
+        ),
+        output_field=SearchVectorField(),
+        db_persist=True,
+    )
+    config = models.SlugField(
+        db_default=DEFAULT_TEXT_SEARCH_CONFIG, default=DEFAULT_TEXT_SEARCH_CONFIG
+    )
 
     objects = DocumentQuerySet.as_manager()
 
@@ -354,7 +354,7 @@ class Meta:
             models.Index(
                 fields=["release", "title"], name="document_release_title_idx"
             ),
-            GinIndex(fields=["search"]),
+            GinIndex(fields=["search_vector"], name="document_search_vector_idx"),
         ]
         unique_together = ("release", "path")
 

docs/migrations/0007_add_search_vector.py

@@ -1,5 +1,5 @@
 from django.contrib.postgres.search import SearchVector
-from django.db.models import F, TextChoices
+from django.db.models import TextChoices
 from django.db.models.fields.json import KeyTextTransform
 from django.utils.translation import gettext_lazy as _
 
@@ -40,15 +40,17 @@
 # https://github.com/postgres/postgres/blob/REL_14_STABLE/src/bin/initdb/initdb.c#L2557
 DEFAULT_TEXT_SEARCH_CONFIG = "simple"
 
-DOCUMENT_SEARCH_VECTOR = (
-    SearchVector("title", weight="A", config=F("config"))
-    + SearchVector(KeyTextTransform("slug", "metadata"), weight="A", config=F("config"))
-    + SearchVector(KeyTextTransform("toc", "metadata"), weight="B", config=F("config"))
-    + SearchVector(KeyTextTransform("body", "metadata"), weight="C", config=F("config"))
-    + SearchVector(
-        KeyTextTransform("parents", "metadata"), weight="D", config=F("config")
+
+def get_document_search_vector(lang=DEFAULT_TEXT_SEARCH_CONFIG):
+    """Return the search vector with the proper language config."""
+    return (
+        SearchVector("title", weight="A", config=lang)
+        + SearchVector(KeyTextTransform("slug", "metadata"), weight="A", config=lang)
+        + SearchVector(KeyTextTransform("toc", "metadata"), weight="B", config=lang)
+        + SearchVector(KeyTextTransform("body", "metadata"), weight="C", config=lang)
+        + SearchVector(KeyTextTransform("parents", "metadata"), weight="D", config=lang)
     )
-)
+
 
 START_SEL = "<mark>"
 STOP_SEL = "</mark>"

docs/models.py

@@ -7,7 +7,7 @@
 
 from releases.models import Release
 
-from ..models import DOCUMENT_SEARCH_VECTOR, Document, DocumentRelease
+from ..models import Document, DocumentRelease
 
 
 class ModelsTests(TestCase):
@@ -350,9 +350,6 @@ def setUpTestData(cls):
         ]
         Document.objects.bulk_create(Document(**doc) for doc in documents)
 
-    def setUp(self):
-        Document.objects.search_update()
-
     def test_search(self):
         expected_list = [
             (
@@ -415,29 +412,17 @@ def test_search_breadcrumbs(self):
             ],
         )
 
-    def test_search_reset(self):
-        self.assertEqual(Document.objects.exclude(search=None).count(), 6)
-        self.assertEqual(Document.objects.search_reset(), 6)
-        self.assertEqual(Document.objects.exclude(search=None).count(), 0)
-
-    def test_search_update(self):
-        self.assertEqual(Document.objects.exclude(search=None).count(), 6)
-        self.assertEqual(Document.objects.search_update(), 6)
-        self.assertEqual(Document.objects.exclude(search=None).count(), 6)
-
     def test_search_highlight_stemmed(self):
         # The issue only manifests itself when the defaut search config is not english
         with connection.cursor() as cursor:
             cursor.execute("SET default_text_search_config TO 'simple'", [])
 
-        doc = self.release.documents.create(
+        self.release.documents.create(
             config="english",
             path="/",
             title="triaging tickets",
             metadata={"body": "text containing the word triaging", "breadcrumbs": []},
         )
-        doc.search = DOCUMENT_SEARCH_VECTOR
-        doc.save(update_fields=["search"])
 
         self.assertQuerySetEqual(
             Document.objects.search("triaging", self.release),

docs/search.py

@@ -195,7 +195,6 @@ def test_code_links(self):
         Document.objects.bulk_create(
             [Document(**queryset_data), Document(**empty_page_data)]
         )
-        Document.objects.search_update()
         base_url = reverse_with_host(
             "document-detail",
             host="docs",