[Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow (#9800) (#9836)

* [Fixes #9799] Thesaurus selectbox performance in metadata editor is very slow

Co-authored-by: mattiagiupponi <[email protected]>

Co-authored-by: Marcel Wallschläger <[email protected]>

Author: mattiagiupponi

Diff for: geonode/base/forms.py

@@ -40,17 +40,18 @@
 from taggit.forms import TagField
 from tinymce.widgets import TinyMCE
 from django.contrib.admin.utils import flatten
+from django.utils.translation import get_language
+
 from geonode.base.enumerations import ALL_LANGUAGES
 from geonode.base.models import (HierarchicalKeyword,
                                  License, Region, ResourceBase, Thesaurus,
                                  ThesaurusKeyword, ThesaurusKeywordLabel, ThesaurusLabel,
                                  TopicCategory)
 from geonode.base.widgets import TaggitSelect2Custom
+from geonode.base.fields import MultiThesauriField
 from geonode.documents.models import Document
 from geonode.layers.models import Dataset
-from django.utils.translation import get_language
-from .fields import MultiThesauriField
-from geonode.base.utils import validate_extra_metadata
+from geonode.base.utils import validate_extra_metadata, remove_country_from_lanugecode
 
 logger = logging.getLogger(__name__)
 
@@ -290,7 +291,14 @@ class Meta:
     )
 
 
+THESAURUS_RESULT_LIST_SEPERATOR = ("", "-------")
+
+
 class ThesaurusAvailableForm(forms.Form):
+
+    # seperator at beginning of thesaurus search result and between
+    # results found in local language and alt label
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         lang = get_language()
@@ -337,16 +345,21 @@ def _define_choicefield(self, item, required, tname, lang):
 
     @staticmethod
     def _get_thesauro_keyword_label(item, lang):
-        qs_local = []
-        qs_non_local = [("", "------")]
-        for key in ThesaurusKeyword.objects.filter(thesaurus_id=item.id):
-            label = ThesaurusKeywordLabel.objects.filter(keyword=key).filter(lang=lang)
-            if label.exists():
-                qs_local.append((label.get().keyword.id, label.get().label))
-            else:
-                qs_non_local.append((key.id, key.alt_label))
 
-        return qs_non_local + qs_local
+        keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=item)
+
+        # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again
+        qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
+        if len(qs_keyword_ids) == 0:
+            lang = remove_country_from_lanugecode(lang)
+            qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
+
+        not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id")
+
+        qs_local = list(ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values_list("keyword_id", "label"))
+        qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids).values_list("id", "alt_label"))
+
+        return [THESAURUS_RESULT_LIST_SEPERATOR] + qs_local + [THESAURUS_RESULT_LIST_SEPERATOR] + qs_non_local
 
     @staticmethod
     def _get_thesauro_title_label(item, lang):

Diff for: geonode/base/tests.py

@@ -58,6 +58,7 @@
 from geonode.storage.manager import storage_manager
 from django.test import Client, TestCase, override_settings, SimpleTestCase
 from django.shortcuts import reverse
+from django.utils import translation
 
 from geonode.base.middleware import ReadOnlyMiddleware, MaintenanceMiddleware
 from geonode.base.templatetags.base_tags import get_visibile_resources, facets
@@ -73,7 +74,7 @@
 from django.core.files import File
 from django.core.management import call_command
 from django.core.management.base import CommandError
-from geonode.base.forms import ThesaurusAvailableForm
+from geonode.base.forms import ThesaurusAvailableForm, THESAURUS_RESULT_LIST_SEPERATOR
 
 
 test_image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0))
@@ -923,6 +924,24 @@ def test_will_return_thesaurus_with_the_defaul_order_as_0(self):
         #  will check if the second element of the tuple is the thesaurus_id = 1
         self.assertEqual(fields[1][0], '2')
 
+    def test_get_thesuro_key_label_with_cmd_language_code(self):
+        # in python test language code look like 'en' this test checks if key label result function
+        # returns correct results
+        tid = 1
+        translation.activate("en")
+        t_available_form = ThesaurusAvailableForm(data={"1": tid})
+        results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language())
+        self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR)
+
+    def test_get_thesuro_key_label_with_browser_language_code(self):
+        # in browser scenario language does not look like "it", but rather include coutry code
+        # like "it-it" this test checks if _get_thesauro_keyword_label can handle this
+        tid = 1
+        translation.activate("en-us")
+        t_available_form = ThesaurusAvailableForm(data={"1": tid})
+        results = t_available_form._get_thesauro_keyword_label(tid, translation.get_language())
+        self.assertNotEqual(results[1], THESAURUS_RESULT_LIST_SEPERATOR)
+
 
 class TestFacets(TestCase):
 

Diff for: geonode/base/utils.py

@@ -189,3 +189,16 @@ def validate_extra_metadata(data, instance):
             raise ValidationError(f"{e} at index {_index} for input json: {json.dumps(_metadata)}")
     # conerted because in this case, we can store a well formated json instead of the user input
     return data
+
+
+@staticmethod
+def remove_country_from_lanugecode(language: str):
+    """ Remove country code (us) from language name (en-us)
+    >>> remove_country_from_lanugecode("en-us")
+    'en'
+    """
+    if "-" not in language:
+        return language
+
+    lang, _, _ = language.lower().partition("-")
+    return lang

Diff for: geonode/base/views.py

@@ -35,6 +35,7 @@
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.urls import reverse
+from django.utils.translation import get_language
 
 # Geonode dependencies
 from geonode.maps.models import Map
@@ -47,7 +48,7 @@
 from geonode.resource.manager import resource_manager
 from geonode.security.utils import get_visible_resources
 from geonode.notifications_helper import send_notification
-from geonode.base.utils import OwnerRightsRequestViewUtils
+from geonode.base.utils import OwnerRightsRequestViewUtils, remove_country_from_lanugecode
 from geonode.base.forms import UserAndGroupPermissionsForm
 
 from geonode.base.forms import (
@@ -301,23 +302,25 @@ def get_results(self, context):
 
 class ThesaurusAvailable(autocomplete.Select2QuerySetView):
     def get_queryset(self):
+
         tid = self.request.GET.get("sysid")
-        lang = self.request.GET.get("lang")
-        qs_local = []
-        qs_non_local = []
-        for key in ThesaurusKeyword.objects.filter(thesaurus_id=tid):
-            label = ThesaurusKeywordLabel.objects.filter(keyword=key).filter(lang=lang)
-            if self.q:
-                label = label.filter(label__icontains=self.q)
-            if label.exists():
-                qs_local.append(label.get())
-            else:
-                if self.q in key.alt_label:
-                    qs_non_local.append(key)
-                elif not self.q:
-                    qs_non_local.append(key)
-
-        return qs_non_local + qs_local
+        lang = get_language()
+        keyword_id_for_given_thesaurus = ThesaurusKeyword.objects.filter(thesaurus_id=tid)
+
+        # try find results found for given language e.g. (en-us) if no results found remove country code from language to (en) and try again
+        qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
+        if len(qs_keyword_ids) == 0:
+            lang = remove_country_from_lanugecode(lang)
+            qs_keyword_ids = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus).values("keyword_id")
+
+        not_qs_ids = ThesaurusKeywordLabel.objects.exclude(keyword_id__in=qs_keyword_ids).order_by("keyword_id").distinct("keyword_id").values("keyword_id")
+        qs = ThesaurusKeywordLabel.objects.filter(lang=lang, keyword_id__in=keyword_id_for_given_thesaurus)
+        if self.q:
+            qs = qs.filter(label__istartswith=self.q)
+
+        qs_local = list(qs)
+        qs_non_local = list(keyword_id_for_given_thesaurus.filter(id__in=not_qs_ids))
+        return qs_local + qs_non_local
 
     def get_results(self, context):
         return [