[Fixes #8913] Duplicate keyword error raised on massive upload (#8933)

GeoNode · Mar 15, 2022 · ab8d7d5 · ab8d7d5
1 parent 166f598
commit ab8d7d5
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 12 deletions.
diff --git a/geonode/base/models.py b/geonode/base/models.py
@@ -25,7 +25,7 @@
 import logging
 import traceback
 
-from django.db import models
+from django.db import models, transaction
 from django.conf import settings
 from django.utils.functional import cached_property
 from django.utils.html import escape
@@ -465,13 +465,22 @@ def add(self, *tags, through_defaults=None, tag_kwargs=None):
         tag_objs = set(tags) - str_tags
         # If str_tags has 0 elements Django actually optimizes that to not do a
         # query.  Malcolm is very smart.
-        existing = self.through.tag_model().objects.filter(
+        '''
+        To avoid concurrency with the keyword in case of a massive upload.
+        With the transaction block and the select_for_update,
+        we can easily handle the concurrency.
+        DOC: https://docs.djangoproject.com/en/3.2/ref/models/querysets/#select-for-update
+        '''
+        existing = self.through.tag_model().objects.select_for_update().filter(
             name__in=str_tags, **tag_kwargs
         )
         tag_objs.update(existing)
         new_ids = set()
-        for new_tag in str_tags - set(t.name for t in existing):
-            if new_tag:
+        with transaction.atomic():
+            tag_objs.update(existing)
+            new_ids = set()
+            _new_keyword = str_tags - set(t.name for t in existing)
+            for new_tag in list(_new_keyword):
                 new_tag = escape(new_tag)
                 new_tag_obj = HierarchicalKeyword.add_root(name=new_tag)
                 tag_objs.add(new_tag_obj)

diff --git a/geonode/upload/utils.py b/geonode/upload/utils.py
@@ -720,18 +720,17 @@ def is_thesaurus_available(thesaurus, keyword):
 
     def _set_free_keyword(self, keywords):
         if len(keywords) > 0:
-            if not self.instance.keywords:
-                self.instance.keywords = keywords
-            else:
-                self.instance.keywords.add(*keywords)
+            if self.instance.keywords.exists():
+                self.instance.keywords.clear()
+
+            self.instance.keywords.add(*keywords)
         return keywords
 
     def _set_tkeyword(self, tkeyword):
         if len(tkeyword) > 0:
-            if not self.instance.tkeywords:
-                self.instance.tkeywords = tkeyword
-            else:
-                self.instance.tkeywords.add(*tkeyword)
+            if self.instance.tkeywords.exists():
+                self.instance.tkeywords.clear()
+            self.instance.tkeywords.add(*tkeyword)
         return [t.alt_label for t in tkeyword]